All posts All posts by this author Paper color Change page color Announcements

Modify and Add Custom Headers in selenium Chrome-driver

To get a solution for custom headers in Chrome-driver, googled a lot. But doesn't get any perfect solution. Then I write this solution myself & now I am sharing - if anyone has better solution or improvements please don't forget to comment.

First download the chromedriver and make sure to put it on executable search PATH.

$ get -c http://chromedriver.storage.googleapis.com/2.9/chromedriver_linux64.zip
$ unzip chromedriver_linux64.zip
$ sudo cp chromedriver /usr/bin/
$ sudo chmod 777 /usr/bin/chromedriver

Create a viarualenv & install all python package after activating the virtialenv

$ virtialenv env
$ source env/bin/activate
(env)$ pip install selenium
(env)$ pip install pyvirtualdisplay

Here goes the actual python code you are looking after.

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By



def create_modheaders_plugin(plugin_path=None, remove_headers=None, add_or_modify_headers=None):
    """Create modheaders extension

    kwargs:
        plugin_path (str): absolute plugin path
        remove_headers (list): headers name to remove
        add_or_modify_headers (dict): ie. {"Header-Name": "Header Value"}

    return str -> plugin path
    """
    import string
    import zipfile

    if plugin_path is None:
        plugin_path = '/tmp/vimm_chrome_modheaders_plugin.zip'

    if remove_headers is None:
        remove_headers = []

    if add_or_modify_headers is None:
        add_or_modify_headers = {}        

    assert isinstance(remove_headers, list), "remove_headers must be a list"
    assert isinstance(add_or_modify_headers, dict), "add_or_modify_headers must be dict"

    # only keeping the unique headers key in remove_headers list
    remove_headers = list(set(remove_headers))


    manifest_json = """
    {
        "version": "1.0.0",
        "manifest_version": 2,
        "name": "Chrome HeaderModV",
        "permissions": [
            "webRequest",
            "tabs",
            "unlimitedStorage",
            "storage",
            "<all_urls>",
            "webRequestBlocking"
        ],
        "background": {
            "scripts": ["background.js"]
        },
        "minimum_chrome_version":"22.0.0"
    }
    """

    background_js = string.Template(
    """
    function callbackFn(details) {
        var remove_headers = ${remove_headers};
        var add_or_modify_headers = ${add_or_modify_headers};

        function inarray(arr, obj) {
            return (arr.indexOf(obj) != -1);
        }

        // remove headers
        for (var i = 0; i < details.requestHeaders.length; ++i) {
            if (inarray(remove_headers, details.requestHeaders[i].name)) {
                details.requestHeaders.splice(i, 1);
                var index = remove_headers.indexOf(5);
                remove_headers.splice(index, 1);
            }
            if (!remove_headers.length) break;
        }

        // modify headers
        for (var i = 0; i < details.requestHeaders.length; ++i) {
            if (add_or_modify_headers.hasOwnProperty(details.requestHeaders[i].name)) {
                details.requestHeaders[i].value = add_or_modify_headers[details.requestHeaders[i].name];
                delete add_or_modify_headers[details.requestHeaders[i].name];
            }
        }

        // add modify
        for (var prop in add_or_modify_headers) {
            details.requestHeaders.push(
                {name: prop, value: add_or_modify_headers[prop]}
            );
        }

        return {requestHeaders: details.requestHeaders};
    }

    chrome.webRequest.onBeforeSendHeaders.addListener(
                callbackFn,
                {urls: ["<all_urls>"]},
                ['blocking', 'requestHeaders']
    );
    """
    ).substitute(
        remove_headers=remove_headers,
        add_or_modify_headers=add_or_modify_headers,
    )
    with zipfile.ZipFile(plugin_path, 'w') as zp:
        zp.writestr("manifest.json", manifest_json)
        zp.writestr("background.js", background_js)

    return plugin_path

mod_headers_plugin_path = create_modheaders_plugin(
    remove_headers=[],
    add_or_modify_headers={
        "User-Agent": 'Chrome driver',
        'X-Vimm': 'Vimmaniac Pvt. Ltd.'
    },
)


co = Options()
co.add_argument("--start-maximized")
co.add_extension(mod_headers_plugin_path)


from pyvirtualdisplay import Display
display = Display(visible=0, size=(800, 600))
display.start()


driver = webdriver.Chrome(chrome_options=co)
driver.get("http://httpbin.org/headers")
try:
    WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.TAG_NAME, "pre")))
except:
    import sys
    sys.exit("Can't fetech desired content")
else:
    print(driver.page_source.encode('utf-8'))
finally:
    driver.quit()

Save it as custom_headers.py

Don't forget to install Xvfb

$ sudo apt-get install xvfb

Now make custom_headers.py executable & run it. Thats all :)

Software Solutions