Header options (#49)

* Configure session not to trust env (#48)

don't pick up .netrc dummy creds

* Handle extra headers from config file or cmd-line

* Set correct order in dict merge

    Command line should overwrite config/creds files

* Update docs for custom header option

* Update _version.py

bump version

* Fix naming for extra headers option

Author: jeffakolb

README.rst

@@ -103,6 +103,16 @@ default location (``.twitter_keys.yaml``) or in environment variables.
     --filename-prefix beyonce_geo \
     --no-print-stream
 
+One or more custom headers can be specified from the command line,
+using the ``--extra-headers`` argument and a JSON-formatted string
+representing a dictionary of extra headers:
+
+.. code:: bash
+
+  search_tweets.py \
+    --filter-rule "beyonce has:hashtags" \
+    --extra-headers '{"<MY_HEADER_KEY>":"<MY_HEADER_VALUE>"}'
+
 
 Options can be passed via a configuration file (either ini or YAML). Example
 files can be found in the ``tools/api_config_example.config`` or
@@ -142,13 +152,26 @@ Or this:
       filename_prefix: kanye
       results_per_file: 10000000
 
+Custom headers can be specified in a config file, under a specific credentials 
+key: 
+
+.. code:: yaml
+
+  search_tweets_api:
+    account_type: premium
+    endpoint: <FULL_URL_OF_ENDPOINT>
+    username: <USERNAME>
+    password: <PW>
+    extra_headers:
+      <MY_HEADER_KEY>: <MY_HEADER_VALUE>
 
 When using a config file in conjunction with the command-line utility, you need
 to specify your config file via the ``--config-file`` parameter. Additional
 command-line arguments will either be *added* to the config file args or
 **overwrite** the config file args if both are specified and present.
 
 
+
 Example::
 
   search_tweets.py \
@@ -175,7 +198,8 @@ Full options are listed below:
                         [--max-results MAX_RESULTS] [--max-pages MAX_PAGES]
                         [--results-per-file RESULTS_PER_FILE]
                         [--filename-prefix FILENAME_PREFIX]
-                        [--no-print-stream] [--print-stream] [--debug]
+                        [--no-print-stream] [--print-stream]
+                        [--extra-headers EXTRA_HEADERS] [--debug]
 
   optional arguments:
     -h, --help            show this help message and exit
@@ -224,7 +248,10 @@ Full options are listed below:
                           prefix for the filename where tweet json data will be
                           stored.
     --no-print-stream     disable print streaming
-    --print-stream        Print tweet stream to stdout
+    --print-stream        Print tweet stream to stdout 
+    --extra-headers EXTRA_HEADERS
+                          JSON-formatted str representing a dict of additional
+                          request headers
     --debug               print all info and warning messages
 
 

examples/base_readme.rst

@@ -112,29 +112,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 5,
+   "metadata": {},
    "outputs": [],
    "source": [
     "from searchtweets import load_credentials"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{'endpoint': '<MY_ENDPOINT>',\n",
+       "{'username': '<MY_USERNAME>',\n",
        " 'password': '<MY_PASSWORD>',\n",
-       " 'username': '<MY_USERNAME>'}"
+       " 'endpoint': '<MY_ENDPOINT>'}"
       ]
      },
-     "execution_count": 14,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -147,17 +145,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
        "{'bearer_token': '<A_VERY_LONG_MAGIC_STRING>',\n",
-       " 'endpoint': 'https://api.twitter.com/1.1/tweets/search/30day/dev.json'}"
+       " 'endpoint': 'https://api.twitter.com/1.1/tweets/search/30day/dev.json',\n",
+       " 'extra_headers_dict': None}"
       ]
      },
-     "execution_count": 15,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -179,7 +178,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -193,12 +192,12 @@
     {
      "data": {
       "text/plain": [
-       "{'endpoint': '<https://endpoint>',\n",
+       "{'username': '<ENV_USERNAME>',\n",
        " 'password': '<ENV_PW>',\n",
-       " 'username': '<ENV_USERNAME>'}"
+       " 'endpoint': '<https://endpoint>'}"
       ]
      },
-     "execution_count": 16,
+     "execution_count": 8,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -226,6 +225,13 @@
     "\n",
     "are used to control credential behavior from the command-line app. "
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -244,7 +250,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.1"
+   "version": "3.7.0"
   }
  },
  "nbformat": 4,

examples/credential_handling.ipynb

@@ -137,9 +137,9 @@ library.
 
 ::
 
-    {'endpoint': '<MY_ENDPOINT>',
+    {'username': '<MY_USERNAME>',
      'password': '<MY_PASSWORD>',
-     'username': '<MY_USERNAME>'}
+     'endpoint': '<MY_ENDPOINT>'}
 
 
 
@@ -155,7 +155,8 @@ library.
 ::
 
     {'bearer_token': '<A_VERY_LONG_MAGIC_STRING>',
-     'endpoint': 'https://api.twitter.com/1.1/tweets/search/30day/dev.json'}
+     'endpoint': 'https://api.twitter.com/1.1/tweets/search/30day/dev.json',
+     'extra_headers_dict': None}
 
 
 
@@ -185,9 +186,9 @@ regardless of a YAML file's validity or existence.
 
 ::
 
-    {'endpoint': '<https://endpoint>',
+    {'username': '<ENV_USERNAME>',
      'password': '<ENV_PW>',
-     'username': '<ENV_USERNAME>'}
+     'endpoint': '<https://endpoint>'}
 
 
 
@@ -201,3 +202,4 @@ the flags:
 -  ``--env-overwrite``
 
 are used to control credential behavior from the command-line app.
+

examples/credential_handling.rst

@@ -2,4 +2,4 @@
 # Copyright 2018 Twitter, Inc.
 # Licensed under the MIT License
 # https://opensource.org/licenses/MIT
-VERSION = "1.7.2"
+VERSION = "1.7.4"

searchtweets/_version.py

@@ -171,6 +171,7 @@ def intify(arg):
              "username": config_dict.get("username"),
              "password": config_dict.get("password"),
              "bearer_token": config_dict.get("bearer_token"),
+             "extra_headers_dict": config_dict.get("extra_headers_dict",None),
              "rule_payload": rule,
              "results_per_file": intify(config_dict.get("results_per_file")),
              "max_results": intify(config_dict.get("max_results")),

searchtweets/api_utils.py

@@ -92,7 +92,8 @@ def _parse_credentials(search_creds, account_type):
 
             search_args = {
                 "bearer_token": search_creds["bearer_token"],
-                "endpoint": search_creds["endpoint"]}
+                "endpoint": search_creds["endpoint"],
+                "extra_headers_dict": search_creds.get("extra_headers",None)}
         if account_type == "enterprise":
             search_args = {"username": search_creds["username"],
                            "password": search_creds["password"],
@@ -123,6 +124,8 @@ def load_credentials(filename=None, account_type=None,
           consumer_secret: <SECRET>
           bearer_token: <TOKEN>
           account_type: <enterprise OR premium>
+          extra_headers: 
+            <MY_HEADER_KEY>: <MY_HEADER_VALUE>
 
     with the appropriate fields filled out for your account. The top-level key
     defaults to ``search_tweets_api`` but can be flexible.

searchtweets/credentials.py

@@ -28,7 +28,7 @@
 logger = logging.getLogger(__name__)
 
 
-def make_session(username=None, password=None, bearer_token=None):
+def make_session(username=None, password=None, bearer_token=None, extra_headers_dict=None): 
     """Creates a Requests Session for use. Accepts a bearer token
     for premiums users and will override username and password information if
     present.
@@ -56,6 +56,8 @@ def make_session(username=None, password=None, bearer_token=None):
         logger.info("using username and password for authentication")
         session.auth = username, password
         session.headers = headers
+    if extra_headers_dict:
+        headers.update(extra_headers_dict) 
     return session
 
 
@@ -144,7 +146,9 @@ class ResultStream:
             tweet parser library to convert each raw tweet package to a Tweet
             with lazy properties.
         max_requests (int): A hard cutoff for the number of API calls this
-        instance will make. Good for testing in sandbox premium environments.
+        instance will make. Good for testing in sandbox premium environments. 
+        extra_headers_dict (dict): custom headers to add
+
 
     Example:
         >>> rs = ResultStream(**search_args, rule_payload=rule, max_pages=1)
@@ -156,12 +160,13 @@ class ResultStream:
     session_request_counter = 0
 
     def __init__(self, endpoint, rule_payload, username=None, password=None,
-                 bearer_token=None, max_results=500,
+                 bearer_token=None, extra_headers_dict=None, max_results=500,
                  tweetify=True, max_requests=None, **kwargs):
 
         self.username = username
         self.password = password
         self.bearer_token = bearer_token
+        self.extra_headers_dict = extra_headers_dict
         if isinstance(rule_payload, str):
             rule_payload = json.loads(rule_payload)
         self.rule_payload = rule_payload
@@ -229,7 +234,8 @@ def init_session(self):
             self.session.close()
         self.session = make_session(self.username,
                                     self.password,
-                                    self.bearer_token)
+                                    self.bearer_token,
+                                    self.extra_headers_dict)
 
     def check_counts(self):
         """

searchtweets/result_stream.py

@@ -126,6 +126,12 @@ def parse_cmd_args():
                            default=True,
                            help="Print tweet stream to stdout")
 
+    argparser.add_argument("--extra-headers",
+                           dest="extra_headers",
+                           type=str,
+                           default=None,
+                           help="JSON-formatted str representing a dict of additional request headers")
+
     argparser.add_argument("--debug",
                            dest="debug",
                            action="store_true",
@@ -149,6 +155,11 @@ def main():
         configfile_dict = read_config(args_dict["config_filename"])
     else:
         configfile_dict = {}
+    
+    extra_headers_str = args_dict.get("extra_headers")
+    if extra_headers_str is not None:
+        args_dict['extra_headers_dict'] = json.loads(extra_headers_str)
+        del args_dict['extra_headers']
 
     logger.debug("config file ({}) arguments sans sensitive args:".format(args_dict["config_filename"]))
     logger.debug(json.dumps(_filter_sensitive_args(configfile_dict), indent=4))
@@ -161,8 +172,8 @@ def main():
     dict_filter = lambda x: {k: v for k, v in x.items() if v is not None}
 
     config_dict = merge_dicts(dict_filter(configfile_dict),
-                              dict_filter(args_dict),
-                              dict_filter(creds_dict))
+                              dict_filter(creds_dict),
+                              dict_filter(args_dict))
 
     logger.debug("combined dict (cli, config, creds) sans password:")
     logger.debug(json.dumps(_filter_sensitive_args(config_dict), indent=4))