Fetch Authentication headers with Selenium

Makefile

@@ -1,26 +1,46 @@
+CHROMEDRIVER_VERSION := 2.41
+SERVERLESS_CHROME_VERSION := v1.0.0-53
+
+TF_BUILD_DIR := ./terraform/build
+
 init:
 	terraform init terraform/
 
-plan: lambda/vendor
+plan: lambda/vendor $(TF_BUILD_DIR)/chromedriver.zip
 	terraform plan terraform/
 
-apply: lambda/vendor
+apply: lambda/vendor $(TF_BUILD_DIR)/chromedriver.zip
 	terraform apply terraform/
 
 deploy: clean apply
 
 lint:
-	flake8 --max-line-length=120 lambda/*.py
+	flake8 --max-line-length=120 lambda/src
 
 test:
 	cd lambda/src; python -m unittest discover ../tests
 
 lambda/vendor: lambda/requirements.txt
-	pip install -r lambda/requirements.txt -t lambda/vendor/python
+	sudo docker run --rm -v "$(shell pwd)":/var/task "public.ecr.aws/sam/build-python3.6" /bin/sh -c "pip install -U pip && pip install -r lambda/requirements.txt -t lambda/vendor/python/; exit"
+	sudo chown -R $(shell id -u):$(shell id -g) lambda/vendor
+
+
+$(TF_BUILD_DIR)/chromedriver.zip:
+	@mkdir -p $(TF_BUILD_DIR)
+
+	curl -SL https://chromedriver.storage.googleapis.com/$(CHROMEDRIVER_VERSION)/chromedriver_linux64.zip > $(TF_BUILD_DIR)/chromedriver.zip
+	unzip $(TF_BUILD_DIR)/chromedriver.zip -d $(TF_BUILD_DIR)/
+	perl -pi -e 's/cdc_/swa_/g' $(TF_BUILD_DIR)/chromedriver
+
+	curl -SL https://github.com/adieuadieu/serverless-chrome/releases/download/$(SERVERLESS_CHROME_VERSION)/stable-headless-chromium-amazonlinux-2017-03.zip > $(TF_BUILD_DIR)/headless-chromium.zip
+	unzip $(TF_BUILD_DIR)/headless-chromium.zip -d $(TF_BUILD_DIR)/
+
+	@rm $(TF_BUILD_DIR)/headless-chromium.zip $(TF_BUILD_DIR)/chromedriver.zip
+	cd $(TF_BUILD_DIR); zip -r chromedriver.zip chromedriver headless-chromium
 
 clean:
 	-rm -rf lambda/vendor/
-	-rm -rf terraform/build/
+	-rm -rf $(TF_BUILD_DIR)/
 	-find . -type f -name '*.pyc' -delete
 
 .PHONY: init plan apply deploy lint test clean

lambda/requirements-dev.txt

@@ -1,5 +1,5 @@
 -r requirements.txt
-boto3==1.12.21
+boto3==1.20.26
 mock==2.0.0
-flake8==3.5.0
+flake8==4.0.1
 vcrpy==4.0.2

lambda/requirements.txt

@@ -1,2 +1,36 @@
+async-generator==1.10
+attrs==21.2.0
+blinker==1.4
+Brotli==1.0.9
+certifi==2021.10.8
+cffi==1.15.0
+chardet==3.0.4
+charset-normalizer==2.0.9
+cryptography==36.0.1
+h11==0.12.0
+h2==4.1.0
+hpack==4.0.0
+hyperframe==6.0.1
+idna==2.8
+kaitaistruct==0.9
+names==0.3.0
+outcome==1.1.0
 pendulum==2.1.2
-requests==2.21.0
+pyasn1==0.4.8
+pycparser==2.21
+pyOpenSSL==21.0.0
+pyparsing==3.0.6
+PySocks==1.7.1
+python-dateutil==2.8.2
+pytzdata==2020.1
+requests==2.26.0
+selenium==3.14.0
+selenium-wire==4.5.6
+six==1.16.0
+sniffio==1.2.0
+sortedcontainers==2.4.0
+trio==0.19.0
+trio-websocket==0.9.2
+urllib3==1.26.7
+wsproto==1.0.0
+zstandard==0.16.0

lambda/src/exceptions.py

@@ -2,5 +2,10 @@ class SouthwestAPIError(Exception):
     pass
 
 
-class ReservationNotFoundError(Exception):
+class ReservationNotFoundError(SouthwestAPIError):
+    pass
+
+
+class TooManyRequestsError(SouthwestAPIError):
+    "This error is thrown when the authentication headers aren't included or they have expired"
     pass

lambda/src/handlers/__init__.py

@@ -1,4 +1,5 @@
-from .receive_email import main as receive_email
-from .schedule_check_in import main as schedule_check_in
-from .check_in import main as check_in
-from .check_in_failure import main as check_in_failure
+from .receive_email import main as receive_email  # noqa: F401
+from .schedule_check_in import main as schedule_check_in  # noqa: F401
+from .check_in import main as check_in  # noqa: F401
+from .check_in_failure import main as check_in_failure  # noqa: F401
+from .update_headers import main as update_headers  # noqa: F401

lambda/src/handlers/check_in.py

@@ -1,7 +1,10 @@
 import logging
-import sys
 
-import swa, exceptions, mail
+import exceptions
+import mail
+import swa
+
+from parameters import get_headers
 
 # Set up logging
 log = logging.getLogger(__name__)
@@ -37,7 +40,7 @@ def main(event, context):
     ))
 
     try:
-        resp = swa.check_in(first_name, last_name, confirmation_number)
+        resp = swa.check_in(first_name, last_name, confirmation_number, get_headers())
         log.info("Checked in successfully!")
         log.debug("Check-in response: {}".format(resp))
     except exceptions.ReservationNotFoundError:

lambda/src/handlers/schedule_check_in.py

@@ -1,7 +1,7 @@
-import json
 import logging
 
-import swa, mail
+import mail
+import swa
 
 # Set up logging
 log = logging.getLogger(__name__)

lambda/src/handlers/update_headers.py

@@ -0,0 +1,83 @@
+import json
+import random
+import re
+import string
+import time
+
+import boto3
+import names
+
+from seleniumwire import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+
+def main(event, context):
+    """
+    This function will update authentication headers for the API by running
+    a headless Chromium session and pulling the correct values
+    """
+
+    confirmation_number = ''.join(random.choices(string.ascii_uppercase, k=6))
+    first_name = names.get_first_name()
+    last_name = names.get_last_name()
+
+    options = Options()
+    options.binary_location = '/opt/headless-chromium'
+    options.headless = True
+    options.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) '
+                         'AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15')
+    options.add_argument('--no-sandbox')
+    options.add_argument('--disable-gpu-sandbox')
+    options.add_argument('--disable-dev-shm-usage')
+    options.add_argument('--disable-gpu')
+    options.add_argument("--single-process")
+    options.add_argument("--homedir=/tmp")
+    options.add_argument('window-size=1920x1080')
+
+    driver = webdriver.Chrome(
+        "/opt/chromedriver",
+        options=options,
+        # fix issue if we don't have permissions for default storage location
+        seleniumwire_options={'request_storage': 'memory'}
+    )
+    driver.scopes = ["page/check-in"]
+    driver.get("https://mobile.southwest.com/check-in")
+
+    # fill out the form once the form fields become available
+    element = WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.NAME, "recordLocator")))
+    element.send_keys(confirmation_number)
+
+    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.NAME, "firstName"))).send_keys(first_name)
+    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.NAME, "lastName"))).send_keys(last_name)
+
+    element.submit()
+
+    while len(driver.requests) == 0:
+        print("Waiting for headers...")
+        time.sleep(1)
+
+    headers = {
+        k: v for k, v in driver.requests[0].headers.items()
+        if re.match(r"x-api-key|x-user-experience-id|x-channel-id|^[\w-]+?-\w$", k, re.I)
+    }
+
+    print(headers)
+
+    if not headers:
+        driver.quit()
+        raise SystemExit("Failed to retrieve headers")
+
+    ssm = boto3.client('ssm')
+    ssm.put_parameter(
+        Name='/southwest/headers',
+        Value=json.dumps(headers),
+        Type='String',
+        Overwrite=True,
+        # use Advanced tier because the headers can be > 4k
+        Tier='Advanced'
+    )
+
+    driver.quit()

lambda/src/mail.py

@@ -237,8 +237,7 @@ def find_name_and_confirmation_number(msg):
         fname = None
 
     if not all([fname, lname, reservation]):
-        raise exceptions.ReservationNotFoundError("Unable to find reservation "
-            "in email id {}".format(msg.message_id))
+        raise exceptions.ReservationNotFoundError("Unable to find reservation in email id {}".format(msg.message_id))
 
     log.info("Passenger: {} {}, Confirmation Number: {}".format(
         fname, lname, reservation))

lambda/src/parameters.py

@@ -0,0 +1,9 @@
+import json
+
+import boto3
+
+
+def get_headers():
+    ssm = boto3.client('ssm')
+    result = ssm.get_parameter(Name='/southwest/headers')
+    return json.loads(result['Parameter']['Value'])

lambda/src/swa.py

@@ -3,46 +3,51 @@
 # Functions for interacting with the Southwest API
 #
 
-import codecs
-
 from urllib.parse import urlencode
 
 import pendulum
 import requests
 
 import exceptions
 
-USER_AGENT = "SouthwestAndroid/7.2.1 android/10"
-# This is not a secret, but obfuscate it to prevent detection
-API_KEY = codecs.decode("y7kk8389n5on9ro24nr68onq068oq1860osp", "rot13")
+# This is the same User-Agent which update_headers uses
+# TODO: Move to a constants file or something so we're not duplicating
+USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 "
+    "(KHTML, like Gecko) Version/13.0.5 Safari/605.1.15"
+)
 
 
-def _make_request(method, page, data='', check_status_code=True):
+def _make_request(method, page, data='', check_status_code=True, headers=None):
     url = f"https://mobile.southwest.com/api/{page}"
-    headers = {
+    _headers = {
         "User-Agent": USER_AGENT,
-        "X-API-Key": API_KEY,
-        "X-Channel-ID": "MWEB",
         "Accept": "application/json"
     }
+
+    if headers:
+        _headers.update(headers)
+
     method = method.lower()
 
     if method == 'get':
-        response = requests.get(url, headers=headers, params=urlencode(data))
+        response = requests.get(url, headers=_headers, params=urlencode(data))
     elif method == 'post':
-        headers['Content-Type'] = 'application/json'
-        response = requests.post(url, headers=headers, json=data)
+        _headers['Content-Type'] = 'application/json'
+        response = requests.post(url, headers=_headers, json=data)
     else:
         raise NotImplementedError()
 
     if check_status_code and not response.ok:
         try:
             msg = response.json()["message"]
-        except:
+        except Exception:
             msg = response.reason
 
         if response.status_code == 404:
             raise exceptions.ReservationNotFoundError()
+        if response.status_code == 429:
+            raise exceptions.TooManyRequestsError()
 
         raise exceptions.SouthwestAPIError("status_code={} msg=\"{}\"".format(
             response.status_code, msg))
@@ -86,9 +91,8 @@ def _get_check_in_time(self, departure_time):
         the checkin time to allow for some clock skew buffer.
         """
         return pendulum.parse(departure_time)\
-                .subtract(days=1)\
-                .add(seconds=self.check_in_seconds)
-
+            .subtract(days=1)\
+            .add(seconds=self.check_in_seconds)
 
     def get_check_in_times(self, expired=False):
         """
@@ -118,12 +122,12 @@ def check_in_times(self):
         return self.get_check_in_times()
 
 
-def check_in(first_name, last_name, confirmation_number):
+def check_in(first_name, last_name, confirmation_number, headers):
     # first we get a session token with a GET request, then issue a POST to check in
     page = "mobile-air-operations/v1/mobile-air-operations/page/check-in"
     params = {'first-name': first_name, 'last-name': last_name}
 
-    session = _make_request("get", page + "/" + confirmation_number, params)
+    session = _make_request("get", page + "/" + confirmation_number, params, headers=headers)
     sessionj = session.json()
 
     try:
@@ -133,7 +137,7 @@ def check_in(first_name, last_name, confirmation_number):
         print(sessionj)
         raise exceptions.SouthwestAPIError("Error getting check-in session")
 
-    response = _make_request("post", page, body)
+    response = _make_request("post", page, body, headers=headers)
     if not response.ok:
         raise exceptions.SouthwestAPIError("Error checking in! response={}".format(response))
 

terraform/cloudwatch.tf

@@ -57,3 +57,32 @@ resource "aws_cloudwatch_metric_alarm" "checkin_receive_email_errors" {
   alarm_actions = [aws_sns_topic.admin_notifications.arn]
 }
 
+resource "aws_cloudwatch_metric_alarm" "update_headers_errors" {
+  alarm_name          = "update-headers-error"
+  alarm_description   = "Monitor the ${aws_lambda_function.sw_update_headers.function_name} Lambda for errors"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = "1"
+  metric_name         = "Errors"
+  namespace           = "AWS/Lambda"
+  period              = "300"
+  statistic           = "Maximum"
+  threshold           = "1"
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    FunctionName = aws_lambda_function.sw_update_headers.function_name
+  }
+
+  alarm_actions = [aws_sns_topic.admin_notifications.arn]
+}
+
+resource "aws_cloudwatch_event_rule" "update_headers" {
+  name                = "sw-update-headers"
+  description         = "Scheduled task to update headers for Southwest API interactions"
+  schedule_expression = "rate(3 hours)"
+}
+
+resource "aws_cloudwatch_event_target" "update_headers" {
+  rule = aws_cloudwatch_event_rule.update_headers.name
+  arn  = aws_lambda_function.sw_update_headers.arn
+}