add logging

Author: chrisjsimpson

.env.example

@@ -1,3 +1,9 @@
+PYTHON_LOG_LEVEL=DEBUG
+EMAIL_LOG_LEVEL=ERROR
+TELEGRAM_PYTHON_LOG_LEVEL=ERROR
+TELEGRAM_TOKEN=123456789:ABCDEFG_HIJKLMN_OPQ
+TELEGRAM_CHAT_ID=-87654321
+
 GITHUB_OAUTH_CLIENT_ID=changeme-get-from-github
 GITHUB_OAUTH_CLIENT_SECRET=changeme-get-from-github
 GITHUB_OAUTH_REDIRECT_URI="https://example.com/githubcallback"
@@ -8,3 +14,13 @@ SSH_PRIVATE_KEY=test
 BASE_PATH=/home/username/path/
 CONTAINER_HOSTING_SSH_SETUP_HANDLER_API_KEY=secret
 DOKKU_WRAPPER_FULL_PATH=/home/dokku/dokku-wrapper.py
+
+LOGGING_STMP_MAILHOST_HOST=email.example.com
+LOGGING_SMTP_MAILHOST_PORT=25
+LOGGING_SMTP_FROMADDR=alerts@example.com
+LOGGING_SMTP_TOADDRS=alerts@example.com
+LOGGING_SMTP_SUBJECT="Log alert"
+LOGGING_SMTP_CREDENTIALS_EMAIL=alerts@example.com
+LOGGING_SMTP_CREDENTIALS_PASSWORD=changeme
+LOGGING_SMTP_SECURE=()
+LOGGING_SMTP_TIMEOUT=3

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+ignore = E501

TelegramHTTPHandler.py

@@ -0,0 +1,26 @@
+from logging.handlers import HTTPHandler
+import requests
+
+
+class TelegramHTTPHandler(HTTPHandler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def emit(self, record):
+        """
+        Emit log record to Telegram
+        """
+        try:
+            host = self.host
+            if self.secure:
+                scheme = "https://"
+            else:
+                scheme = "http://"
+            url = self.url
+            data = str(self.mapLogRecord(record))[
+                0:4000
+            ]  # Truncate to max Telegram message size
+            url = f"{url}{data}"
+            requests.get(scheme + host + "/" + url)
+        except Exception:
+            self.handleError(record)

app.py

@@ -6,6 +6,7 @@
 from starlette.responses import HTMLResponse
 from starlette.requests import Request
 
+from logger import logger
 
 from cryptography.hazmat.primitives.asymmetric import rsa
 from cryptography.hazmat.primitives import serialization
@@ -23,6 +24,8 @@
 from dotenv import load_dotenv
 import subprocess
 
+log = logger
+
 load_dotenv(verbose=True)
 
 templates = Jinja2Templates(directory="templates")
@@ -52,7 +55,7 @@ def amber_encrypt(key: str, value: str, amber_file_location="./amber.yaml"):
 def generate_ssh_keys():
     """Generate public/private ssh keys
 
-    See also https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/
+    See also https://cryptography.io/en/latest/hazmat/primitives/asymmetric/rsa/ # noqa: E501
     """
     key = rsa.generate_private_key(
         public_exponent=65537,
@@ -90,9 +93,7 @@ def encrypt_github_secret(public_key: str, secret_value: str) -> str:
 
 async def homepage(request):
     """Homepage and display connect Github link"""
-    body = await request.body()
-    print(body)
-
+    log.debug("Homepage")
     client_id = GITHUB_OAUTH_CLIENT_ID
     state = f"{secrets.token_urlsafe(30)}---no-framework"
     state_rails = f"{secrets.token_urlsafe(30)}---rails"
@@ -181,7 +182,7 @@ async def githubcallback(request):
         "https://api.github.com/user/repos", headers=headers, data=json.dumps(data)
     )
     repo_url = req.json()["html_url"]
-    print(repo_url)
+    log.info(f"New container host started: {repo_url}")
 
     # Create repo secrets
     req = requests.get(
@@ -197,7 +198,7 @@ def github_store_secret(SECRET_NAME, SECRET_VALUE: str):
             "encrypted_value": secret_Encrypted,
             "key_id": github_repo_public_key_id,
         }
-        req = requests.put(
+        req = requests.put(  # noqa: 203
             f"https://api.github.com/repos/{username}/{repo_name}/actions/secrets/{SECRET_NAME}",
             headers=headers,
             data=json.dumps(data),
@@ -465,6 +466,8 @@ def github_store_secret(SECRET_NAME, SECRET_VALUE: str):
     origin = repo.remotes[0]
     repo.heads.main.set_tracking_branch(origin.refs.main)
     fetch = origin.fetch()[0]
+    log.info(fetch)
+    log.info("Oh no")
 
     # Commit amber.yaml secrets file to repo
     index = repo.index
@@ -532,7 +535,8 @@ async def blog(request):
 
 
 async def health(request):
-    print(request)
+    log.debug(request)
+    log.error("Testing error logging is working")
     return PlainTextResponse("OK")
 
 

logger.py

@@ -0,0 +1,117 @@
+import logging
+import coloredlogs
+from logging.handlers import QueueHandler, QueueListener, SMTPHandler
+from TelegramHTTPHandler import TelegramHTTPHandler
+import os
+import sys
+import queue
+from dotenv import load_dotenv
+
+load_dotenv(verbose=True)
+
+LOGGING_SMTP_MAILHOST_PORT = int(os.getenv("LOGGING_SMTP_MAILHOST_PORT"))
+LOGGING_SMTP_MAILHOST_HOST = os.getenv("LOGGING_SMTP_MAILHOST_HOST")
+LOGGING_SMTP_MAILHOST = (LOGGING_SMTP_MAILHOST_HOST, LOGGING_SMTP_MAILHOST_PORT)
+LOGGING_SMTP_FROMADDR = os.getenv("LOGGING_SMTP_FROMADDR")
+LOGGING_SMTP_TOADDRS = os.getenv("LOGGING_SMTP_TOADDRS")
+LOGGING_SMTP_SUBJECT = os.getenv("LOGGING_SMTP_SUBJECT")
+LOGGING_SMTP_CREDENTIALS_EMAIL = os.getenv("LOGGING_SMTP_CREDENTIALS_EMAIL")
+LOGGING_SMTP_CREDENTIALS_PASSWORD = os.getenv("LOGGING_SMTP_CREDENTIALS_PASSWORD")
+LOGGING_SMTP_CREDENTIALS = (
+    LOGGING_SMTP_CREDENTIALS_EMAIL,
+    LOGGING_SMTP_CREDENTIALS_PASSWORD,
+)
+LOGGING_SMTP_SECURE = os.getenv("LOGGING_SMTP_SECURE")
+if LOGGING_SMTP_SECURE == "()":
+    LOGGING_SMTP_SECURE = ()
+else:
+    LOGGING_SMTP_SECURE = None
+
+LOGGING_SMTP_TIMEOUT = int(os.getenv("LOGGING_SMTP_TIMEOUT"))
+
+
+PYTHON_LOG_LEVEL = os.getenv("PYTHON_LOG_LEVEL", "DEBUG")
+TELEGRAM_PYTHON_LOG_LEVEL = os.getenv("TELEGRAM_PYTHON_LOG_LEVEL", "ERROR")
+EMAIL_LOG_LEVEL = os.getenv("EMAIL_LOG_LEVEL", "ERROR")
+
+logger = logging.getLogger()
+handler = logging.StreamHandler()  # sys.stderr will be used by default
+
+
+class RequestFormatter(coloredlogs.ColoredFormatter):
+    def format(self, record):
+        record.url = None
+        record.remote_addr = None
+
+        return super().format(record)
+
+
+formatter = RequestFormatter(
+    "[%(asctime)s] %(remote_addr)s requested %(url)s %(name)-12s %(levelname)-8s %(message)s %(funcName)s %(pathname)s:%(lineno)d"  # noqa
+)
+
+handler.setFormatter(formatter)
+handler.setLevel(
+    PYTHON_LOG_LEVEL
+)  # Both loggers and handlers have a setLevel method noqa: E501
+logger.addHandler(handler)
+logger.setLevel(PYTHON_LOG_LEVEL)
+
+
+# Log all uncuaght exceptions
+def handle_exception(exc_type, exc_value, exc_traceback):
+    if issubclass(exc_type, KeyboardInterrupt):
+        sys.__excepthook__(exc_type, exc_value, exc_traceback)
+        return
+
+    logger.critical(
+        "Uncaught exception", exc_info=(exc_type, exc_value, exc_traceback)
+    )  # noqa: E501
+
+
+sys.excepthook = handle_exception
+
+
+# Telegram logging
+if os.getenv("APP_ENV", None) != "development":
+    # See https://docs.python.org/3/howto/logging-cookbook.html#dealing-with-handlers-that-block # noqa
+    que = queue.Queue(-1)  # no limit on size
+    queue_handler = QueueHandler(que)
+
+    telegram_token = os.getenv("TELEGRAM_TOKEN", None)
+    telegram_chat_id = os.getenv("TELEGRAM_CHAT_ID", None)
+    telegramHandlerHost = "api.telegram.org"
+
+    telegramHandlerUrl = (
+        f"bot{telegram_token}/sendMessage?chat_id={telegram_chat_id}&text="
+    )
+
+    telegramHandler = TelegramHTTPHandler(
+        telegramHandlerHost, url=telegramHandlerUrl, secure=True
+    )
+    logger.info(
+        f"Setting TELEGRAM_PYTHON_LOG_LEVEL to {TELEGRAM_PYTHON_LOG_LEVEL}"
+    )  # noqa: E501
+    telegramHandler.setLevel(TELEGRAM_PYTHON_LOG_LEVEL)
+    listener = QueueListener(que, telegramHandler, respect_handler_level=True)
+    logger.addHandler(queue_handler)
+    listener.start()
+
+    SMTP_log_que = queue.Queue(-1)  # no limit on size
+    SMTP_queue_handler = QueueHandler(SMTP_log_que)
+
+    smtpHandler = SMTPHandler(
+        LOGGING_SMTP_MAILHOST,
+        LOGGING_SMTP_FROMADDR,
+        LOGGING_SMTP_TOADDRS,
+        LOGGING_SMTP_SUBJECT,
+        LOGGING_SMTP_CREDENTIALS,
+        LOGGING_SMTP_SECURE,
+        LOGGING_SMTP_TIMEOUT,
+    )
+    smtpHandler.setFormatter(formatter)
+    smtpHandler.setLevel(EMAIL_LOG_LEVEL)
+    SMTP_listener = QueueListener(SMTP_log_que, smtpHandler, respect_handler_level=True)
+
+    logger.addHandler(SMTP_queue_handler)
+    SMTP_listener.start()

requirements.txt

@@ -7,3 +7,4 @@ bcrypt
 cryptography
 PyNaCl
 GitPython
+coloredlogs