feat(python): catch all exceptions thrown through fastapi route paths

This commit does quite a bit:
- Catches unhandled exceptions raised in the route handler and
  produces a 500 Internal Server Error Arch-themed response.
- Each unhandled exception causes a notification to be sent to new
  `notifications.postmaster` email with a "Traceback ID."
- Traceback ID is logged to the server along with the traceback which
  caused the 500: `docker-compose logs fastapi | grep '<traceback_id>'`
- If `options.traceback` is set to `1`, traceback is displayed in
  the new 500.html template.

Signed-off-by: Kevin Morris <kevr@0cost.org>

aurweb/asgi.py

@@ -1,7 +1,10 @@
+import hashlib
 import http
+import io
 import os
 import re
 import sys
+import traceback
 import typing
 
 from urllib.parse import quote_plus
@@ -26,7 +29,9 @@ from aurweb.db import get_engine, query
 from aurweb.models import AcceptedTerm, Term
 from aurweb.packages.util import get_pkg_or_base
 from aurweb.prometheus import instrumentator
+from aurweb.redis import redis_connection
 from aurweb.routers import APP_ROUTES
+from aurweb.scripts import notify
 from aurweb.templates import make_context, render_template
 
 logger = logging.get_logger(__name__)
@@ -95,6 +100,61 @@ def child_exit(server, worker):  # pragma: no cover
     multiprocess.mark_process_dead(worker.pid)
 
 
+async def internal_server_error(request: Request, exc: Exception) -> Response:
+    """
+    Catch all uncaught Exceptions thrown in a route.
+
+    :param request: FastAPI Request
+    :return: Rendered 500.html template with status_code 500
+    """
+    context = make_context(request, "Internal Server Error")
+
+    # Print out the exception via `traceback` and store the value
+    # into the `traceback` context variable.
+    tb_io = io.StringIO()
+    traceback.print_exc(file=tb_io)
+    tb = tb_io.getvalue()
+    context["traceback"] = tb
+
+    # Produce a SHA1 hash of the traceback string.
+    tb_hash = hashlib.sha1(tb.encode()).hexdigest()
+
+    # Use the first 7 characters of the sha1 for the traceback id.
+    # We will use this to log and include in the notification.
+    tb_id = tb_hash[:7]
+
+    redis = redis_connection()
+    pipe = redis.pipeline()
+    key = f"tb:{tb_hash}"
+    pipe.get(key)
+    retval, = pipe.execute()
+    if not retval:
+        # Expire in one hour; this is just done to make sure we
+        # don't infinitely store these values, but reduce the number
+        # of automated reports (notification below). At this time of
+        # writing, unexpected exceptions are not common, thus this
+        # will not produce a large memory footprint in redis.
+        pipe.set(key, tb)
+        pipe.expire(key, 3600)
+        pipe.execute()
+
+        # Send out notification about it.
+        notif = notify.ServerErrorNotification(
+            tb_id, context.get("version"), context.get("utcnow"))
+        notif.send()
+
+        retval = tb
+    else:
+        retval = retval.decode()
+
+    # Log details about the exception traceback.
+    logger.error(f"FATAL[{tb_id}]: An unexpected exception has occurred.")
+    logger.error(retval)
+
+    return render_template(request, "errors/500.html", context,
+                           status_code=http.HTTPStatus.INTERNAL_SERVER_ERROR)
+
+
 @app.exception_handler(StarletteHTTPException)
 async def http_exception_handler(request: Request, exc: HTTPException) \
         -> Response:
@@ -133,7 +193,10 @@ async def add_security_headers(request: Request, call_next: typing.Callable):
     RP: Referrer-Policy
     XFO: X-Frame-Options
     """
-    response = await util.error_or_result(call_next, request)
+    try:
+        response = await util.error_or_result(call_next, request)
+    except Exception as exc:
+        return await internal_server_error(request, exc)
 
     # Add CSP header.
     nonce = request.user.nonce

aurweb/models/user.py

@@ -1,12 +1,11 @@
 import hashlib
 
 from datetime import datetime
-from http import HTTPStatus
 from typing import List, Set
 
 import bcrypt
 
-from fastapi import HTTPException, Request
+from fastapi import Request
 from sqlalchemy import or_
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import backref, relationship
@@ -142,11 +141,7 @@ class User(Base):
                 exc = exc_
 
         if exc:
-            detail = ("Unable to generate a unique session ID in "
-                      f"{tries} iterations.")
-            logger.error(str(exc))
-            raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
-                                detail=detail)
+            raise exc
 
         return self.session.SessionID
 

aurweb/scripts/notify.py

@@ -7,15 +7,22 @@ import subprocess
 import sys
 import textwrap
 
+from typing import List, Tuple
+
 from sqlalchemy import and_, or_
 
 import aurweb.config
 import aurweb.db
 import aurweb.l10n
 
-from aurweb import db, logging
-from aurweb.models import (PackageBase, PackageComaintainer, PackageComment, PackageNotification, PackageRequest, RequestType,
-                           TUVote, User)
+from aurweb import db, l10n, logging
+from aurweb.models import PackageBase, User
+from aurweb.models.package_comaintainer import PackageComaintainer
+from aurweb.models.package_comment import PackageComment
+from aurweb.models.package_notification import PackageNotification
+from aurweb.models.package_request import PackageRequest
+from aurweb.models.request_type import RequestType
+from aurweb.models.tu_vote import TUVote
 
 logger = logging.get_logger(__name__)
 
@@ -122,6 +129,48 @@ class Notification:
             logger.error(str(exc))
 
 
+class ServerErrorNotification(Notification):
+    """ A notification used to represent an internal server error. """
+
+    def __init__(self, traceback_id: int, version: str, utc: int):
+        """
+        Construct a ServerErrorNotification.
+
+        :param traceback_id: Traceback ID
+        :param version: aurweb version
+        :param utc: UTC timestamp
+        """
+        self._tb_id = traceback_id
+        self._version = version
+        self._utc = utc
+
+        postmaster = aurweb.config.get("notifications", "postmaster")
+        self._to = postmaster
+
+        super().__init__()
+
+    def get_recipients(self) -> List[Tuple[str, str]]:
+        from aurweb.auth import AnonymousUser
+        user = (db.query(User).filter(User.Email == self._to).first()
+                or AnonymousUser())
+        return [(self._to, user.LangPreference)]
+
+    def get_subject(self, lang: str) -> str:
+        return l10n.translator.translate("AUR Server Error", lang)
+
+    def get_body(self, lang: str) -> str:
+        """ A forcibly English email body. """
+        dt = aurweb.util.timestamp_to_datetime(self._utc)
+        dts = dt.strftime("%Y-%m-%d %H:%M")
+        return (f"Traceback ID: {self._tb_id}\n"
+                f"Location: {aur_location}\n"
+                f"Version: {self._version}\n"
+                f"Datetime: {dts} UTC\n")
+
+    def get_refs(self):
+        return (aur_location,)
+
+
 class ResetKeyNotification(Notification):
     def __init__(self, uid):