use mysql backend in config.dev

First off: This commit changes the default development database
backend to mysql. sqlite, however, is still completely supported
with the caveat that a user must now modify config.dev to use
the sqlite backend.

While looking into this, it was discovered that our SQLAlchemy
backend for mysql (mysql-connector) completely broke model
attributes when we switched to utf8mb4_bin (binary) -- it does
not correct the correct conversion to and from binary utf8mb4.

The new, replacement dependency mysqlclient does. mysqlclient
is also recommended in SQLAlchemy documentation as the "best"
one available.

The mysqlclient backend uses a different exception flow then
sqlite, and so tests expecting IntegrityError has to be modified
to expect OperationalError from sqlalchemy.exc.

So, for each model that we define, check keys that can't be
NULL and raise sqlalchemy.exc.IntegrityError if we have to.
This way we keep our exceptions uniform.

Signed-off-by: Kevin Morris <kevr@0cost.org>

aurweb/db.py

@@ -98,9 +98,11 @@ def get_sqlalchemy_url():
             param_query = None
         else:
             port = None
-            param_query = {'unix_socket': aurweb.config.get('database', 'socket')}
+            param_query = {
+                'unix_socket': aurweb.config.get('database', 'socket')
+            }
         return constructor(
-            'mysql+mysqlconnector',
+            'mysql+mysqldb',
             username=aurweb.config.get('database', 'user'),
             password=aurweb.config.get('database', 'password'),
             host=aurweb.config.get('database', 'host'),
@@ -117,7 +119,7 @@ def get_sqlalchemy_url():
         raise ValueError('unsupported database backend')
 
 
-def get_engine():
+def get_engine(echo: bool = False):
     """
     Return the global SQLAlchemy engine.
 
@@ -135,13 +137,24 @@ def get_engine():
             # check_same_thread is for a SQLite technicality
             # https://fastapi.tiangolo.com/tutorial/sql-databases/#note
             connect_args["check_same_thread"] = False
-        engine = create_engine(get_sqlalchemy_url(), connect_args=connect_args)
+
+        engine = create_engine(get_sqlalchemy_url(),
+                               connect_args=connect_args,
+                               echo=echo)
         Session = sessionmaker(autocommit=False, autoflush=True, bind=engine)
         session = Session()
 
     return engine
 
 
+def kill_engine():
+    global engine, Session, session
+    if engine:
+        session.close()
+        engine.dispose()
+    engine = Session = session = None
+
+
 def connect():
     """
     Return an SQLAlchemy connection. Connections are usually pooled. See
@@ -160,8 +173,7 @@ class ConnectionExecutor:
     def __init__(self, conn, backend=aurweb.config.get("database", "backend")):
         self._conn = conn
         if backend == "mysql":
-            import mysql.connector
-            self._paramstyle = mysql.connector.paramstyle
+            self._paramstyle = "format"
         elif backend == "sqlite":
             import sqlite3
             self._paramstyle = sqlite3.paramstyle
@@ -197,18 +209,17 @@ class Connection:
         aur_db_backend = aurweb.config.get('database', 'backend')
 
         if aur_db_backend == 'mysql':
-            import mysql.connector
+            import MySQLdb
             aur_db_host = aurweb.config.get('database', 'host')
             aur_db_name = aurweb.config.get('database', 'name')
             aur_db_user = aurweb.config.get('database', 'user')
             aur_db_pass = aurweb.config.get('database', 'password')
             aur_db_socket = aurweb.config.get('database', 'socket')
-            self._conn = mysql.connector.connect(host=aur_db_host,
-                                                 user=aur_db_user,
-                                                 passwd=aur_db_pass,
-                                                 db=aur_db_name,
-                                                 unix_socket=aur_db_socket,
-                                                 buffered=True)
+            self._conn = MySQLdb.connect(host=aur_db_host,
+                                         user=aur_db_user,
+                                         passwd=aur_db_pass,
+                                         db=aur_db_name,
+                                         unix_socket=aur_db_socket)
         elif aur_db_backend == 'sqlite':
             import sqlite3
             aur_db_name = aurweb.config.get('database', 'name')
@@ -217,7 +228,7 @@ class Connection:
         else:
             raise ValueError('unsupported database backend')
 
-        self._conn = ConnectionExecutor(self._conn)
+        self._conn = ConnectionExecutor(self._conn, aur_db_backend)
 
     def execute(self, query, params=()):
         return self._conn.execute(query, params)

aurweb/initdb.py

@@ -2,7 +2,6 @@ import argparse
 
 import alembic.command
 import alembic.config
-import sqlalchemy
 
 import aurweb.db
 import aurweb.schema
@@ -34,6 +33,8 @@ def feed_initial_data(conn):
 
 
 def run(args):
+    aurweb.config.rehash()
+
     # Ensure Alembic is fine before we do the real work, in order not to fail at
     # the last step and leave the database in an inconsistent state. The
     # configuration is loaded lazily, so we query it to force its loading.
@@ -42,8 +43,7 @@ def run(args):
         alembic_config.get_main_option('script_location')
         alembic_config.attributes["configure_logger"] = False
 
-    engine = sqlalchemy.create_engine(aurweb.db.get_sqlalchemy_url(),
-                                      echo=(args.verbose >= 1))
+    engine = aurweb.db.get_engine(echo=(args.verbose >= 1))
     aurweb.schema.metadata.create_all(engine)
     feed_initial_data(engine.connect())
 

aurweb/models/accepted_term.py

@@ -1,3 +1,4 @@
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.db import make_relationship
@@ -11,7 +12,19 @@ class AcceptedTerm:
                  User: User = None, Term: Term = None,
                  Revision: int = None):
         self.User = User
+        if not self.User:
+            raise IntegrityError(
+                statement="Foreign key UserID cannot be null.",
+                orig="AcceptedTerms.UserID",
+                params=("NULL"))
+
         self.Term = Term
+        if not self.Term:
+            raise IntegrityError(
+                statement="Foreign key TermID cannot be null.",
+                orig="AcceptedTerms.TermID",
+                params=("NULL"))
+
         self.Revision = Revision
 
 

aurweb/models/api_rate_limit.py

@@ -1,3 +1,4 @@
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.schema import ApiRateLimit as _ApiRateLimit
@@ -8,8 +9,20 @@ class ApiRateLimit:
                  Requests: int = None,
                  WindowStart: int = None):
         self.IP = IP
+
         self.Requests = Requests
+        if self.Requests is None:
+            raise IntegrityError(
+                statement="Column Requests cannot be null.",
+                orig="ApiRateLimit.Requests",
+                params=("NULL"))
+
         self.WindowStart = WindowStart
+        if self.WindowStart is None:
+            raise IntegrityError(
+                statement="Column WindowStart cannot be null.",
+                orig="ApiRateLimit.WindowStart",
+                params=("NULL"))
 
 
 mapper(ApiRateLimit, _ApiRateLimit, primary_key=[_ApiRateLimit.c.IP])

aurweb/models/group.py

@@ -1,3 +1,4 @@
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.schema import Groups
@@ -6,6 +7,11 @@ from aurweb.schema import Groups
 class Group:
     def __init__(self, Name: str = None):
         self.Name = Name
+        if not self.Name:
+            raise IntegrityError(
+                statement="Column Name cannot be null.",
+                orig="Groups.Name",
+                params=("NULL"))
 
 
 mapper(Group, Groups)

aurweb/models/license.py

@@ -1,3 +1,4 @@
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.schema import Licenses
@@ -6,6 +7,11 @@ from aurweb.schema import Licenses
 class License:
     def __init__(self, Name: str = None):
         self.Name = Name
+        if not self.Name:
+            raise IntegrityError(
+                statement="Column Name cannot be null.",
+                orig="Licenses.Name",
+                params=("NULL"))
 
 
 mapper(License, Licenses)

aurweb/models/package.py

@@ -1,3 +1,4 @@
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.db import make_relationship
@@ -11,7 +12,19 @@ class Package:
                  Name: str = None, Version: str = None,
                  Description: str = None, URL: str = None):
         self.PackageBase = PackageBase
+        if not self.PackageBase:
+            raise IntegrityError(
+                statement="Foreign key UserID cannot be null.",
+                orig="Packages.PackageBaseID",
+                params=("NULL"))
+
         self.Name = Name
+        if not self.Name:
+            raise IntegrityError(
+                statement="Column Name cannot be null.",
+                orig="Packages.Name",
+                params=("NULL"))
+
         self.Version = Version
         self.Description = Description
         self.URL = URL

aurweb/models/package_base.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.db import make_relationship
@@ -12,6 +13,12 @@ class PackageBase:
                  Maintainer: User = None, Submitter: User = None,
                  Packager: User = None, **kwargs):
         self.Name = Name
+        if not self.Name:
+            raise IntegrityError(
+                statement="Column Name cannot be null.",
+                orig="PackageBases.Name",
+                params=("NULL"))
+
         self.Flagger = Flagger
         self.Maintainer = Maintainer
         self.Submitter = Submitter

aurweb/models/package_dependency.py

@@ -1,3 +1,4 @@
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.db import make_relationship
@@ -12,8 +13,26 @@ class PackageDependency:
                  DepName: str = None, DepDesc: str = None,
                  DepCondition: str = None, DepArch: str = None):
         self.Package = Package
+        if not self.Package:
+            raise IntegrityError(
+                statement="Foreign key PackageID cannot be null.",
+                orig="PackageDependencies.PackageID",
+                params=("NULL"))
+
         self.DependencyType = DependencyType
-        self.DepName = DepName  # nullable=False
+        if not self.DependencyType:
+            raise IntegrityError(
+                statement="Foreign key DepTypeID cannot be null.",
+                orig="PackageDependencies.DepTypeID",
+                params=("NULL"))
+
+        self.DepName = DepName
+        if not self.DepName:
+            raise IntegrityError(
+                statement="Column DepName cannot be null.",
+                orig="PackageDependencies.DepName",
+                params=("NULL"))
+
         self.DepDesc = DepDesc
         self.DepCondition = DepCondition
         self.DepArch = DepArch

aurweb/models/package_group.py

@@ -1,5 +1,5 @@
-from sqlalchemy.orm import mapper
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import mapper
 
 from aurweb.db import make_relationship
 from aurweb.models.group import Group

aurweb/models/package_keyword.py

@@ -1,5 +1,5 @@
-from sqlalchemy.orm import mapper
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import mapper
 
 from aurweb.db import make_relationship
 from aurweb.models.package_base import PackageBase

aurweb/models/package_license.py

@@ -1,5 +1,5 @@
-from sqlalchemy.orm import mapper
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import mapper
 
 from aurweb.db import make_relationship
 from aurweb.models.license import License

aurweb/models/package_relation.py

@@ -1,3 +1,4 @@
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.db import make_relationship
@@ -12,8 +13,26 @@ class PackageRelation:
                  RelName: str = None, RelCondition: str = None,
                  RelArch: str = None):
         self.Package = Package
+        if not self.Package:
+            raise IntegrityError(
+                statement="Foreign key PackageID cannot be null.",
+                orig="PackageRelations.PackageID",
+                params=("NULL"))
+
         self.RelationType = RelationType
+        if not self.RelationType:
+            raise IntegrityError(
+                statement="Foreign key RelTypeID cannot be null.",
+                orig="PackageRelations.RelTypeID",
+                params=("NULL"))
+
         self.RelName = RelName  # nullable=False
+        if not self.RelName:
+            raise IntegrityError(
+                statement="Column RelName cannot be null.",
+                orig="PackageRelations.RelName",
+                params=("NULL"))
+
         self.RelCondition = RelCondition
         self.RelArch = RelArch
 

aurweb/models/session.py

@@ -1,16 +1,20 @@
-from sqlalchemy import Column, Integer
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import backref, mapper, relationship
 
-from aurweb.db import make_random_value
+from aurweb.db import make_random_value, query
 from aurweb.models.user import User
 from aurweb.schema import Sessions
 
 
 class Session:
-    UsersID = Column(Integer, nullable=True)
-
     def __init__(self, **kwargs):
         self.UsersID = kwargs.get("UsersID")
+        if not query(User, User.ID == self.UsersID).first():
+            raise IntegrityError(
+                statement="Foreign key UsersID cannot be null.",
+                orig="Sessions.UsersID",
+                params=("NULL"))
+
         self.SessionID = kwargs.get("SessionID")
         self.LastUpdateTS = kwargs.get("LastUpdateTS")
 

aurweb/models/term.py

@@ -1,3 +1,4 @@
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import mapper
 
 from aurweb.schema import Terms
@@ -8,7 +9,19 @@ class Term:
                  Description: str = None, URL: str = None,
                  Revision: int = None):
         self.Description = Description
+        if not self.Description:
+            raise IntegrityError(
+                statement="Column Description cannot be null.",
+                orig="Terms.Description",
+                params=("NULL"))
+
         self.URL = URL
+        if not self.URL:
+            raise IntegrityError(
+                statement="Column URL cannot be null.",
+                orig="Terms.URL",
+                params=("NULL"))
+
         self.Revision = Revision