mirror of
https://gitlab.archlinux.org/archlinux/aurweb.git
synced 2025-02-03 10:43:03 +01:00
change(aurweb): add parallel tests and improve aurweb.db
This change utilizes pytest-xdist to perform a multiproc test
run and reworks aurweb.db's code. We no longer use a global
engine, session or Session, but we now use a memo of engines
and sessions as they are requested, based on the PYTEST_CURRENT_TEST
environment variable, which is available during testing.
Additionally, this change strips several SQLite components
out of the Python code-base.
SQLite is still compatible with PHP and sharness tests, but
not with our FastAPI implementation.
More changes:
------------
- Remove use of aurweb.db.session global in other code.
- Use new aurweb.db.name() dynamic db name function in env.py.
- Added 'addopts' to pytest.ini which utilizes multiprocessing.
- Highly recommended to leave this be or modify `-n auto` to
`-n {cpu_threads}` where cpu_threads is at least 2.
Signed-off-by: Kevin Morris <kevr@0cost.org>
This commit is contained in:
Kevin Morris
parent
07aac768d6
commit
fa43f6bc3e
55 changed files with 781 additions and 884 deletions
251
aurweb/db.py
251
aurweb/db.py
|
|
@ -1,29 +1,34 @@
|
|||
import functools
|
||||
import hashlib
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
|
||||
from typing import Iterable, NewType
|
||||
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.orm import Query, scoped_session
|
||||
import sqlalchemy
|
||||
|
||||
from sqlalchemy import create_engine, event
|
||||
from sqlalchemy.engine.base import Engine
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlalchemy.orm import Query, Session, SessionTransaction, scoped_session, sessionmaker
|
||||
|
||||
import aurweb.config
|
||||
import aurweb.util
|
||||
|
||||
# See get_engine.
|
||||
engine = None
|
||||
from aurweb import logging
|
||||
|
||||
# ORM Session class.
|
||||
Session = None
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
# Global ORM Session object.
|
||||
session = None
|
||||
DRIVERS = {
|
||||
"mysql": "mysql+mysqldb"
|
||||
}
|
||||
|
||||
# Global introspected object memo.
|
||||
introspected = dict()
|
||||
|
||||
# A mocked up type.
|
||||
Base = NewType("aurweb.models.declarative_base.Base", "Base")
|
||||
# Some types we don't get access to in this module.
|
||||
Base = NewType("Base", "aurweb.models.declarative_base.Base")
|
||||
|
||||
|
||||
def make_random_value(table: str, column: str):
|
||||
|
|
@ -56,14 +61,85 @@ def make_random_value(table: str, column: str):
|
|||
length = col.type.length
|
||||
|
||||
string = aurweb.util.make_random_string(length)
|
||||
while session.query(table).filter(column == string).first():
|
||||
while query(table).filter(column == string).first():
|
||||
string = aurweb.util.make_random_string(length)
|
||||
return string
|
||||
|
||||
|
||||
def get_session():
|
||||
def test_name() -> str:
|
||||
"""
|
||||
Return the unhashed database name.
|
||||
|
||||
The unhashed database name is determined (lower = higher priority) by:
|
||||
-------------------------------------------
|
||||
1. {test_suite} portion of PYTEST_CURRENT_TEST
|
||||
2. aurweb.config.get("database", "name")
|
||||
|
||||
During `pytest` runs, the PYTEST_CURRENT_TEST environment variable
|
||||
is set to the current test in the format `{test_suite}::{test_func}`.
|
||||
|
||||
This allows tests to use a suite-specific database for its runs,
|
||||
which decouples database state from test suites.
|
||||
|
||||
:return: Unhashed database name
|
||||
"""
|
||||
db = os.environ.get("PYTEST_CURRENT_TEST",
|
||||
aurweb.config.get("database", "name"))
|
||||
return db.split(":")[0]
|
||||
|
||||
|
||||
def name() -> str:
|
||||
"""
|
||||
Return sanitized database name that can be used for tests or production.
|
||||
|
||||
If test_name() starts with "test/", the database name is SHA-1 hashed,
|
||||
prefixed with 'db', and returned. Otherwise, test_name() is passed
|
||||
through and not hashed at all.
|
||||
|
||||
:return: SHA1-hashed database name prefixed with 'db'
|
||||
"""
|
||||
dbname = test_name()
|
||||
if not dbname.startswith("test/"):
|
||||
return dbname
|
||||
sha1 = hashlib.sha1(dbname.encode()).hexdigest()
|
||||
return "db" + sha1
|
||||
|
||||
|
||||
# Module-private global memo used to store SQLAlchemy sessions.
|
||||
_sessions = dict()
|
||||
|
||||
|
||||
def get_session(engine: Engine = None) -> Session:
|
||||
""" Return aurweb.db's global session. """
|
||||
return session
|
||||
dbname = name()
|
||||
|
||||
global _sessions
|
||||
if dbname not in _sessions:
|
||||
|
||||
if not engine: # pragma: no cover
|
||||
engine = get_engine()
|
||||
|
||||
Session = scoped_session(
|
||||
sessionmaker(autocommit=True, autoflush=False, bind=engine))
|
||||
_sessions[dbname] = Session()
|
||||
|
||||
# If this is the first grab of this session, log out the
|
||||
# database name used.
|
||||
raw_dbname = test_name()
|
||||
logger.debug(f"DBName({raw_dbname}): {dbname}")
|
||||
|
||||
return _sessions.get(dbname)
|
||||
|
||||
|
||||
def pop_session(dbname: str) -> None:
|
||||
"""
|
||||
Pop a Session out of the private _sessions memo.
|
||||
|
||||
:param dbname: Database name
|
||||
:raises KeyError: When `dbname` does not exist in the memo
|
||||
"""
|
||||
global _sessions
|
||||
_sessions.pop(dbname)
|
||||
|
||||
|
||||
def refresh(model: Base) -> Base:
|
||||
|
|
@ -121,41 +197,40 @@ def add(model: Base) -> Base:
|
|||
return model
|
||||
|
||||
|
||||
def begin():
|
||||
def begin() -> SessionTransaction:
|
||||
""" Begin an SQLAlchemy SessionTransaction. """
|
||||
return get_session().begin()
|
||||
|
||||
|
||||
def get_sqlalchemy_url():
|
||||
def get_sqlalchemy_url() -> URL:
|
||||
"""
|
||||
Build an SQLAlchemy for use with create_engine based on the aurweb configuration.
|
||||
"""
|
||||
import sqlalchemy
|
||||
Build an SQLAlchemy URL for use with create_engine.
|
||||
|
||||
constructor = sqlalchemy.engine.url.URL
|
||||
:return: sqlalchemy.engine.url.URL
|
||||
"""
|
||||
constructor = URL
|
||||
|
||||
parts = sqlalchemy.__version__.split('.')
|
||||
major = int(parts[0])
|
||||
minor = int(parts[1])
|
||||
if major == 1 and minor >= 4: # pragma: no cover
|
||||
constructor = sqlalchemy.engine.url.URL.create
|
||||
constructor = URL.create
|
||||
|
||||
aur_db_backend = aurweb.config.get('database', 'backend')
|
||||
if aur_db_backend == 'mysql':
|
||||
if aurweb.config.get_with_fallback('database', 'port', fallback=None):
|
||||
port = aurweb.config.get('database', 'port')
|
||||
param_query = None
|
||||
else:
|
||||
port = None
|
||||
param_query = {
|
||||
'unix_socket': aurweb.config.get('database', 'socket')
|
||||
}
|
||||
param_query = {}
|
||||
port = aurweb.config.get_with_fallback("database", "port", None)
|
||||
if not port:
|
||||
param_query["unix_socket"] = aurweb.config.get(
|
||||
"database", "socket")
|
||||
|
||||
return constructor(
|
||||
'mysql+mysqldb',
|
||||
DRIVERS.get(aur_db_backend),
|
||||
username=aurweb.config.get('database', 'user'),
|
||||
password=aurweb.config.get('database', 'password'),
|
||||
password=aurweb.config.get_with_fallback('database', 'password',
|
||||
fallback=None),
|
||||
host=aurweb.config.get('database', 'host'),
|
||||
database=aurweb.config.get('database', 'name'),
|
||||
database=name(),
|
||||
port=port,
|
||||
query=param_query
|
||||
)
|
||||
|
|
@ -168,58 +243,83 @@ def get_sqlalchemy_url():
|
|||
raise ValueError('unsupported database backend')
|
||||
|
||||
|
||||
def get_engine(echo: bool = False):
|
||||
def sqlite_regexp(regex, item) -> bool: # pragma: no cover
|
||||
""" Method which mimics SQL's REGEXP for SQLite. """
|
||||
return bool(re.search(regex, str(item)))
|
||||
|
||||
|
||||
def setup_sqlite(engine: Engine) -> None: # pragma: no cover
|
||||
""" Perform setup for an SQLite engine. """
|
||||
@event.listens_for(engine, "connect")
|
||||
def do_begin(conn, record):
|
||||
create_deterministic_function = functools.partial(
|
||||
conn.create_function,
|
||||
deterministic=True
|
||||
)
|
||||
create_deterministic_function("REGEXP", 2, sqlite_regexp)
|
||||
|
||||
|
||||
# Module-private global memo used to store SQLAlchemy engines.
|
||||
_engines = dict()
|
||||
|
||||
|
||||
def get_engine(dbname: str = None, echo: bool = False) -> Engine:
|
||||
"""
|
||||
Return the global SQLAlchemy engine.
|
||||
Return the SQLAlchemy engine for `dbname`.
|
||||
|
||||
The engine is created on the first call to get_engine and then stored in the
|
||||
`engine` global variable for the next calls.
|
||||
|
||||
:param dbname: Database name (default: aurweb.db.name())
|
||||
:param echo: Flag passed through to sqlalchemy.create_engine
|
||||
:return: SQLAlchemy Engine instance
|
||||
"""
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
if not dbname:
|
||||
dbname = name()
|
||||
|
||||
global engine, session, Session
|
||||
|
||||
if engine is None:
|
||||
global _engines
|
||||
if dbname not in _engines:
|
||||
db_backend = aurweb.config.get("database", "backend")
|
||||
connect_args = dict()
|
||||
|
||||
db_backend = aurweb.config.get("database", "backend")
|
||||
if db_backend == "sqlite":
|
||||
# check_same_thread is for a SQLite technicality
|
||||
# https://fastapi.tiangolo.com/tutorial/sql-databases/#note
|
||||
is_sqlite = bool(db_backend == "sqlite")
|
||||
if is_sqlite: # pragma: no cover
|
||||
connect_args["check_same_thread"] = False
|
||||
|
||||
engine = create_engine(get_sqlalchemy_url(),
|
||||
connect_args=connect_args,
|
||||
echo=echo)
|
||||
kwargs = {
|
||||
"echo": echo,
|
||||
"connect_args": connect_args
|
||||
}
|
||||
_engines[dbname] = create_engine(get_sqlalchemy_url(), **kwargs)
|
||||
|
||||
Session = scoped_session(
|
||||
sessionmaker(autocommit=True, autoflush=False, bind=engine))
|
||||
session = Session()
|
||||
if is_sqlite: # pragma: no cover
|
||||
setup_sqlite(_engines.get(dbname))
|
||||
|
||||
if db_backend == "sqlite":
|
||||
# For SQLite, we need to add some custom functions as
|
||||
# they are used in the reference graph method.
|
||||
def regexp(regex, item):
|
||||
return bool(re.search(regex, str(item)))
|
||||
|
||||
@event.listens_for(engine, "connect")
|
||||
def do_begin(conn, record):
|
||||
create_deterministic_function = functools.partial(
|
||||
conn.create_function,
|
||||
deterministic=True
|
||||
)
|
||||
create_deterministic_function("REGEXP", 2, regexp)
|
||||
|
||||
return engine
|
||||
return _engines.get(dbname)
|
||||
|
||||
|
||||
def kill_engine():
|
||||
global engine, Session, session
|
||||
if engine:
|
||||
session.close()
|
||||
engine.dispose()
|
||||
engine = Session = session = None
|
||||
def pop_engine(dbname: str) -> None:
|
||||
"""
|
||||
Pop an Engine out of the private _engines memo.
|
||||
|
||||
:param dbname: Database name
|
||||
:raises KeyError: When `dbname` does not exist in the memo
|
||||
"""
|
||||
global _engines
|
||||
_engines.pop(dbname)
|
||||
|
||||
|
||||
def kill_engine() -> None:
|
||||
""" Close the current session and dispose of the engine. """
|
||||
dbname = name()
|
||||
|
||||
session = get_session()
|
||||
session.close()
|
||||
pop_session(dbname)
|
||||
|
||||
engine = get_engine()
|
||||
engine.dispose()
|
||||
pop_engine(dbname)
|
||||
|
||||
|
||||
def connect():
|
||||
|
|
@ -248,7 +348,9 @@ class ConnectionExecutor:
|
|||
def paramstyle(self):
|
||||
return self._paramstyle
|
||||
|
||||
def execute(self, query, params=()):
|
||||
def execute(self, query, params=()): # pragma: no cover
|
||||
# TODO: SQLite support has been removed in FastAPI. It remains
|
||||
# here to fund its support for PHP until it is removed.
|
||||
if self._paramstyle in ('format', 'pyformat'):
|
||||
query = query.replace('%', '%%').replace('?', '%s')
|
||||
elif self._paramstyle == 'qmark':
|
||||
|
|
@ -278,16 +380,19 @@ class Connection:
|
|||
if aur_db_backend == 'mysql':
|
||||
import MySQLdb
|
||||
aur_db_host = aurweb.config.get('database', 'host')
|
||||
aur_db_name = aurweb.config.get('database', 'name')
|
||||
aur_db_name = name()
|
||||
aur_db_user = aurweb.config.get('database', 'user')
|
||||
aur_db_pass = aurweb.config.get('database', 'password')
|
||||
aur_db_pass = aurweb.config.get_with_fallback(
|
||||
'database', 'password', str())
|
||||
aur_db_socket = aurweb.config.get('database', 'socket')
|
||||
self._conn = MySQLdb.connect(host=aur_db_host,
|
||||
user=aur_db_user,
|
||||
passwd=aur_db_pass,
|
||||
db=aur_db_name,
|
||||
unix_socket=aur_db_socket)
|
||||
elif aur_db_backend == 'sqlite':
|
||||
elif aur_db_backend == 'sqlite': # pragma: no cover
|
||||
# TODO: SQLite support has been removed in FastAPI. It remains
|
||||
# here to fund its support for PHP until it is removed.
|
||||
import sqlite3
|
||||
aur_db_name = aurweb.config.get('database', 'name')
|
||||
self._conn = sqlite3.connect(aur_db_name)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue