external/aurweb
0
0
Fork 0
mirror of https://gitlab.archlinux.org/archlinux/aurweb.git synced 2025-02-03 10:43:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

change(aurweb): add parallel tests and improve aurweb.db

Browse source 
This change utilizes pytest-xdist to perform a multiproc test
run and reworks aurweb.db's code. We no longer use a global
engine, session or Session, but we now use a memo of engines
and sessions as they are requested, based on the PYTEST_CURRENT_TEST
environment variable, which is available during testing.

Additionally, this change strips several SQLite components
out of the Python code-base.

SQLite is still compatible with PHP and sharness tests, but
not with our FastAPI implementation.

More changes:
------------
- Remove use of aurweb.db.session global in other code.
- Use new aurweb.db.name() dynamic db name function in env.py.
- Added 'addopts' to pytest.ini which utilizes multiprocessing.
    - Highly recommended to leave this be or modify `-n auto` to
      `-n {cpu_threads}` where cpu_threads is at least 2.

Signed-off-by: Kevin Morris <kevr@0cost.org>
This commit is contained in:
Kevin Morris 2021-11-17 00:33:41 -08:00
parent 07aac768d6
commit fa43f6bc3e
No known key found for this signature in database
GPG key ID: F7E46DED420788F3
55 changed files with 781 additions and 884 deletions
Download patch file Download diff file Expand all files Collapse all files

68
aurweb/testing/__init__.py
View file

@ -1,26 +1,6 @@
from itertools import chain
import aurweb.db
def references_graph(table):
""" Taken from Django's sqlite3/operations.py. """
query = """
WITH tables AS (
SELECT :table name
UNION
SELECT sqlite_master.name
FROM sqlite_master
JOIN tables ON (sql REGEXP :regexp_1 || tables.name || :regexp_2)
) SELECT name FROM tables;
"""
params = {
"table": table,
"regexp_1": r'(?i)\s+references\s+("|\')?',
"regexp_2": r'("|\')?\s*\(',
}
cursor = aurweb.db.get_session().execute(query, params=params)
return [row[0] for row in cursor.fetchall()]
from aurweb import models
def setup_test_db(*args):
@ -47,22 +27,38 @@ def setup_test_db(*args):
aurweb.db.get_engine()
tables = list(args)
if not tables:
tables = [
models.AcceptedTerm.__tablename__,
models.ApiRateLimit.__tablename__,
models.Ban.__tablename__,
models.Group.__tablename__,
models.License.__tablename__,
models.OfficialProvider.__tablename__,
models.Package.__tablename__,
models.PackageBase.__tablename__,
models.PackageBlacklist.__tablename__,
models.PackageComaintainer.__tablename__,
models.PackageComment.__tablename__,
models.PackageDependency.__tablename__,
models.PackageGroup.__tablename__,
models.PackageKeyword.__tablename__,
models.PackageLicense.__tablename__,
models.PackageNotification.__tablename__,
models.PackageRelation.__tablename__,
models.PackageRequest.__tablename__,
models.PackageSource.__tablename__,
models.PackageVote.__tablename__,
models.Session.__tablename__,
models.SSHPubKey.__tablename__,
models.Term.__tablename__,
models.TUVote.__tablename__,
models.TUVoteInfo.__tablename__,
models.User.__tablename__,
]
db_backend = aurweb.config.get("database", "backend")
if db_backend != "sqlite": # pragma: no cover
aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 0")
else:
# We're using sqlite, setup tables to be deleted without violating
# foreign key constraints by graphing references.
tables = set(chain.from_iterable(
references_graph(table) for table in tables))
aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 0")
for table in tables:
aurweb.db.get_session().execute(f"DELETE FROM {table}")
if db_backend != "sqlite": # pragma: no cover
aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 1")
# Expunge all objects from SQLAlchemy's IdentityMap.
aurweb.db.get_session().execute("SET FOREIGN_KEY_CHECKS = 1")
aurweb.db.get_session().expunge_all()