change(aurblup): converted to use aurweb.db ORM

Introduces:
- aurweb.testing.alpm.AlpmDatabase
    - Used to mock up and manage a remote repository.
- templates/testing/alpm_package.j2
    - Used to generate a single ALPM package desc.
- Removed aurblup sharness test

Signed-off-by: Kevin Morris <kevr@0cost.org>

aurweb/scripts/aurblup.py

@@ -4,30 +4,34 @@ import re
 
 import pyalpm
 
+from sqlalchemy import and_
+
 import aurweb.config
-import aurweb.db
 
-db_path = aurweb.config.get('aurblup', 'db-path')
-sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ')
-server = aurweb.config.get('aurblup', 'server')
+from aurweb import db, util
+from aurweb.models import OfficialProvider
 
 
-def main():
+def _main(force: bool = False):
     blacklist = set()
     providers = set()
     repomap = dict()
 
+    db_path = aurweb.config.get("aurblup", "db-path")
+    sync_dbs = aurweb.config.get('aurblup', 'sync-dbs').split(' ')
+    server = aurweb.config.get('aurblup', 'server')
+
     h = pyalpm.Handle("/", db_path)
     for sync_db in sync_dbs:
         repo = h.register_syncdb(sync_db, pyalpm.SIG_DATABASE_OPTIONAL)
         repo.servers = [server.replace("%s", sync_db)]
         t = h.init_transaction()
-        repo.update(False)
+        repo.update(force)
         t.release()
 
         for pkg in repo.pkgcache:
             blacklist.add(pkg.name)
-            [blacklist.add(x) for x in pkg.replaces]
+            util.apply_all(pkg.replaces, blacklist.add)
             providers.add((pkg.name, pkg.name))
             repomap[(pkg.name, pkg.name)] = repo.name
             for provision in pkg.provides:
@@ -35,21 +39,29 @@ def main():
                 providers.add((pkg.name, provisionname))
                 repomap[(pkg.name, provisionname)] = repo.name
 
-    conn = aurweb.db.Connection()
+    with db.begin():
+        old_providers = set(
+            db.query(OfficialProvider).with_entities(
+                OfficialProvider.Name.label("Name"),
+                OfficialProvider.Provides.label("Provides")
+            ).distinct().order_by("Name").all()
+        )
 
-    cur = conn.execute("SELECT Name, Provides FROM OfficialProviders")
-    oldproviders = set(cur.fetchall())
+        for name, provides in old_providers.difference(providers):
+            db.delete_all(db.query(OfficialProvider).filter(
+                and_(OfficialProvider.Name == name,
+                     OfficialProvider.Provides == provides)
+            ))
 
-    for pkg, provides in oldproviders.difference(providers):
-        conn.execute("DELETE FROM OfficialProviders "
-                     "WHERE Name = ? AND Provides = ?", [pkg, provides])
-    for pkg, provides in providers.difference(oldproviders):
-        repo = repomap[(pkg, provides)]
-        conn.execute("INSERT INTO OfficialProviders (Name, Repo, Provides) "
-                     "VALUES (?, ?, ?)", [pkg, repo, provides])
+        for name, provides in providers.difference(old_providers):
+            repo = repomap.get((name, provides))
+            db.create(OfficialProvider, Name=name,
+                      Repo=repo, Provides=provides)
 
-    conn.commit()
-    conn.close()
+
+def main(force: bool = False):
+    db.get_engine()
+    _main(force)
 
 
 if __name__ == '__main__':

aurweb/templates.py

@@ -125,6 +125,11 @@ async def make_variable_context(request: Request, title: str, next: str = None):
     return context
 
 
+def base_template(path: str):
+    templates = copy.copy(_env)
+    return templates.get_template(path)
+
+
 def render_raw_template(request: Request, path: str, context: dict):
     """ Render a Jinja2 multi-lingual template with some context. """
     # Create a deep copy of our jinja2 _environment. The _environment in

aurweb/testing/alpm.py

@@ -0,0 +1,87 @@
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+
+from typing import List
+
+from aurweb import logging, util
+from aurweb.templates import base_template
+
+logger = logging.get_logger(__name__)
+
+
+class AlpmDatabase:
+    """
+    Fake libalpm database management class.
+
+    This class can be used to add or remove packages from a
+    test repository.
+    """
+    repo = "test"
+
+    def __init__(self, database_root: str):
+        self.root = database_root
+        self.local = os.path.join(self.root, "local")
+        self.remote = os.path.join(self.root, "remote")
+        self.repopath = os.path.join(self.remote, self.repo)
+
+        # Make directories.
+        os.makedirs(self.local)
+        os.makedirs(self.remote)
+
+    def _get_pkgdir(self, pkgname: str, pkgver: str, repo: str) -> str:
+        pkgfile = f"{pkgname}-{pkgver}-1"
+        pkgdir = os.path.join(self.remote, repo, pkgfile)
+        os.makedirs(pkgdir)
+        return pkgdir
+
+    def add(self, pkgname: str, pkgver: str, arch: str,
+            provides: List[str] = []) -> None:
+        context = {
+            "pkgname": pkgname,
+            "pkgver": pkgver,
+            "arch": arch,
+            "provides": provides
+        }
+        template = base_template("testing/alpm_package.j2")
+        pkgdir = self._get_pkgdir(pkgname, pkgver, self.repo)
+        desc = os.path.join(pkgdir, "desc")
+        with open(desc, "w") as f:
+            f.write(template.render(context))
+
+        self.compile()
+
+    def remove(self, pkgname: str):
+        files = os.listdir(self.repopath)
+        logger.info(f"Files: {files}")
+        expr = "^" + pkgname + r"-[0-9.]+-1$"
+        logger.info(f"Expression: {expr}")
+        to_delete = filter(lambda e: re.match(expr, e), files)
+
+        for target in to_delete:
+            logger.info(f"Deleting {target}")
+            path = os.path.join(self.repopath, target)
+            shutil.rmtree(path)
+
+        self.compile()
+
+    def clean(self) -> None:
+        db_file = os.path.join(self.remote, "test.db")
+        try:
+            os.remove(db_file)
+        except Exception:
+            pass
+
+    def compile(self) -> None:
+        self.clean()
+        cmdline = ["bash", "-c", "bsdtar -czvf ../test.db *"]
+        proc = subprocess.run(cmdline, cwd=self.repopath)
+        assert proc.returncode == 0, \
+            f"Bad return code while creating alpm database: {proc.returncode}"
+
+        # Print out the md5 hash value of the new test.db.
+        test_db = os.path.join(self.remote, "test.db")
+        db_hash = util.file_hash(test_db, hashlib.md5)
+        logger.debug(f"{test_db}: {db_hash}")

aurweb/util.py

@@ -176,3 +176,20 @@ def strtobool(value: str) -> bool:
     if isinstance(value, str):
         return _strtobool(value)
     return value
+
+
+def file_hash(filepath: str, hash_function: Callable) -> str:
+    """
+    Return a hash of filepath contents using `hash_function`.
+
+    `hash_function` can be any one of the hashlib module's hash
+    functions which implement the `hexdigest()` method -- e.g.
+    hashlib.sha1, hashlib.md5, etc.
+
+    :param filepath: Path to file you want to hash
+    :param hash_function: hashlib hash function
+    :return: hash_function(filepath_content).hexdigest()
+    """
+    with open(filepath, "rb") as f:
+        hash_ = hash_function(f.read())
+    return hash_.hexdigest()