diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 60db2962..036e3441 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -1,7 +1,10 @@ from http import HTTPStatus +from typing import List + +import orjson from fastapi import HTTPException -from sqlalchemy import and_ +from sqlalchemy import and_, orm from aurweb import db from aurweb.models.official_provider import OFFICIAL_BASE, OfficialProvider @@ -10,6 +13,7 @@ from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_relation import PackageRelation from aurweb.models.relation_type import PROVIDES_ID, RelationType +from aurweb.redis import redis_connection from aurweb.templates import register_filter @@ -111,3 +115,52 @@ def get_pkgbase(name: str) -> PackageBase: raise HTTPException(status_code=int(HTTPStatus.NOT_FOUND)) return pkgbase + + +@register_filter("out_of_date") +def out_of_date(packages: orm.Query) -> orm.Query: + return packages.filter(PackageBase.OutOfDateTS.isnot(None)) + + +def updated_packages(limit: int = 0, cache_ttl: int = 600) -> List[Package]: + """ Return a list of valid Package objects ordered by their + ModifiedTS column in descending order from cache, after setting + the cache when no key yet exists. + + :param limit: Optional record limit + :param cache_ttl: Cache expiration time (in seconds) + :return: A list of Packages + """ + redis = redis_connection() + packages = redis.get("package_updates") + if packages: + # If we already have a cache, deserialize it and return. + return orjson.loads(packages) + + query = db.query(Package).join(PackageBase).filter( + PackageBase.PackagerUID.isnot(None) + ).order_by( + PackageBase.ModifiedTS.desc() + ) + + if limit: + query = query.limit(limit) + + packages = [] + for pkg in query: + # For each Package returned by the query, append a dict + # containing Package columns we're interested in. + packages.append({ + "Name": pkg.Name, + "Version": pkg.Version, + "PackageBase": { + "ModifiedTS": pkg.PackageBase.ModifiedTS + } + }) + + # Store the JSON serialization of the package_updates key into Redis. + redis.set("package_updates", orjson.dumps(packages)) + redis.expire("package_updates", cache_ttl) + + # Return the deserialized list of packages. + return packages diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index f6f1a54e..ae012901 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -1,14 +1,21 @@ """ AURWeb's primary routing module. Define all routes via @app.app.{get,post} decorators in some way; more complex routes should be defined in their own modules and imported here. """ +from datetime import datetime from http import HTTPStatus from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, RedirectResponse +from sqlalchemy import and_, or_ import aurweb.config -from aurweb import util +from aurweb import db, util +from aurweb.cache import db_count_cache +from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID +from aurweb.models.package_base import PackageBase +from aurweb.models.user import User +from aurweb.packages.util import updated_packages from aurweb.templates import make_context, render_template router = APIRouter() @@ -60,6 +67,71 @@ async def index(request: Request): context = make_context(request, "Home") context['ssh_fingerprints'] = util.get_ssh_fingerprints() + bases = db.query(PackageBase) + + redis = aurweb.redis.redis_connection() + stats_expire = 300 # Five minutes. + updates_expire = 600 # Ten minutes. + + # Package statistics. + query = bases.filter(PackageBase.PackagerUID.isnot(None)) + context["package_count"] = await db_count_cache( + redis, "package_count", query, expire=stats_expire) + + query = bases.filter( + and_(PackageBase.MaintainerUID.is_(None), + PackageBase.PackagerUID.isnot(None)) + ) + context["orphan_count"] = await db_count_cache( + redis, "orphan_count", query, expire=stats_expire) + + query = db.query(User) + context["user_count"] = await db_count_cache( + redis, "user_count", query, expire=stats_expire) + + query = query.filter( + or_(User.AccountTypeID == TRUSTED_USER_ID, + User.AccountTypeID == TRUSTED_USER_AND_DEV_ID)) + context["trusted_user_count"] = await db_count_cache( + redis, "trusted_user_count", query, expire=stats_expire) + + # Current timestamp. + now = int(datetime.utcnow().timestamp()) + + seven_days = 86400 * 7 # Seven days worth of seconds. + seven_days_ago = now - seven_days + + one_hour = 3600 + updated = bases.filter( + and_(PackageBase.ModifiedTS - PackageBase.SubmittedTS >= one_hour, + PackageBase.PackagerUID.isnot(None)) + ) + + query = bases.filter( + and_(PackageBase.SubmittedTS >= seven_days_ago, + PackageBase.PackagerUID.isnot(None)) + ) + context["seven_days_old_added"] = await db_count_cache( + redis, "seven_days_old_added", query, expire=stats_expire) + + query = updated.filter(PackageBase.ModifiedTS >= seven_days_ago) + context["seven_days_old_updated"] = await db_count_cache( + redis, "seven_days_old_updated", query, expire=stats_expire) + + year = seven_days * 52 # Fifty two weeks worth: one year. + year_ago = now - year + query = updated.filter(PackageBase.ModifiedTS >= year_ago) + context["year_old_updated"] = await db_count_cache( + redis, "year_old_updated", query, expire=stats_expire) + + query = bases.filter( + PackageBase.ModifiedTS - PackageBase.SubmittedTS < 3600) + context["never_updated"] = await db_count_cache( + redis, "never_updated", query, expire=stats_expire) + + # Get the 15 most recently updated packages. + context["package_updates"] = updated_packages(15, updates_expire) + return render_template(request, "index.html", context) diff --git a/templates/home.html b/templates/home.html new file mode 100644 index 00000000..a8cae5b8 --- /dev/null +++ b/templates/home.html @@ -0,0 +1,101 @@ +
+

AUR {% trans %}Home{% endtrans %}

+

+ {{ "Welcome to the AUR! Please read the %sAUR User Guidelines%s and %sAUR TU Guidelines%s for more information." + | tr + | format('', "", + '', "") + | safe + }} + {{ "Contributed PKGBUILDs %smust%s conform to the %sArch Packaging Standards%s otherwise they will be deleted!" + | tr + | format("", "", + '', + "") + | safe + }} + {% trans %}Remember to vote for your favourite packages!{% endtrans %} + {% trans %}Some packages may be provided as binaries in [community].{% endtrans %} +

+

+ {% trans %}DISCLAIMER{% endtrans %}: + {% trans %}AUR packages are user produced content. Any use of the provided files is at your own risk.{% endtrans %} +

+

{% trans %}Learn more...{% endtrans %}

+
+
+

{% trans %}Support{% endtrans %}

+

{% trans %}Package Requests{% endtrans %}

+
+

+ {{ "There are three types of requests that can be filed in the %sPackage Actions%s box on the package details page:" + | tr + | format("", "") + | safe + }} +

+ +

+ {{ "If you want to discuss a request, you can use the %saur-requests%s mailing list. However, please do not use that list to file requests." + | tr + | format('', "") + | safe + }} +

+
+

{% trans %}Submitting Packages{% endtrans %}

+
+

+ {{ "Git over SSH is now used to submit packages to the AUR. See the %sSubmitting packages%s section of the Arch User Repository ArchWiki page for more details." + | tr + | format('', "") + | safe + }} +

+ {% if ssh_fingerprints %} +

+ {% trans %}The following SSH fingerprints are used for the AUR:{% endtrans %} +

+

+ {% endif %} +
+

{% trans %}Discussion{% endtrans %}

+
+

+ {{ "General discussion regarding the Arch User Repository (AUR) and Trusted User structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." + | tr + | format('', "", + '', "") + | safe + }} +

+

+

{% trans %}Bug Reporting{% endtrans %}

+
+

+ {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the package maintainer or leave a comment on the appropriate package page." + | tr + | format('', "", + "", "") + | safe + }} +

+
+
+ + + + + + diff --git a/templates/index.html b/templates/index.html index f8745f33..e50a99cd 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,106 +1,15 @@ {% extends 'partials/layout.html' %} {% block pageContent %} -
-

AUR {% trans %}Home{% endtrans %}

-

- {{ "Welcome to the AUR! Please read the %sAUR User Guidelines%s and %sAUR TU Guidelines%s for more information." - | tr - | format('', "", - '', "") - | safe - }} - {{ "Contributed PKGBUILDs %smust%s conform to the %sArch Packaging Standards%s otherwise they will be deleted!" - | tr - | format("", "", - '', - "") - | safe - }} - {% trans %}Remember to vote for your favourite packages!{% endtrans %} - {% trans %}Some packages may be provided as binaries in [community].{% endtrans %} -

- {% trans %}DISCLAIMER{% endtrans %}: - {% trans %}AUR packages are user produced content. Any use of the provided files is at your own risk.{% endtrans %} -

-

{% trans %}Learn more...{% endtrans %}

-

-
-
-

{% trans %}Support{% endtrans %}

-

{% trans %}Package Requests{% endtrans %}

-
-

- {{ "There are three types of requests that can be filed in the %sPackage Actions%s box on the package details page:" - | tr - | format("", "") - | safe - }} -

- -

- {{ "If you want to discuss a request, you can use the %saur-requests%s mailing list. However, please do not use that list to file requests." - | tr - | format('', "") - | safe - }} -

+
+
+ {% include 'home.html' %} +
-

{% trans %}Submitting Packages{% endtrans %}

-
-

- {{ "Git over SSH is now used to submit packages to the AUR. See the %sSubmitting packages%s section of the Arch User Repository ArchWiki page for more details." - | tr - | format('', "") - | safe - }} -

- {% if ssh_fingerprints %} -

- {% trans %}The following SSH fingerprints are used for the AUR:{% endtrans %} -

-

    - {% for keytype in ssh_fingerprints %} -
  • {{ keytype }}: {{ ssh_fingerprints[keytype] }} - {% endfor %} -
- {% endif %} +
+ {% include 'partials/packages/widgets/search.html' %} + {% include 'partials/packages/widgets/updates.html' %} + {% include 'partials/packages/widgets/statistics.html' %}
-

{% trans %}Discussion{% endtrans %}

-
-

- {{ "General discussion regarding the Arch User Repository (AUR) and Trusted User structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." - | tr - | format('', "", - '', "") - | safe - }} -

-

-

{% trans %}Bug Reporting{% endtrans %}

-
-

- {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the package maintainer or leave a comment on the appropriate package page." - | tr - | format('', "", - "", "") - | safe - }} -

-
-
- - - - - - {% endblock %} diff --git a/templates/partials/packages/widgets/search.html b/templates/partials/packages/widgets/search.html new file mode 100644 index 00000000..106b93ea --- /dev/null +++ b/templates/partials/packages/widgets/search.html @@ -0,0 +1,14 @@ +
+
+
+ + + +
+
+
diff --git a/templates/partials/packages/widgets/statistics.html b/templates/partials/packages/widgets/statistics.html new file mode 100644 index 00000000..f841ae0e --- /dev/null +++ b/templates/partials/packages/widgets/statistics.html @@ -0,0 +1,55 @@ +
+

{{ "Statistics" | tr }}

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
{{ "Packages" | tr }}{{ package_count }}
{{ "Orphan Packages" | tr }}{{ orphan_count }}
+ {{ "Packages added in the past 7 days" | tr }} + {{ seven_days_old_added }}
+ {{ "Packages updated in the past 7 days" | tr }} + {{ seven_days_old_updated }}
+ {{ "Packages updated in the past year" | tr }} + {{ year_old_updated }}
+ {{ "Packages never updated" | tr }} + {{ never_updated }}
+ {{ "Registered Users" | tr }} + {{ user_count }}
+ {{ "Trusted Users" | tr }} + {{ trusted_user_count }}
+
+ +{% if request.user.is_authenticated() %} + + {% include 'partials/widgets/statistics.html' %} +{% endif %} diff --git a/templates/partials/packages/widgets/updates.html b/templates/partials/packages/widgets/updates.html new file mode 100644 index 00000000..3ee1b98e --- /dev/null +++ b/templates/partials/packages/widgets/updates.html @@ -0,0 +1,35 @@ +
+

+ {{ "Recent Updates" | tr }} + + ({{ "more" | tr }}) + +

+ + RSS Feed + + + RSS Feed + + + + + {% for pkg in package_updates %} + + + + + {% endfor %} + +
+ + {{ pkg.Name }} {{ pkg.Version }} + + + {% set modified = pkg.PackageBase.ModifiedTS | dt | as_timezone(timezone) %} + {{ modified.strftime("%Y-%m-%d %H:%M") }} +
+ +
diff --git a/templates/partials/widgets/statistics.html b/templates/partials/widgets/statistics.html new file mode 100644 index 00000000..0bf844b6 --- /dev/null +++ b/templates/partials/widgets/statistics.html @@ -0,0 +1,27 @@ +
+

{{ "My Statistics" | tr }}

+ + {% set bases = request.user.maintained_bases %} + + + + + + + {% set out_of_date_packages = bases | out_of_date %} + + + + + +
+ + {{ "Packages" | tr }} + + {{ bases.count() }}
+ + {{ "Out of Date" | tr }} + + {{ out_of_date_packages.count() }}
+ +
diff --git a/test/test_homepage.py b/test/test_homepage.py index 23d7185f..a629b98c 100644 --- a/test/test_homepage.py +++ b/test/test_homepage.py @@ -1,13 +1,82 @@ +import re + +from datetime import datetime from http import HTTPStatus from unittest.mock import patch +import pytest + from fastapi.testclient import TestClient +from aurweb import db from aurweb.asgi import app +from aurweb.models.account_type import USER_ID +from aurweb.models.package import Package +from aurweb.models.package_base import PackageBase +from aurweb.models.user import User +from aurweb.redis import redis_connection +from aurweb.testing import setup_test_db +from aurweb.testing.html import parse_root client = TestClient(app) +@pytest.fixture(autouse=True) +def setup(): + yield setup_test_db( + User.__tablename__, + Package.__tablename__, + PackageBase.__tablename__ + ) + + +@pytest.fixture +def user(): + yield db.create(User, Username="test", Email="test@example.org", + Passwd="testPassword", AccountTypeID=USER_ID) + + +@pytest.fixture +def redis(): + redis = redis_connection() + + def delete_keys(): + # Cleanup keys if they exist. + for key in ("package_count", "orphan_count", "user_count", + "trusted_user_count", "seven_days_old_added", + "seven_days_old_updated", "year_old_updated", + "never_updated", "package_updates"): + if redis.get(key) is not None: + redis.delete(key) + + delete_keys() + yield redis + delete_keys() + + +@pytest.fixture +def packages(user): + """ Yield a list of num_packages Package objects maintained by user. """ + num_packages = 50 # Tunable + + # For i..num_packages, create a package named pkg_{i}. + pkgs = [] + now = int(datetime.utcnow().timestamp()) + for i in range(num_packages): + pkgbase = db.create(PackageBase, Name=f"pkg_{i}", + Maintainer=user, Packager=user, + autocommit=False, SubmittedTS=now, + ModifiedTS=now) + pkg = db.create(Package, PackageBase=pkgbase, + Name=pkgbase.Name, autocommit=False) + pkgs.append(pkg) + now += 1 + + db.commit() + + yield pkgs + + def test_homepage(): with client as request: response = request.get("/") @@ -34,3 +103,49 @@ def test_homepage_no_ssh_fingerprints(get_ssh_fingerprints_mock): response = request.get("/") assert 'The following SSH fingerprints are used for the AUR' not in response.content.decode() + + +def test_homepage_stats(redis, packages): + with client as request: + response = request.get("/") + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + + expectations = [ + ("Packages", r'\d+'), + ("Orphan Packages", r'\d+'), + ("Packages added in the past 7 days", r'\d+'), + ("Packages updated in the past 7 days", r'\d+'), + ("Packages updated in the past year", r'\d+'), + ("Packages never updated", r'\d+'), + ("Registered Users", r'\d+'), + ("Trusted Users", r'\d+') + ] + + stats = root.xpath('//div[@id="pkg-stats"]//tr') + for i, expected in enumerate(expectations): + expected_key, expected_regex = expected + key, value = stats[i].xpath('./td') + assert key.text.strip() == expected_key + assert re.match(expected_regex, value.text.strip()) + + +def test_homepage_updates(redis, packages): + with client as request: + response = request.get("/") + assert response.status_code == int(HTTPStatus.OK) + # Run the request a second time to exercise the Redis path. + response = request.get("/") + assert response.status_code == int(HTTPStatus.OK) + + root = parse_root(response.text) + + # We expect to see the latest 15 packages, which happens to be + # pkg_49 .. pkg_34. So, create a list of expectations using a range + # starting at 49, stepping down to 49 - 15, -1 step at a time. + expectations = [f"pkg_{i}" for i in range(50 - 1, 50 - 1 - 15, -1)] + updates = root.xpath('//div[@id="pkg-updates"]/table/tbody/tr') + for i, expected in enumerate(expectations): + pkgname = updates[i].xpath('./td/a').pop(0) + assert pkgname.text.strip() == expected diff --git a/test/test_packages_util.py b/test/test_packages_util.py index 17978490..bc6a941c 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -9,6 +9,7 @@ from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.user import User from aurweb.packages import util +from aurweb.redis import kill_redis from aurweb.testing import setup_test_db @@ -33,7 +34,8 @@ def maintainer() -> User: @pytest.fixture def package(maintainer: User) -> Package: - pkgbase = db.create(PackageBase, Name="test-pkg", Maintainer=maintainer) + pkgbase = db.create(PackageBase, Name="test-pkg", + Packager=maintainer, Maintainer=maintainer) yield db.create(Package, Name=pkgbase.Name, PackageBase=pkgbase) @@ -49,3 +51,18 @@ def test_package_link(client: TestClient, maintainer: User, package: Package): Provides=package.Name) expected = f"{OFFICIAL_BASE}/packages/?q={package.Name}" assert util.package_link(package) == expected + + +def test_updated_packages(maintainer: User, package: Package): + expected = { + "Name": package.Name, + "Version": package.Version, + "PackageBase": { + "ModifiedTS": package.PackageBase.ModifiedTS + } + } + + kill_redis() # Kill it here to ensure we're on a fake instance. + assert util.updated_packages(1, 0) == [expected] + assert util.updated_packages(1, 600) == [expected] + kill_redis() # Kill it again, in case other tests use a real instance.