From 3005e82f607e7b20ac5e32d50f0ffb124a8737e0 Mon Sep 17 00:00:00 2001 From: moson Date: Fri, 18 Aug 2023 22:04:55 +0200 Subject: [PATCH 01/89] fix: Cleanup prometheus metrics for dead workers The current "cleanup" function that is removing orphan prometheus files is actually never invoked. We move this to a default gunicorn config file to register our hook(s). https://docs.gunicorn.org/en/stable/configure.html https://docs.gunicorn.org/en/stable/settings.html#child-exit Signed-off-by: moson --- aurweb/asgi.py | 7 ------- gunicorn.conf.py | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 gunicorn.conf.py diff --git a/aurweb/asgi.py b/aurweb/asgi.py index 1be77ff9..9b6ffcb3 100644 --- a/aurweb/asgi.py +++ b/aurweb/asgi.py @@ -13,7 +13,6 @@ from fastapi import FastAPI, HTTPException, Request, Response from fastapi.responses import RedirectResponse from fastapi.staticfiles import StaticFiles from jinja2 import TemplateNotFound -from prometheus_client import multiprocess from sqlalchemy import and_ from starlette.exceptions import HTTPException as StarletteHTTPException from starlette.middleware.authentication import AuthenticationMiddleware @@ -91,12 +90,6 @@ async def app_startup(): get_engine() -def child_exit(server, worker): # pragma: no cover - """This function is required for gunicorn customization - of prometheus multiprocessing.""" - multiprocess.mark_process_dead(worker.pid) - - async def internal_server_error(request: Request, exc: Exception) -> Response: """ Catch all uncaught Exceptions thrown in a route. diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 00000000..4f1c3a8c --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,7 @@ +from prometheus_client import multiprocess + + +def child_exit(server, worker): # pragma: no cover + """This function is required for gunicorn customization + of prometheus multiprocessing.""" + multiprocess.mark_process_dead(worker.pid) From 6c610b26a39a56db562e4b1ed3c95420a73ee766 Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Fri, 28 Jul 2023 22:42:44 +0200 Subject: [PATCH 02/89] feat: Add terraform config for review-app[1] Also removed the logic for deploying to the long gone aur-dev box. Ansible will be added in a upcoming commit for configurating and deploying aurweb on the VM. [1] https://docs.gitlab.com/ee/ci/review_apps/ --- .gitignore | 4 +++ .gitlab-ci.yml | 71 +++++++++++++++++++++++---------------- ci/tf/.terraform.lock.hcl | 61 +++++++++++++++++++++++++++++++++ ci/tf/main.tf | 67 ++++++++++++++++++++++++++++++++++++ ci/tf/terraform.tfvars | 4 +++ ci/tf/variables.tf | 36 ++++++++++++++++++++ ci/tf/versions.tf | 13 +++++++ 7 files changed, 227 insertions(+), 29 deletions(-) create mode 100644 ci/tf/.terraform.lock.hcl create mode 100644 ci/tf/main.tf create mode 100644 ci/tf/terraform.tfvars create mode 100644 ci/tf/variables.tf create mode 100644 ci/tf/versions.tf diff --git a/.gitignore b/.gitignore index 68de7cd5..97157118 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,7 @@ test-emails/ env/ venv/ .venv/ + +# Ignore some terraform files +/ci/tf/.terraform +/ci/tf/terraform.tfstate* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10dd1787..4bd71920 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,34 +61,47 @@ test: coverage_format: cobertura path: coverage.xml -deploy: - stage: deploy - tags: - - secure - rules: - - if: $CI_COMMIT_BRANCH == "pu" - when: manual - variables: - FASTAPI_BACKEND: gunicorn - FASTAPI_WORKERS: 5 - AURWEB_FASTAPI_PREFIX: https://aur-dev.archlinux.org - AURWEB_SSHD_PREFIX: ssh://aur@aur-dev.archlinux.org:2222 - COMMIT_HASH: $CI_COMMIT_SHA - GIT_DATA_DIR: git_data - script: - - pacman -Syu --noconfirm docker docker-compose socat openssh - - chmod 600 ${SSH_KEY} - - socat "UNIX-LISTEN:/tmp/docker.sock,reuseaddr,fork" EXEC:"ssh -o UserKnownHostsFile=${SSH_KNOWN_HOSTS} -Ti ${SSH_KEY} ${SSH_USER}@${SSH_HOST}" & - - export DOCKER_HOST="unix:///tmp/docker.sock" - # Set secure login config for aurweb. - - sed -ri "s/^(disable_http_login).*$/\1 = 1/" conf/config.dev - - docker-compose build - - docker-compose -f docker-compose.yml -f docker-compose.aur-dev.yml down --remove-orphans - - docker-compose -f docker-compose.yml -f docker-compose.aur-dev.yml up -d - - docker image prune -f - - docker container prune -f - - docker volume prune -f +.init_tf: &init_tf + - pacman -Syu --needed --noconfirm --cachedir .pkg-cache terraform + - export TF_VAR_name="aurweb-${CI_COMMIT_REF_SLUG}" + - TF_ADDRESS="${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}" + - cd ci/tf + - > + terraform init \ + -backend-config="address=${TF_ADDRESS}" \ + -backend-config="lock_address=${TF_ADDRESS}/lock" \ + -backend-config="unlock_address=${TF_ADDRESS}/lock" \ + -backend-config="username=x-access-token" \ + -backend-config="password=${TF_STATE_GITLAB_ACCESS_TOKEN}" \ + -backend-config="lock_method=POST" \ + -backend-config="unlock_method=DELETE" \ + -backend-config="retry_wait_min=5" +deploy_review: + stage: deploy + script: + - *init_tf + - terraform apply -auto-approve environment: - name: development - url: https://aur-dev.archlinux.org + name: review/$CI_COMMIT_REF_NAME + url: https://aurweb-$CI_ENVIRONMENT_SLUG.sandbox.archlinux.page + on_stop: stop_review + auto_stop_in: 1 week + rules: + - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" + when: manual + +stop_review: + stage: deploy + needs: + - deploy_review + script: + - *init_tf + - terraform destroy -auto-approve + - 'curl --silent --show-error --fail --header "Private-Token: ${TF_STATE_GITLAB_ACCESS_TOKEN}" --request DELETE "${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}"' + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + rules: + - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" + when: manual diff --git a/ci/tf/.terraform.lock.hcl b/ci/tf/.terraform.lock.hcl new file mode 100644 index 00000000..aa5501c4 --- /dev/null +++ b/ci/tf/.terraform.lock.hcl @@ -0,0 +1,61 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/dns" { + version = "3.3.2" + hashes = [ + "h1:HjskPLRqmCw8Q/kiSuzti3iJBSpcAvcBFdlwFFQuoDE=", + "zh:05d2d50e301318362a4a82e6b7a9734ace07bc01abaaa649c566baf98814755f", + "zh:1e9fd1c3bfdda777e83e42831dd45b7b9e794250a0f351e5fd39762e8a0fe15b", + "zh:40e715fc7a2ede21f919567249b613844692c2f8a64f93ee64e5b68bae7ac2a2", + "zh:454d7aa83000a6e2ba7a7bfde4bcf5d7ed36298b22d760995ca5738ab02ee468", + "zh:46124ded51b4153ad90f12b0305fdbe0c23261b9669aa58a94a31c9cca2f4b19", + "zh:55a4f13d20f73534515a6b05701abdbfc54f4e375ba25b2dffa12afdad20e49d", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7903b1ceb8211e2b8c79290e2e70906a4b88f4fba71c900eb3a425ce12f1716a", + "zh:b79fc4f444ef7a2fd7111a80428c070ad824f43a681699e99ab7f83074dfedbd", + "zh:ca9f45e0c4cb94e7d62536c226024afef3018b1de84f1ea4608b51bcd497a2a0", + "zh:ddc8bd894559d7d176e0ceb0bb1ae266519b01b315362ebfee8327bb7e7e5fa8", + "zh:e77334c0794ef8f9354b10e606040f6b0b67b373f5ff1db65bddcdd4569b428b", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.4" + hashes = [ + "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=", + "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", + "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", + "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", + "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", + "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", + "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", + "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", + "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", + "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", + "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", + "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hetznercloud/hcloud" { + version = "1.42.0" + hashes = [ + "h1:cr9lh26H3YbWSHb7OUnCoYw169cYO3Cjpt3yPnRhXS0=", + "zh:153b5f39d780e9a18bc1ea377d872647d328d943813cbd25d3d20863f8a37782", + "zh:35b9e95760c58cca756e34ad5f4138ac6126aa3e8c41b4a0f1d5dc9ee5666c73", + "zh:47a3cdbce982f2b4e17f73d4934bdb3e905a849b36fb59b80f87d852496ed049", + "zh:6a718c244c2ba300fbd43791661a061ad1ab16225ef3e8aeaa3db8c9eff12c85", + "zh:a2cbfc95c5e2c9422ed0a7b6292192c38241220d5b7813c678f937ab3ef962ae", + "zh:b837e118e08fd36aa8be48af7e9d0d3d112d2680c79cfc71cfe2501fb40dbefa", + "zh:bf66db8c680e18b77e16dc1f20ed1cdcc7876bfb7848c320ccb86f0fb80661ed", + "zh:c1ad80bbe48dc8a272a02dcdb4b12f019606f445606651c01e561b9d72d816b1", + "zh:d4e616701128ad14a6b5a427b0e9145ece4cad02aa3b5f9945c6d0b9ada8ab70", + "zh:d9d01f727037d028720100a5bc9fd213cb01e63e4b439a16f2f482c147976530", + "zh:dea047ee4d679370d4376fb746c4b959bf51dd06047c1c2656b32789c2433643", + "zh:e5ad7a3c556894bd40b28a874e7d2f6924876fa75fa443136a7d6ab9a00abbaa", + "zh:edf6e7e129157bd45e3da4a330d1ace17a336d417c3b77c620f302d440c368e8", + "zh:f610bc729866d58da9cffa4deae34dbfdba96655e855a87c6bb2cb7b35a8961c", + ] +} diff --git a/ci/tf/main.tf b/ci/tf/main.tf new file mode 100644 index 00000000..b149a621 --- /dev/null +++ b/ci/tf/main.tf @@ -0,0 +1,67 @@ +terraform { + backend "http" { + } +} + +provider "hcloud" { + token = var.hcloud_token +} + +provider "dns" { + update { + server = var.dns_server + key_name = var.dns_tsig_key + key_algorithm = var.dns_tsig_algorithm + key_secret = var.dns_tsig_secret + } +} + +resource "tls_private_key" "this" { + algorithm = "ED25519" +} + +resource "hcloud_ssh_key" "this" { + name = var.name + public_key = tls_private_key.this.public_key_openssh +} + +data "hcloud_image" "this" { + with_selector = "custom_image=archlinux" + most_recent = true + with_status = ["available"] +} + +resource "hcloud_server" "this" { + name = var.name + image = data.hcloud_image.this.id + server_type = var.server_type + datacenter = var.datacenter + ssh_keys = [hcloud_ssh_key.this.name] + + public_net { + ipv4_enabled = true + ipv6_enabled = true + } +} + +resource "hcloud_rdns" "this" { + for_each = { ipv4 : hcloud_server.this.ipv4_address, ipv6 : hcloud_server.this.ipv6_address } + + server_id = hcloud_server.this.id + ip_address = each.value + dns_ptr = "${var.name}.${var.dns_zone}" +} + +resource "dns_a_record_set" "this" { + zone = "${var.dns_zone}." + name = var.name + addresses = [hcloud_server.this.ipv4_address] + ttl = 300 +} + +resource "dns_aaaa_record_set" "this" { + zone = "${var.dns_zone}." + name = var.name + addresses = [hcloud_server.this.ipv6_address] + ttl = 300 +} diff --git a/ci/tf/terraform.tfvars b/ci/tf/terraform.tfvars new file mode 100644 index 00000000..14818592 --- /dev/null +++ b/ci/tf/terraform.tfvars @@ -0,0 +1,4 @@ +server_type = "cpx11" +datacenter = "fsn1-dc14" +dns_server = "redirect.archlinux.org" +dns_zone = "sandbox.archlinux.page" diff --git a/ci/tf/variables.tf b/ci/tf/variables.tf new file mode 100644 index 00000000..a4e710ee --- /dev/null +++ b/ci/tf/variables.tf @@ -0,0 +1,36 @@ +variable "hcloud_token" { + type = string + sensitive = true +} + +variable "dns_server" { + type = string +} + +variable "dns_tsig_key" { + type = string +} + +variable "dns_tsig_algorithm" { + type = string +} + +variable "dns_tsig_secret" { + type = string +} + +variable "dns_zone" { + type = string +} + +variable "name" { + type = string +} + +variable "server_type" { + type = string +} + +variable "datacenter" { + type = string +} diff --git a/ci/tf/versions.tf b/ci/tf/versions.tf new file mode 100644 index 00000000..2c72215a --- /dev/null +++ b/ci/tf/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + tls = { + source = "hashicorp/tls" + } + hcloud = { + source = "hetznercloud/hcloud" + } + dns = { + source = "hashicorp/dns" + } + } +} From 9eda6a42c69581dfdc14dc1b0d51f744985c7202 Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 27 Aug 2023 13:54:39 +0200 Subject: [PATCH 03/89] feat: Add ansible provisioning step for review-app Clone infrastructure repository and run playbook to provision our VM with aurweb. Signed-off-by: moson --- .gitlab-ci.yml | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4bd71920..cf80ab24 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,8 @@ variables: TEST_RECURSION_LIMIT: 10000 CURRENT_DIR: "$(pwd)" LOG_CONFIG: logging.test.conf + DEV_FQDN: aurweb-$CI_COMMIT_REF_SLUG.sandbox.archlinux.page + INFRASTRUCTURE_REPO: https://gitlab.archlinux.org/archlinux/infrastructure.git lint: stage: .pre @@ -84,13 +86,63 @@ deploy_review: - terraform apply -auto-approve environment: name: review/$CI_COMMIT_REF_NAME - url: https://aurweb-$CI_ENVIRONMENT_SLUG.sandbox.archlinux.page + url: https://$DEV_FQDN on_stop: stop_review auto_stop_in: 1 week rules: - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" when: manual +provision_review: + stage: deploy + needs: + - deploy_review + script: + - *init_tf + - pacman -Syu --noconfirm --needed --cachedir .pkg-cache ansible git openssh jq + # Get ssh key from terraform state file + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - terraform show -json | + jq -r '.values.root_module.resources[] | + select(.address == "tls_private_key.this") | + .values.private_key_openssh' > ~/.ssh/id_ed25519 + - chmod 400 ~/.ssh/id_ed25519 + # Clone infra repo + - git clone $INFRASTRUCTURE_REPO + - cd infrastructure + # Remove vault files + - rm $(git grep -l 'ANSIBLE_VAULT;1.1;AES256$') + # Remove vault config + - sed -i '/^vault/d' ansible.cfg + # Add host config + - mkdir -p host_vars/$DEV_FQDN + - 'echo "filesystem: btrfs" > host_vars/$DEV_FQDN/misc' + # Add host + - echo "$DEV_FQDN" > hosts + # Add our pubkey and hostkeys + - ssh-keyscan $DEV_FQDN >> ~/.ssh/known_hosts + - ssh-keygen -f ~/.ssh/id_ed25519 -y > pubkeys/aurweb-dev.pub + # Run our ansible playbook + - > + ansible-playbook playbooks/aur-dev.archlinux.org.yml \ + -e "aurdev_fqdn=$DEV_FQDN" \ + -e "aurweb_repository=$CI_REPOSITORY_URL" \ + -e "aurweb_version=$CI_COMMIT_SHA" \ + -e "{\"vault_mariadb_users\":{\"root\":\"aur\"}}" \ + -e "vault_aurweb_db_password=aur" \ + -e "vault_aurweb_gitlab_instance=https://does.not.exist" \ + -e "vault_aurweb_error_project=aur" \ + -e "vault_aurweb_error_token=aur" \ + -e "vault_aurweb_secret=aur" \ + -e "vault_goaurrpc_metrics_token=aur" \ + -e '{"root_additional_keys": ["moson.pub", "aurweb-dev.pub"]}' + environment: + name: review/$CI_COMMIT_REF_NAME + action: access + rules: + - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" + stop_review: stage: deploy needs: From 5699e9bb41638fc1d6040f3e70a90fab38257458 Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 26 Aug 2023 14:47:21 +0200 Subject: [PATCH 04/89] fix(test): Remove file locking and semaphore All tests within a file run in the same worker and out test DB names are unique per file as well. We don't really need a locking mechanism here. Same is valid for the test-emails. The only potential issue is that it might try to create the same directory multiple times and thus run into an error. However, that can be covered by specifying "exist_ok=True" with os.makedirs such that those errors are ignored. Signed-off-by: moson --- test/conftest.py | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 15a982aa..c36f78dd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -43,7 +43,6 @@ from multiprocessing import Lock import py import pytest -from posix_ipc import O_CREAT, Semaphore from sqlalchemy import create_engine from sqlalchemy.engine import URL from sqlalchemy.engine.base import Engine @@ -54,7 +53,6 @@ import aurweb.config import aurweb.db from aurweb import aur_logging, initdb, testing from aurweb.testing.email import Email -from aurweb.testing.filelock import FileLock from aurweb.testing.git import GitRepository logger = aur_logging.get_logger(__name__) @@ -133,20 +131,16 @@ def _drop_database(engine: Engine, dbname: str) -> None: def setup_email(): - # TODO: Fix this data race! This try/catch is ugly; why is it even - # racing here? Perhaps we need to multiproc + multithread lock - # inside of setup_database to block the check? - with Semaphore("/test-emails", flags=O_CREAT, initial_value=1): - if not os.path.exists(Email.TEST_DIR): - # Create the directory. - os.makedirs(Email.TEST_DIR) + if not os.path.exists(Email.TEST_DIR): + # Create the directory. + os.makedirs(Email.TEST_DIR, exist_ok=True) - # Cleanup all email files for this test suite. - prefix = Email.email_prefix(suite=True) - files = os.listdir(Email.TEST_DIR) - for file in files: - if file.startswith(prefix): - os.remove(os.path.join(Email.TEST_DIR, file)) + # Cleanup all email files for this test suite. + prefix = Email.email_prefix(suite=True) + files = os.listdir(Email.TEST_DIR) + for file in files: + if file.startswith(prefix): + os.remove(os.path.join(Email.TEST_DIR, file)) @pytest.fixture(scope="module") @@ -155,20 +149,8 @@ def setup_database(tmp_path_factory: pathlib.Path, worker_id: str) -> None: engine = test_engine() dbname = aurweb.db.name() - if worker_id == "master": # pragma: no cover - # If we're not running tests through multiproc pytest-xdist. - setup_email() - yield _create_database(engine, dbname) - _drop_database(engine, dbname) - return - - def setup(path): - setup_email() - _create_database(engine, dbname) - - tmpdir = tmp_path_factory.getbasetemp().parent - file_lock = FileLock(tmpdir, dbname) - file_lock.lock(on_create=setup) + setup_email() + _create_database(engine, dbname) yield # Run the test function depending on this fixture. _drop_database(engine, dbname) # Cleanup the database. From 1433553c05993b097e812e43496bf140df49144c Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 26 Aug 2023 17:08:36 +0200 Subject: [PATCH 05/89] fix(test): Clear previous prometheus data for test It could happen that test data is already generated by a previous test. (running in the same worker) Make sure we clear everything before performing our checks. Signed-off-by: moson --- test/test_statistics.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_statistics.py b/test/test_statistics.py index db262fa3..80223cbd 100644 --- a/test/test_statistics.py +++ b/test/test_statistics.py @@ -1,7 +1,7 @@ import pytest from prometheus_client import REGISTRY, generate_latest -from aurweb import cache, db, time +from aurweb import cache, db, prometheus, time from aurweb.models import Package, PackageBase, PackageRequest from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID from aurweb.models.package_request import ( @@ -140,6 +140,11 @@ def test_get_count_change(stats: Statistics, test_data): def test_update_prometheus_metrics(test_data): + # Make sure any previous data is cleared + prometheus.USERS.clear() + prometheus.PACKAGES.clear() + prometheus.REQUESTS.clear() + metrics = str(generate_latest(REGISTRY)) assert "aur_users{" not in metrics From 0a7b02956feeaaeea4813650b12ead15cfc822af Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 3 Sep 2023 14:17:11 +0200 Subject: [PATCH 06/89] feat: Indicate dependency source Dependencies might reside in the AUR or official repositories. Add "AUR" as superscript letters to indicate if a package/provider is present in the AUR. Signed-off-by: moson --- aurweb/models/package_dependency.py | 7 +++- aurweb/packages/util.py | 8 ++-- .../partials/packages/package_metadata.html | 7 +++- test/test_package_dependency.py | 17 ++++++++ test/test_packages_util.py | 40 +++++++++++++++++++ 5 files changed, 72 insertions(+), 7 deletions(-) diff --git a/aurweb/models/package_dependency.py b/aurweb/models/package_dependency.py index 587ba68d..9cf1eda0 100644 --- a/aurweb/models/package_dependency.py +++ b/aurweb/models/package_dependency.py @@ -57,14 +57,17 @@ class PackageDependency(Base): params=("NULL"), ) - def is_package(self) -> bool: + def is_aur_package(self) -> bool: pkg = db.query(_Package).filter(_Package.Name == self.DepName).exists() + return db.query(pkg).scalar() + + def is_package(self) -> bool: official = ( db.query(_OfficialProvider) .filter(_OfficialProvider.Name == self.DepName) .exists() ) - return db.query(pkg).scalar() or db.query(official).scalar() + return self.is_aur_package() or db.query(official).scalar() def provides(self) -> list[PackageRelation]: from aurweb.models.relation_type import PROVIDES_ID diff --git a/aurweb/packages/util.py b/aurweb/packages/util.py index 78d79508..cfd1e9e9 100644 --- a/aurweb/packages/util.py +++ b/aurweb/packages/util.py @@ -83,9 +83,11 @@ def package_link(package: Union[Package, OfficialProvider]) -> str: @register_filter("provides_markup") def provides_markup(provides: Providers) -> str: - return ", ".join( - [f'{pkg.Name}' for pkg in provides] - ) + links = [] + for pkg in provides: + aur = "ᴬᵁᴿ" if not pkg.is_official else "" + links.append(f'{pkg.Name}{aur}') + return ", ".join(links) def get_pkg_or_base( diff --git a/templates/partials/packages/package_metadata.html b/templates/partials/packages/package_metadata.html index 50d38b48..c8d583a1 100644 --- a/templates/partials/packages/package_metadata.html +++ b/templates/partials/packages/package_metadata.html @@ -14,12 +14,15 @@ {% endif %} {{ dep.DepName }} - {% if broken %} + {%- if broken %} {% if not provides %} {% endif %} - {% else %} + {% else -%} + {%- if dep.is_aur_package() -%} + ᴬᵁᴿ + {% endif %} {% endif %} {% if provides %} diff --git a/test/test_package_dependency.py b/test/test_package_dependency.py index 9366bb55..1cd2d305 100644 --- a/test/test_package_dependency.py +++ b/test/test_package_dependency.py @@ -4,6 +4,7 @@ from sqlalchemy.exc import IntegrityError from aurweb import db from aurweb.models.account_type import USER_ID from aurweb.models.dependency_type import DEPENDS_ID +from aurweb.models.official_provider import OfficialProvider from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency @@ -58,6 +59,22 @@ def test_package_dependencies(user: User, package: Package): db.create(Package, PackageBase=base, Name=pkgdep.DepName) assert pkgdep.is_package() + assert pkgdep.is_aur_package() + + # Test with OfficialProvider + with db.begin(): + pkgdep = db.create( + PackageDependency, + Package=package, + DepTypeID=DEPENDS_ID, + DepName="test-repo-pkg", + ) + db.create( + OfficialProvider, Name=pkgdep.DepName, Repo="extra", Provides=pkgdep.DepName + ) + + assert pkgdep.is_package() + assert not pkgdep.is_aur_package() def test_package_dependencies_null_package_raises(): diff --git a/test/test_packages_util.py b/test/test_packages_util.py index bae84614..b429181b 100644 --- a/test/test_packages_util.py +++ b/test/test_packages_util.py @@ -10,8 +10,10 @@ from aurweb.models.package import Package from aurweb.models.package_base import PackageBase from aurweb.models.package_dependency import PackageDependency from aurweb.models.package_notification import PackageNotification +from aurweb.models.package_relation import PackageRelation from aurweb.models.package_source import PackageSource from aurweb.models.package_vote import PackageVote +from aurweb.models.relation_type import PROVIDES_ID from aurweb.models.user import User from aurweb.packages import util @@ -155,3 +157,41 @@ def test_pkg_required(package: Package): # We should have 1 record assert qry.count() == 1 + + +def test_provides_markup(package: Package): + # Create dependency and provider for AUR pkg + with db.begin(): + dep = db.create( + PackageDependency, + Package=package, + DepName="test", + DepTypeID=DEPENDS_ID, + ) + rel_pkg = db.create(Package, PackageBase=package.PackageBase, Name=dep.DepName) + db.create( + PackageRelation, + Package=rel_pkg, + RelName=dep.DepName, + RelTypeID=PROVIDES_ID, + ) + + # AUR provider links should end with ᴬᵁᴿ + link = util.provides_markup(dep.provides()) + assert link.endswith("ᴬᵁᴿ") + assert OFFICIAL_BASE not in link + + # Remove AUR provider and add official one + with db.begin(): + db.delete(rel_pkg) + db.create( + OfficialProvider, + Name="official-pkg", + Repo="extra", + Provides=dep.DepName, + ) + + # Repo provider links should not have any suffix + link = util.provides_markup(dep.provides()) + assert link.endswith("") + assert OFFICIAL_BASE in link From 7466e964498cd9d19b93e1f38394ae358a5e6a5f Mon Sep 17 00:00:00 2001 From: moson Date: Tue, 26 Sep 2023 13:47:03 +0200 Subject: [PATCH 07/89] fix(ci): Exclude review-app jobs for renovate MR's Signed-off-by: moson --- .gitlab-ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf80ab24..fb40d414 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,6 +90,8 @@ deploy_review: on_stop: stop_review auto_stop_in: 1 week rules: + - if: $CI_COMMIT_REF_NAME =~ /^renovate\// + when: never - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" when: manual @@ -141,6 +143,8 @@ provision_review: name: review/$CI_COMMIT_REF_NAME action: access rules: + - if: $CI_COMMIT_REF_NAME =~ /^renovate\// + when: never - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" stop_review: @@ -155,5 +159,7 @@ stop_review: name: review/$CI_COMMIT_REF_NAME action: stop rules: + - if: $CI_COMMIT_REF_NAME =~ /^renovate\// + when: never - if: $CI_MERGE_REQUEST_ID && $CI_PROJECT_PATH == "archlinux/aurweb" when: manual From 1702075875514de8170b4393d5326cb61e7c5e6e Mon Sep 17 00:00:00 2001 From: moson Date: Fri, 1 Sep 2023 13:25:21 +0200 Subject: [PATCH 08/89] housekeep: TU rename - code changes Renaming of symbols. Functions, variables, values, DB values, etc. Basically everything that is not user-facing. This only covers "Trusted User" things: tests, comments, etc. will covered in a following commit. --- aurweb/auth/__init__.py | 4 +- aurweb/auth/creds.py | 76 ++++++++++--------- aurweb/initdb.py | 4 +- aurweb/models/account_type.py | 12 +-- aurweb/models/user.py | 12 +-- aurweb/pkgbase/actions.py | 2 +- aurweb/routers/__init__.py | 4 +- aurweb/routers/accounts.py | 16 ++-- ...{trusted_user.py => package_maintainer.py} | 59 +++++++------- aurweb/statistics.py | 16 ++-- aurweb/users/validate.py | 2 +- ...d126029_rename_tu_to_package_maintainer.py | 37 +++++++++ schema/gendummydata.py | 28 +++---- templates/addvote.html | 12 +-- templates/partials/archdev-navbar.html | 2 +- .../partials/packages/search_actions.html | 2 +- templates/partials/packages/statistics.html | 2 +- templates/partials/tu/proposals.html | 2 +- templates/tu/index.html | 6 +- test/test_accounts_routes.py | 48 ++++++------ test/test_adduser.py | 4 +- test/test_auth.py | 4 +- test/test_homepage.py | 2 +- test/test_html.py | 10 +-- test/test_notify.py | 4 +- test/test_packages_routes.py | 4 +- test/test_pkgbase_routes.py | 4 +- test/test_requests.py | 4 +- test/test_statistics.py | 14 ++-- test/test_trusted_user_routes.py | 14 ++-- test/test_tu_vote.py | 4 +- test/test_tu_voteinfo.py | 4 +- test/test_tuvotereminder.py | 8 +- test/test_user.py | 42 +++++----- 34 files changed, 265 insertions(+), 203 deletions(-) rename aurweb/routers/{trusted_user.py => package_maintainer.py} (88%) create mode 100644 migrations/versions/6a64dd126029_rename_tu_to_package_maintainer.py diff --git a/aurweb/auth/__init__.py b/aurweb/auth/__init__.py index 83dd424c..e895dcdb 100644 --- a/aurweb/auth/__init__.py +++ b/aurweb/auth/__init__.py @@ -71,7 +71,7 @@ class AnonymousUser: return False @staticmethod - def is_trusted_user(): + def is_package_maintainer(): return False @staticmethod @@ -205,7 +205,7 @@ def account_type_required(one_of: set): @router.get('/some_route') @auth_required(True) - @account_type_required({"Trusted User", "Trusted User & Developer"}) + @account_type_required({"Package Maintainer", "Package Maintainer & Developer"}) async def some_route(request: fastapi.Request): return Response() diff --git a/aurweb/auth/creds.py b/aurweb/auth/creds.py index 17d02a5b..594188ca 100644 --- a/aurweb/auth/creds.py +++ b/aurweb/auth/creds.py @@ -1,7 +1,7 @@ from aurweb.models.account_type import ( DEVELOPER_ID, - TRUSTED_USER_AND_DEV_ID, - TRUSTED_USER_ID, + PACKAGE_MAINTAINER_AND_DEV_ID, + PACKAGE_MAINTAINER_ID, USER_ID, ) from aurweb.models.user import User @@ -30,47 +30,49 @@ PKGBASE_VOTE = 16 PKGREQ_FILE = 23 PKGREQ_CLOSE = 17 PKGREQ_LIST = 18 -TU_ADD_VOTE = 19 -TU_LIST_VOTES = 20 -TU_VOTE = 21 +PM_ADD_VOTE = 19 +PM_LIST_VOTES = 20 +PM_VOTE = 21 PKGBASE_MERGE = 29 -user_developer_or_trusted_user = set( - [USER_ID, TRUSTED_USER_ID, DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID] +user_developer_or_package_maintainer = set( + [USER_ID, PACKAGE_MAINTAINER_ID, DEVELOPER_ID, PACKAGE_MAINTAINER_AND_DEV_ID] ) -trusted_user_or_dev = set([TRUSTED_USER_ID, DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID]) -developer = set([DEVELOPER_ID, TRUSTED_USER_AND_DEV_ID]) -trusted_user = set([TRUSTED_USER_ID, TRUSTED_USER_AND_DEV_ID]) +package_maintainer_or_dev = set( + [PACKAGE_MAINTAINER_ID, DEVELOPER_ID, PACKAGE_MAINTAINER_AND_DEV_ID] +) +developer = set([DEVELOPER_ID, PACKAGE_MAINTAINER_AND_DEV_ID]) +package_maintainer = set([PACKAGE_MAINTAINER_ID, PACKAGE_MAINTAINER_AND_DEV_ID]) cred_filters = { - PKGBASE_FLAG: user_developer_or_trusted_user, - PKGBASE_NOTIFY: user_developer_or_trusted_user, - PKGBASE_VOTE: user_developer_or_trusted_user, - PKGREQ_FILE: user_developer_or_trusted_user, - ACCOUNT_CHANGE_TYPE: trusted_user_or_dev, - ACCOUNT_EDIT: trusted_user_or_dev, - ACCOUNT_LAST_LOGIN: trusted_user_or_dev, - ACCOUNT_LIST_COMMENTS: trusted_user_or_dev, - ACCOUNT_SEARCH: trusted_user_or_dev, - COMMENT_DELETE: trusted_user_or_dev, - COMMENT_UNDELETE: trusted_user_or_dev, - COMMENT_VIEW_DELETED: trusted_user_or_dev, - COMMENT_EDIT: trusted_user_or_dev, - COMMENT_PIN: trusted_user_or_dev, - PKGBASE_ADOPT: trusted_user_or_dev, - PKGBASE_SET_KEYWORDS: trusted_user_or_dev, - PKGBASE_DELETE: trusted_user_or_dev, - PKGBASE_EDIT_COMAINTAINERS: trusted_user_or_dev, - PKGBASE_DISOWN: trusted_user_or_dev, - PKGBASE_LIST_VOTERS: trusted_user_or_dev, - PKGBASE_UNFLAG: trusted_user_or_dev, - PKGREQ_CLOSE: trusted_user_or_dev, - PKGREQ_LIST: trusted_user_or_dev, - TU_ADD_VOTE: trusted_user, - TU_LIST_VOTES: trusted_user_or_dev, - TU_VOTE: trusted_user, + PKGBASE_FLAG: user_developer_or_package_maintainer, + PKGBASE_NOTIFY: user_developer_or_package_maintainer, + PKGBASE_VOTE: user_developer_or_package_maintainer, + PKGREQ_FILE: user_developer_or_package_maintainer, + ACCOUNT_CHANGE_TYPE: package_maintainer_or_dev, + ACCOUNT_EDIT: package_maintainer_or_dev, + ACCOUNT_LAST_LOGIN: package_maintainer_or_dev, + ACCOUNT_LIST_COMMENTS: package_maintainer_or_dev, + ACCOUNT_SEARCH: package_maintainer_or_dev, + COMMENT_DELETE: package_maintainer_or_dev, + COMMENT_UNDELETE: package_maintainer_or_dev, + COMMENT_VIEW_DELETED: package_maintainer_or_dev, + COMMENT_EDIT: package_maintainer_or_dev, + COMMENT_PIN: package_maintainer_or_dev, + PKGBASE_ADOPT: package_maintainer_or_dev, + PKGBASE_SET_KEYWORDS: package_maintainer_or_dev, + PKGBASE_DELETE: package_maintainer_or_dev, + PKGBASE_EDIT_COMAINTAINERS: package_maintainer_or_dev, + PKGBASE_DISOWN: package_maintainer_or_dev, + PKGBASE_LIST_VOTERS: package_maintainer_or_dev, + PKGBASE_UNFLAG: package_maintainer_or_dev, + PKGREQ_CLOSE: package_maintainer_or_dev, + PKGREQ_LIST: package_maintainer_or_dev, + PM_ADD_VOTE: package_maintainer, + PM_LIST_VOTES: package_maintainer_or_dev, + PM_VOTE: package_maintainer, ACCOUNT_EDIT_DEV: developer, - PKGBASE_MERGE: trusted_user_or_dev, + PKGBASE_MERGE: package_maintainer_or_dev, } diff --git a/aurweb/initdb.py b/aurweb/initdb.py index ee59212c..7181ea3e 100644 --- a/aurweb/initdb.py +++ b/aurweb/initdb.py @@ -13,9 +13,9 @@ def feed_initial_data(conn): aurweb.schema.AccountTypes.insert(), [ {"ID": 1, "AccountType": "User"}, - {"ID": 2, "AccountType": "Trusted User"}, + {"ID": 2, "AccountType": "Package Maintainer"}, {"ID": 3, "AccountType": "Developer"}, - {"ID": 4, "AccountType": "Trusted User & Developer"}, + {"ID": 4, "AccountType": "Package Maintainer & Developer"}, ], ) conn.execute( diff --git a/aurweb/models/account_type.py b/aurweb/models/account_type.py index 315800a7..70bfc2c5 100644 --- a/aurweb/models/account_type.py +++ b/aurweb/models/account_type.py @@ -2,21 +2,21 @@ from aurweb import schema from aurweb.models.declarative import Base USER = "User" -TRUSTED_USER = "Trusted User" +PACKAGE_MAINTAINER = "Package Maintainer" DEVELOPER = "Developer" -TRUSTED_USER_AND_DEV = "Trusted User & Developer" +PACKAGE_MAINTAINER_AND_DEV = "Package Maintainer & Developer" USER_ID = 1 -TRUSTED_USER_ID = 2 +PACKAGE_MAINTAINER_ID = 2 DEVELOPER_ID = 3 -TRUSTED_USER_AND_DEV_ID = 4 +PACKAGE_MAINTAINER_AND_DEV_ID = 4 # Map string constants to integer constants. ACCOUNT_TYPE_ID = { USER: USER_ID, - TRUSTED_USER: TRUSTED_USER_ID, + PACKAGE_MAINTAINER: PACKAGE_MAINTAINER_ID, DEVELOPER: DEVELOPER_ID, - TRUSTED_USER_AND_DEV: TRUSTED_USER_AND_DEV_ID, + PACKAGE_MAINTAINER_AND_DEV: PACKAGE_MAINTAINER_AND_DEV_ID, } # Reversed ACCOUNT_TYPE_ID mapping. diff --git a/aurweb/models/user.py b/aurweb/models/user.py index 8612c259..f90d19eb 100644 --- a/aurweb/models/user.py +++ b/aurweb/models/user.py @@ -157,25 +157,25 @@ class User(Base): with db.begin(): db.delete(self.session) - def is_trusted_user(self): + def is_package_maintainer(self): return self.AccountType.ID in { - aurweb.models.account_type.TRUSTED_USER_ID, - aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_AND_DEV_ID, } def is_developer(self): return self.AccountType.ID in { aurweb.models.account_type.DEVELOPER_ID, - aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_AND_DEV_ID, } def is_elevated(self): """A User is 'elevated' when they have either a Trusted User or Developer AccountType.""" return self.AccountType.ID in { - aurweb.models.account_type.TRUSTED_USER_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_ID, aurweb.models.account_type.DEVELOPER_ID, - aurweb.models.account_type.TRUSTED_USER_AND_DEV_ID, + aurweb.models.account_type.PACKAGE_MAINTAINER_AND_DEV_ID, } def can_edit_user(self, target: "User") -> bool: diff --git a/aurweb/pkgbase/actions.py b/aurweb/pkgbase/actions.py index 00efc1ff..f3688f54 100644 --- a/aurweb/pkgbase/actions.py +++ b/aurweb/pkgbase/actions.py @@ -187,7 +187,7 @@ def pkgbase_merge_instance( # Log this out for accountability purposes. logger.info( - f"Trusted User '{request.user.Username}' merged " + f"Package Maintainer '{request.user.Username}' merged " f"'{pkgbasename}' into '{target.Name}'." ) diff --git a/aurweb/routers/__init__.py b/aurweb/routers/__init__.py index f77bce4f..ccd70662 100644 --- a/aurweb/routers/__init__.py +++ b/aurweb/routers/__init__.py @@ -7,13 +7,13 @@ from . import ( accounts, auth, html, + package_maintainer, packages, pkgbase, requests, rpc, rss, sso, - trusted_user, ) """ @@ -28,7 +28,7 @@ APP_ROUTES = [ packages, pkgbase, requests, - trusted_user, + package_maintainer, rss, rpc, sso, diff --git a/aurweb/routers/accounts.py b/aurweb/routers/accounts.py index 1c81ec1d..a2d167bc 100644 --- a/aurweb/routers/accounts.py +++ b/aurweb/routers/accounts.py @@ -184,9 +184,9 @@ def make_account_form_context( lambda e: request.user.AccountTypeID >= e[0], [ (at.USER_ID, f"Normal {at.USER}"), - (at.TRUSTED_USER_ID, at.TRUSTED_USER), + (at.PACKAGE_MAINTAINER_ID, at.PACKAGE_MAINTAINER), (at.DEVELOPER_ID, at.DEVELOPER), - (at.TRUSTED_USER_AND_DEV_ID, at.TRUSTED_USER_AND_DEV), + (at.PACKAGE_MAINTAINER_AND_DEV_ID, at.PACKAGE_MAINTAINER_AND_DEV), ], ) ) @@ -520,7 +520,9 @@ async def account_comments(request: Request, username: str): @router.get("/accounts") @requires_auth -@account_type_required({at.TRUSTED_USER, at.DEVELOPER, at.TRUSTED_USER_AND_DEV}) +@account_type_required( + {at.PACKAGE_MAINTAINER, at.DEVELOPER, at.PACKAGE_MAINTAINER_AND_DEV} +) async def accounts(request: Request): context = make_context(request, "Accounts") return render_template(request, "account/search.html", context) @@ -529,7 +531,9 @@ async def accounts(request: Request): @router.post("/accounts") @handle_form_exceptions @requires_auth -@account_type_required({at.TRUSTED_USER, at.DEVELOPER, at.TRUSTED_USER_AND_DEV}) +@account_type_required( + {at.PACKAGE_MAINTAINER, at.DEVELOPER, at.PACKAGE_MAINTAINER_AND_DEV} +) async def accounts_post( request: Request, O: int = Form(default=0), # Offset @@ -564,9 +568,9 @@ async def accounts_post( # Convert parameter T to an AccountType ID. account_types = { "u": at.USER_ID, - "t": at.TRUSTED_USER_ID, + "t": at.PACKAGE_MAINTAINER_ID, "d": at.DEVELOPER_ID, - "td": at.TRUSTED_USER_AND_DEV_ID, + "td": at.PACKAGE_MAINTAINER_AND_DEV_ID, } account_type_id = account_types.get(T, None) diff --git a/aurweb/routers/trusted_user.py b/aurweb/routers/package_maintainer.py similarity index 88% rename from aurweb/routers/trusted_user.py rename to aurweb/routers/package_maintainer.py index 4248347d..c5e70dcf 100644 --- a/aurweb/routers/trusted_user.py +++ b/aurweb/routers/package_maintainer.py @@ -11,7 +11,10 @@ from aurweb import aur_logging, db, l10n, models, time from aurweb.auth import creds, requires_auth from aurweb.exceptions import handle_form_exceptions from aurweb.models import User -from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID +from aurweb.models.account_type import ( + PACKAGE_MAINTAINER_AND_DEV_ID, + PACKAGE_MAINTAINER_ID, +) from aurweb.templates import make_context, make_variable_context, render_template router = APIRouter() @@ -26,32 +29,32 @@ ADDVOTE_SPECIFICS = { # When a proposal is added, duration is added to the current # timestamp. # "addvote_type": (duration, quorum) - "add_tu": (7 * 24 * 60 * 60, 0.66), - "remove_tu": (7 * 24 * 60 * 60, 0.75), - "remove_inactive_tu": (5 * 24 * 60 * 60, 0.66), + "add_pm": (7 * 24 * 60 * 60, 0.66), + "remove_pm": (7 * 24 * 60 * 60, 0.75), + "remove_inactive_pm": (5 * 24 * 60 * 60, 0.66), "bylaws": (7 * 24 * 60 * 60, 0.75), } -def populate_trusted_user_counts(context: dict[str, Any]) -> None: - tu_query = db.query(User).filter( +def populate_package_maintainer_counts(context: dict[str, Any]) -> None: + pm_query = db.query(User).filter( or_( - User.AccountTypeID == TRUSTED_USER_ID, - User.AccountTypeID == TRUSTED_USER_AND_DEV_ID, + User.AccountTypeID == PACKAGE_MAINTAINER_ID, + User.AccountTypeID == PACKAGE_MAINTAINER_AND_DEV_ID, ) ) - context["trusted_user_count"] = tu_query.count() + context["package_maintainer_count"] = pm_query.count() # In case any records have a None InactivityTS. - active_tu_query = tu_query.filter( + active_pm_query = pm_query.filter( or_(User.InactivityTS.is_(None), User.InactivityTS == 0) ) - context["active_trusted_user_count"] = active_tu_query.count() + context["active_package_maintainer_count"] = active_pm_query.count() @router.get("/tu") @requires_auth -async def trusted_user( +async def package_maintainer( request: Request, coff: int = 0, # current offset cby: str = "desc", # current by @@ -63,7 +66,7 @@ async def trusted_user( if not request.user.has_credential(creds.TU_LIST_VOTES): return RedirectResponse("/", status_code=HTTPStatus.SEE_OTHER) - context = make_context(request, "Trusted User") + context = make_context(request, "Package Maintainer") current_by, past_by = cby, pby current_off, past_off = coff, poff @@ -108,7 +111,7 @@ async def trusted_user( context["past_off"] = past_off last_vote = func.max(models.TUVote.VoteID).label("LastVote") - last_votes_by_tu = ( + last_votes_by_pm = ( db.query(models.TUVote) .join(models.User) .join(models.TUVoteInfo, models.TUVoteInfo.ID == models.TUVote.VoteID) @@ -124,12 +127,12 @@ async def trusted_user( .group_by(models.TUVote.UserID) .order_by(last_vote.desc(), models.User.Username.asc()) ) - context["last_votes_by_tu"] = last_votes_by_tu.all() + context["last_votes_by_pm"] = last_votes_by_pm.all() context["current_by_next"] = "asc" if current_by == "desc" else "desc" context["past_by_next"] = "asc" if past_by == "desc" else "desc" - populate_trusted_user_counts(context) + populate_package_maintainer_counts(context) context["q"] = { "coff": current_off, @@ -178,11 +181,11 @@ def render_proposal( @router.get("/tu/{proposal}") @requires_auth -async def trusted_user_proposal(request: Request, proposal: int): +async def package_maintainer_proposal(request: Request, proposal: int): if not request.user.has_credential(creds.TU_LIST_VOTES): return RedirectResponse("/tu", status_code=HTTPStatus.SEE_OTHER) - context = await make_variable_context(request, "Trusted User") + context = await make_variable_context(request, "Package Maintainer") proposal = int(proposal) voteinfo = ( @@ -221,13 +224,13 @@ async def trusted_user_proposal(request: Request, proposal: int): @router.post("/tu/{proposal}") @handle_form_exceptions @requires_auth -async def trusted_user_proposal_post( +async def package_maintainer_proposal_post( request: Request, proposal: int, decision: str = Form(...) ): if not request.user.has_credential(creds.TU_LIST_VOTES): return RedirectResponse("/tu", status_code=HTTPStatus.SEE_OTHER) - context = await make_variable_context(request, "Trusted User") + context = await make_variable_context(request, "Package Maintainer") proposal = int(proposal) # Make sure it's an int. voteinfo = ( @@ -285,8 +288,8 @@ async def trusted_user_proposal_post( @router.get("/addvote") @requires_auth -async def trusted_user_addvote( - request: Request, user: str = str(), type: str = "add_tu", agenda: str = str() +async def package_maintainer_addvote( + request: Request, user: str = str(), type: str = "add_pm", agenda: str = str() ): if not request.user.has_credential(creds.TU_ADD_VOTE): return RedirectResponse("/tu", status_code=HTTPStatus.SEE_OTHER) @@ -295,7 +298,7 @@ async def trusted_user_addvote( if type not in ADDVOTE_SPECIFICS: context["error"] = "Invalid type." - type = "add_tu" # Default it. + type = "add_pm" # Default it. context["user"] = user context["type"] = type @@ -308,7 +311,7 @@ async def trusted_user_addvote( @router.post("/addvote") @handle_form_exceptions @requires_auth -async def trusted_user_addvote_post( +async def package_maintainer_addvote_post( request: Request, user: str = Form(default=str()), type: str = Form(default=str()), @@ -352,7 +355,7 @@ async def trusted_user_addvote_post( if type not in ADDVOTE_SPECIFICS: context["error"] = "Invalid type." - context["type"] = type = "add_tu" # Default for rendering. + context["type"] = type = "add_pm" # Default for rendering. return render_addvote(context, HTTPStatus.BAD_REQUEST) if not agenda: @@ -364,11 +367,11 @@ async def trusted_user_addvote_post( timestamp = time.utcnow() # Active TU types we filter for. - types = {TRUSTED_USER_ID, TRUSTED_USER_AND_DEV_ID} + types = {PACKAGE_MAINTAINER_ID, PACKAGE_MAINTAINER_AND_DEV_ID} # Create a new TUVoteInfo (proposal)! with db.begin(): - active_tus = ( + active_pms = ( db.query(User) .filter( and_( @@ -386,7 +389,7 @@ async def trusted_user_addvote_post( Submitted=timestamp, End=(timestamp + duration), Quorum=quorum, - ActiveTUs=active_tus, + ActiveTUs=active_pms, Submitter=request.user, ) diff --git a/aurweb/statistics.py b/aurweb/statistics.py index f301b59c..00a5c151 100644 --- a/aurweb/statistics.py +++ b/aurweb/statistics.py @@ -3,7 +3,11 @@ from sqlalchemy import func from aurweb import config, db, time from aurweb.cache import db_count_cache, db_query_cache from aurweb.models import PackageBase, PackageRequest, RequestType, User -from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID +from aurweb.models.account_type import ( + PACKAGE_MAINTAINER_AND_DEV_ID, + PACKAGE_MAINTAINER_ID, + USER_ID, +) from aurweb.models.package_request import ( ACCEPTED_ID, CLOSED_ID, @@ -22,7 +26,7 @@ HOMEPAGE_COUNTERS = [ "year_old_updated", "never_updated", "user_count", - "trusted_user_count", + "package_maintainer_count", ] REQUEST_COUNTERS = [ "total_requests", @@ -32,7 +36,7 @@ REQUEST_COUNTERS = [ "rejected_requests", ] PROMETHEUS_USER_COUNTERS = [ - ("trusted_user_count", "tu"), + ("package_maintainer_count", "package_maintainer"), ("regular_user_count", "user"), ] PROMETHEUS_PACKAGE_COUNTERS = [ @@ -92,12 +96,12 @@ class Statistics: # Users case "user_count": query = self.user_query - case "trusted_user_count": + case "package_maintainer_count": query = self.user_query.filter( User.AccountTypeID.in_( ( - TRUSTED_USER_ID, - TRUSTED_USER_AND_DEV_ID, + PACKAGE_MAINTAINER_ID, + PACKAGE_MAINTAINER_AND_DEV_ID, ) ) ) diff --git a/aurweb/users/validate.py b/aurweb/users/validate.py index 8fc68864..5f1fcd43 100644 --- a/aurweb/users/validate.py +++ b/aurweb/users/validate.py @@ -220,7 +220,7 @@ def invalid_account_type( raise ValidationError([error]) logger.debug( - f"Trusted User '{request.user.Username}' has " + f"Package Maintainer '{request.user.Username}' has " f"modified '{user.Username}' account's type to" f" {name}." ) diff --git a/migrations/versions/6a64dd126029_rename_tu_to_package_maintainer.py b/migrations/versions/6a64dd126029_rename_tu_to_package_maintainer.py new file mode 100644 index 00000000..005549b0 --- /dev/null +++ b/migrations/versions/6a64dd126029_rename_tu_to_package_maintainer.py @@ -0,0 +1,37 @@ +"""Rename TU to Package Maintainer + +Revision ID: 6a64dd126029 +Revises: c5a6a9b661a0 +Create Date: 2023-09-01 13:48:15.315244 + +""" +from aurweb import db +from aurweb.models import AccountType + +# revision identifiers, used by Alembic. +revision = "6a64dd126029" +down_revision = "c5a6a9b661a0" +branch_labels = None +depends_on = None + +# AccountTypes +# ID 2 -> Trusted User / Package Maintainer +# ID 4 -> Trusted User & Developer / Package Maintainer & Developer + + +def upgrade(): + with db.begin(): + tu = db.query(AccountType).filter(AccountType.ID == 2).first() + tudev = db.query(AccountType).filter(AccountType.ID == 4).first() + + tu.AccountType = "Package Maintainer" + tudev.AccountType = "Package Maintainer & Developer" + + +def downgrade(): + with db.begin(): + pm = db.query(AccountType).filter(AccountType.ID == 2).first() + pmdev = db.query(AccountType).filter(AccountType.ID == 4).first() + + pm.AccountType = "Trusted User" + pmdev.AccountType = "Trusted User & Developer" diff --git a/schema/gendummydata.py b/schema/gendummydata.py index dfc8eee5..25f85c74 100755 --- a/schema/gendummydata.py +++ b/schema/gendummydata.py @@ -156,9 +156,9 @@ contents = None # developer/tu IDs # developers = [] -trustedusers = [] +packagemaintainers = [] has_devs = 0 -has_tus = 0 +has_pms = 0 # Just let python throw the errors if any happen # @@ -170,7 +170,7 @@ out.write("BEGIN;\n") log.debug("Creating SQL statements for users.") for u in user_keys: account_type = 1 # default to normal user - if not has_devs or not has_tus: + if not has_devs or not has_pms: account_type = random.randrange(1, 4) if account_type == 3 and not has_devs: # this will be a dev account @@ -178,12 +178,12 @@ for u in user_keys: developers.append(seen_users[u]) if len(developers) >= MAX_DEVS * MAX_USERS: has_devs = 1 - elif account_type == 2 and not has_tus: + elif account_type == 2 and not has_pms: # this will be a trusted user account # - trustedusers.append(seen_users[u]) - if len(trustedusers) >= MAX_TUS * MAX_USERS: - has_tus = 1 + packagemaintainers.append(seen_users[u]) + if len(packagemaintainers) >= MAX_TUS * MAX_USERS: + has_pms = 1 else: # a normal user account # @@ -205,8 +205,10 @@ for u in user_keys: out.write(s) log.debug("Number of developers: %d" % len(developers)) -log.debug("Number of trusted users: %d" % len(trustedusers)) -log.debug("Number of users: %d" % (MAX_USERS - len(developers) - len(trustedusers))) +log.debug("Number of package maintainers: %d" % len(packagemaintainers)) +log.debug( + "Number of users: %d" % (MAX_USERS - len(developers) - len(packagemaintainers)) +) log.debug("Number of packages: %d" % MAX_PKGS) log.debug("Gathering text from fortune file...") @@ -224,8 +226,8 @@ for p in list(seen_pkgs.keys()): muid = developers[random.randrange(0, len(developers))] puid = developers[random.randrange(0, len(developers))] else: - muid = trustedusers[random.randrange(0, len(trustedusers))] - puid = trustedusers[random.randrange(0, len(trustedusers))] + muid = packagemaintainers[random.randrange(0, len(packagemaintainers))] + puid = packagemaintainers[random.randrange(0, len(packagemaintainers))] if count % 20 == 0: # every so often, there are orphans... muid = "NULL" @@ -339,7 +341,7 @@ for p in seen_pkgs_keys: # Create trusted user proposals # -log.debug("Creating SQL statements for trusted user proposals.") +log.debug("Creating SQL statements for package maintainer proposals.") count = 0 for t in range(0, OPEN_PROPOSALS + CLOSE_PROPOSALS): now = int(time.time()) @@ -353,7 +355,7 @@ for t in range(0, OPEN_PROPOSALS + CLOSE_PROPOSALS): user = "" else: user = user_keys[random.randrange(0, len(user_keys))] - suid = trustedusers[random.randrange(0, len(trustedusers))] + suid = packagemaintainers[random.randrange(0, len(packagemaintainers))] s = ( "INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End," " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n" diff --git a/templates/addvote.html b/templates/addvote.html index 8777cbf3..30b65c0e 100644 --- a/templates/addvote.html +++ b/templates/addvote.html @@ -19,22 +19,22 @@

- + - +

diff --git a/templates/addvote.html b/templates/addvote.html index 30b65c0e..cc12f42b 100644 --- a/templates/addvote.html +++ b/templates/addvote.html @@ -24,21 +24,21 @@ selected {% endif %} > - {{ "Addition of a TU" | tr }} + {{ "Addition of a Package Maintainer" | tr }}

  • - {% trans %}Trusted User{% endtrans %} + {% trans %}Package Maintainer{% endtrans %}
    • {% endif %} diff --git a/templates/partials/packages/statistics.html b/templates/partials/packages/statistics.html index 7c3c3ef6..7ce5fba1 100644 --- a/templates/partials/packages/statistics.html +++ b/templates/partials/packages/statistics.html @@ -42,7 +42,7 @@ - {{ "Trusted Users" | tr }} + {{ "Package Maintainers" | tr }} {{ package_maintainer_count }} diff --git a/templates/partials/support.html b/templates/partials/support.html index a2890cc5..b175a040 100644 --- a/templates/partials/support.html +++ b/templates/partials/support.html @@ -10,7 +10,7 @@

    • {% trans %}Orphan Request{% endtrans %}: {% trans %}Request a package to be disowned, e.g. when the maintainer is inactive and the package has been flagged out-of-date for a long time.{% endtrans %}
      • -
    • {% trans %}Deletion Request{% endtrans %}: {%trans %}Request a package to be removed from the Arch User Repository. Please do not use this if a package is broken and can be fixed easily. Instead, contact the package maintainer and file orphan request if necessary.{% endtrans %}
      • +
    • {% trans %}Deletion Request{% endtrans %}: {%trans %}Request a package to be removed from the Arch User Repository. Please do not use this if a package is broken and can be fixed easily. Instead, contact the maintainer and file orphan request if necessary.{% endtrans %}
    • {% trans %}Merge Request{% endtrans %}: {% trans %}Request a package to be merged into another one. Can be used when a package needs to be renamed or replaced by a split package.{% endtrans %}

    @@ -44,7 +44,7 @@

    {% trans %}Discussion{% endtrans %}

    - {{ "General discussion regarding the Arch User Repository (AUR) and Trusted User structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." + {{ "General discussion regarding the Arch User Repository (AUR) and Package Maintainer structure takes place on %saur-general%s. For discussion relating to the development of the AUR web interface, use the %saur-dev%s mailing list." | tr | format('', "", '', "") @@ -55,7 +55,7 @@

    {% trans %}Bug Reporting{% endtrans %}

    - {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the package maintainer or leave a comment on the appropriate package page." + {{ "If you find a bug in the AUR web interface, please fill out a bug report on our %sbug tracker%s. Use the tracker to report bugs in the AUR web interface %sonly%s. To report packaging bugs contact the maintainer or leave a comment on the appropriate package page." | tr | format('', "", "", "") diff --git a/templates/partials/tu/proposal/details.html b/templates/partials/tu/proposal/details.html index 4cbee9ad..c74a5c5e 100644 --- a/templates/partials/tu/proposal/details.html +++ b/templates/partials/tu/proposal/details.html @@ -22,7 +22,7 @@

    - {{ "Active" | tr }} {{ "Trusted Users" | tr }} {{ "assigned" | tr }}: + {{ "Active" | tr }} {{ "Package Maintainers" | tr }} {{ "assigned" | tr }}: {{ voteinfo.ActiveTUs }}
    diff --git a/templates/pkgbase/request.html b/templates/pkgbase/request.html index 61654a49..3ffa2d2d 100644 --- a/templates/pkgbase/request.html +++ b/templates/pkgbase/request.html @@ -69,8 +69,8 @@

    {{ - "By submitting a deletion request, you ask a Trusted " - "User to delete the package base. This type of " + "By submitting a deletion request, you ask a Package " + "Maintainer to delete the package base. This type of " "request should be used for duplicates, software " "abandoned by upstream, as well as illegal and " "irreparably broken packages." | tr @@ -79,8 +79,8 @@