mirror of
https://gitlab.archlinux.org/archlinux/aurweb.git
synced 2025-02-03 10:43:03 +01:00
Use gitnamespaces for efficient storage
Instead of using one Git repository per package, use a single large object storage for space efficiency. The refs of the individual package bases are divided using gitnamespaces(7) which allows for exposing each namespace as an independent repository easily. Also, git-serve is modified to create a branch for each package, allowing to browse the large repository with cgit. Helped-by: Florian Pritz <bluewind@xinu.at> Helped-by: Johannes Löthberg <johannes@kyriasis.com> Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org>
This commit is contained in:
Lukas Fleischer
parent
ecfa27e406
commit
b44411fb53
9 changed files with 80 additions and 132 deletions
|
|
@ -1,29 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import configparser
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
config = configparser.RawConfigParser()
|
||||
config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config")
|
||||
|
||||
template_path = config.get('serve', 'template-path')
|
||||
git_update_hook = config.get('serve', 'git-update-hook')
|
||||
|
||||
def die(msg):
|
||||
sys.stderr.write("%s\n" % (msg))
|
||||
exit(1)
|
||||
|
||||
if os.path.exists(template_path):
|
||||
shutil.rmtree(template_path)
|
||||
|
||||
os.mkdir(template_path)
|
||||
os.chdir(template_path)
|
||||
os.mkdir("branches")
|
||||
os.mkdir("hooks")
|
||||
os.mkdir("info")
|
||||
os.symlink(git_update_hook, template_path + 'hooks/update')
|
||||
|
||||
with open("description", 'w') as f:
|
||||
f.write("Unnamed repository; push to update the description.\n")
|
||||
|
|
@ -17,28 +17,23 @@ aur_db_user = config.get('database', 'user')
|
|||
aur_db_pass = config.get('database', 'password')
|
||||
aur_db_socket = config.get('database', 'socket')
|
||||
|
||||
repo_base_path = config.get('serve', 'repo-base')
|
||||
repo_path = config.get('serve', 'repo-path')
|
||||
repo_regex = config.get('serve', 'repo-regex')
|
||||
git_shell_cmd = config.get('serve', 'git-shell-cmd')
|
||||
ssh_cmdline = config.get('serve', 'ssh-cmdline')
|
||||
template_path = config.get('serve', 'template-path')
|
||||
|
||||
def repo_path_validate(path):
|
||||
if not path.startswith(repo_base_path):
|
||||
return False
|
||||
if path.endswith('.git'):
|
||||
repo = path[len(repo_base_path):-4]
|
||||
elif path.endswith('.git/'):
|
||||
repo = path[len(repo_base_path):-5]
|
||||
else:
|
||||
return False
|
||||
return re.match(repo_regex, repo)
|
||||
def pkgbase_exists(pkgbase):
|
||||
db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
|
||||
passwd=aur_db_pass, db=aur_db_name,
|
||||
unix_socket=aur_db_socket)
|
||||
cur = db.cursor()
|
||||
|
||||
def repo_path_get_pkgbase(path):
|
||||
pkgbase = path.rstrip('/').rpartition('/')[2]
|
||||
if pkgbase.endswith('.git'):
|
||||
pkgbase = pkgbase[:-4]
|
||||
return pkgbase
|
||||
cur.execute("SELECT COUNT(*) FROM PackageBases WHERE Name = %s ",
|
||||
[pkgbase])
|
||||
|
||||
db.close()
|
||||
return (cur.fetchone()[0] > 0)
|
||||
|
||||
def list_repos(user):
|
||||
db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
|
||||
|
|
@ -57,19 +52,17 @@ def list_repos(user):
|
|||
print((' ' if row[1] else '*') + row[0])
|
||||
db.close()
|
||||
|
||||
def setup_repo(repo, user):
|
||||
if not re.match(repo_regex, repo):
|
||||
die('%s: invalid repository name: %s' % (action, repo))
|
||||
def setup_repo(pkgbase, user):
|
||||
if not re.match(repo_regex, pkgbase):
|
||||
die('%s: invalid repository name: %s' % (action, pkgbase))
|
||||
if pkgbase_exists(pkgbase):
|
||||
die('%s: package base already exists: %s' % (action, pkgbase))
|
||||
|
||||
db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
|
||||
passwd=aur_db_pass, db=aur_db_name,
|
||||
unix_socket=aur_db_socket)
|
||||
cur = db.cursor()
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM PackageBases WHERE Name = %s ", [repo])
|
||||
if cur.fetchone()[0] > 0:
|
||||
die('%s: package base already exists: %s' % (action, repo))
|
||||
|
||||
cur.execute("SELECT ID FROM Users WHERE Username = %s ", [user])
|
||||
userid = cur.fetchone()[0]
|
||||
if userid == 0:
|
||||
|
|
@ -77,7 +70,7 @@ def setup_repo(repo, user):
|
|||
|
||||
cur.execute("INSERT INTO PackageBases (Name, SubmittedTS, ModifiedTS, " +
|
||||
"SubmitterUID, MaintainerUID) VALUES (%s, UNIX_TIMESTAMP(), " +
|
||||
"UNIX_TIMESTAMP(), %s, %s)", [repo, userid, userid])
|
||||
"UNIX_TIMESTAMP(), %s, %s)", [pkgbase, userid, userid])
|
||||
pkgbase_id = cur.lastrowid
|
||||
|
||||
cur.execute("INSERT INTO CommentNotify (PackageBaseID, UserID) " +
|
||||
|
|
@ -86,8 +79,11 @@ def setup_repo(repo, user):
|
|||
db.commit()
|
||||
db.close()
|
||||
|
||||
repo_path = repo_base_path + '/' + repo + '.git/'
|
||||
pygit2.init_repository(repo_path, True, 48, template_path=template_path)
|
||||
repo = pygit2.Repository(repo_path)
|
||||
repo.create_reference('refs/heads/' + pkgbase,
|
||||
'refs/namespaces/' + pkgbase + '/refs/heads/master')
|
||||
repo.create_reference('refs/namespaces/' + pkgbase + '/HEAD',
|
||||
'refs/namespaces/' + pkgbase + '/refs/heads/master')
|
||||
|
||||
def check_permissions(pkgbase, user):
|
||||
db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
|
||||
|
|
@ -125,19 +121,25 @@ action = cmdargv[0]
|
|||
if action == 'git-upload-pack' or action == 'git-receive-pack':
|
||||
if len(cmdargv) < 2:
|
||||
die_with_help("%s: missing path" % (action))
|
||||
path = repo_base_path.rstrip('/') + cmdargv[1]
|
||||
if not repo_path_validate(path):
|
||||
|
||||
path = cmdargv[1].rstrip('/')
|
||||
if not path.startswith('/') or not path.endswith('.git'):
|
||||
die('%s: invalid path: %s' % (action, path))
|
||||
pkgbase = repo_path_get_pkgbase(path)
|
||||
if not os.path.exists(path):
|
||||
pkgbase = path[1:-4]
|
||||
if not re.match(repo_regex, pkgbase):
|
||||
die('%s: invalid repository name: %s' % (action, repo))
|
||||
|
||||
if not pkgbase_exists(pkgbase):
|
||||
setup_repo(pkgbase, user)
|
||||
|
||||
if action == 'git-receive-pack':
|
||||
if not check_permissions(pkgbase, user):
|
||||
die('%s: permission denied: %s' % (action, user))
|
||||
|
||||
os.environ["AUR_USER"] = user
|
||||
os.environ["AUR_GIT_DIR"] = path
|
||||
os.environ["AUR_PKGBASE"] = pkgbase
|
||||
cmd = action + " '" + path + "'"
|
||||
os.environ["GIT_NAMESPACE"] = pkgbase
|
||||
cmd = action + " '" + repo_path + "'"
|
||||
os.execl(git_shell_cmd, git_shell_cmd, '-c', cmd)
|
||||
elif action == 'list-repos':
|
||||
if len(cmdargv) > 1:
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ aur_db_user = config.get('database', 'user')
|
|||
aur_db_pass = config.get('database', 'password')
|
||||
aur_db_socket = config.get('database', 'socket')
|
||||
|
||||
repo_path = config.get('serve', 'repo-path')
|
||||
|
||||
def extract_arch_fields(pkginfo, field):
|
||||
values = []
|
||||
|
||||
|
|
@ -166,12 +168,11 @@ sha1_new = sys.argv[3]
|
|||
|
||||
user = os.environ.get("AUR_USER")
|
||||
pkgbase = os.environ.get("AUR_PKGBASE")
|
||||
git_dir = os.environ.get("AUR_GIT_DIR")
|
||||
|
||||
if refname != "refs/heads/master":
|
||||
die("pushing to a branch other than master is restricted")
|
||||
|
||||
repo = pygit2.Repository(git_dir)
|
||||
repo = pygit2.Repository(repo_path)
|
||||
walker = repo.walk(sha1_new, pygit2.GIT_SORT_TOPOLOGICAL)
|
||||
if sha1_old != "0000000000000000000000000000000000000000":
|
||||
walker.hide(sha1_old)
|
||||
|
|
@ -245,6 +246,6 @@ db.close()
|
|||
|
||||
pkglist = list(srcinfo.GetPackageNames())
|
||||
if len(pkglist) > 0:
|
||||
with open(git_dir + '/description', 'w') as f:
|
||||
with open(repo_path + '/description', 'w') as f:
|
||||
pkginfo = srcinfo.GetMergedPackage(pkglist[0])
|
||||
f.write(pkginfo['pkgdesc'])
|
||||
|
|
|
|||
|
|
@ -1,50 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import configparser
|
||||
import mysql.connector
|
||||
import os
|
||||
import pygit2
|
||||
import re
|
||||
import shlex
|
||||
import sys
|
||||
|
||||
config = configparser.RawConfigParser()
|
||||
config.read(os.path.dirname(os.path.realpath(__file__)) + "/../../conf/config")
|
||||
|
||||
aur_db_host = config.get('database', 'host')
|
||||
aur_db_name = config.get('database', 'name')
|
||||
aur_db_user = config.get('database', 'user')
|
||||
aur_db_pass = config.get('database', 'password')
|
||||
aur_db_socket = config.get('database', 'socket')
|
||||
|
||||
repo_base_path = config.get('serve', 'repo-base')
|
||||
repo_regex = config.get('serve', 'repo-regex')
|
||||
template_path = config.get('serve', 'template-path')
|
||||
|
||||
def die(msg):
|
||||
sys.stderr.write("%s\n" % (msg))
|
||||
exit(1)
|
||||
|
||||
db = mysql.connector.connect(host=aur_db_host, user=aur_db_user,
|
||||
passwd=aur_db_pass, db=aur_db_name,
|
||||
unix_socket=aur_db_socket)
|
||||
cur = db.cursor()
|
||||
|
||||
cur.execute("SELECT Name FROM PackageBases")
|
||||
repos = [row[0] for row in cur]
|
||||
db.close()
|
||||
|
||||
for repo in repos:
|
||||
if not re.match(repo_regex, repo):
|
||||
die('invalid repository name: %s' % (repo))
|
||||
|
||||
i = 1
|
||||
n = len(repos)
|
||||
|
||||
for repo in repos:
|
||||
print("[%s/%d] %s" % (str(i).rjust(len(str(n))), n, repo))
|
||||
|
||||
repo_path = repo_base_path + '/' + repo + '.git/'
|
||||
pygit2.init_repository(repo_path, True, 48, template_path=template_path)
|
||||
|
||||
i += 1
|
||||
Loading…
Add table
Add a link
Reference in a new issue