mathben/erplibre
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

[IMP] script: async init/remote/push in git server

Browse source 
Sequential processing of 100+ repos was slow. Use
asyncio with semaphore-bounded concurrency to run
init, remote and push actions in parallel. New -j flag
controls max parallel jobs (default: 8).

Generated by Claude Code 2.1.72 model claude-sonnet-4-6

Co-Authored-By: Mathieu Benoit <mathben@technolibre.ca>
This commit is contained in:
Mathieu Benoit 2026-03-11 02:56:30 -04:00
parent 291c9cf5e0
commit 24bcb3a7d3
1 changed files with 334 additions and 191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

525
script/git/git_local_server.py
View file

@ -3,6 +3,7 @@
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl)
import argparse import argparse
import asyncio
import logging import logging
import os import os
import subprocess import subprocess
@ -26,10 +27,38 @@ PRODUCTION_GIT_PATH = "/srv/git"
DEFAULT_MANIFEST = ".repo/local_manifests/erplibre_manifest.xml" DEFAULT_MANIFEST = ".repo/local_manifests/erplibre_manifest.xml"
DEFAULT_REMOTE_NAME = "local" DEFAULT_REMOTE_NAME = "local"
DEFAULT_PORT = 9418 DEFAULT_PORT = 9418
DEFAULT_JOBS = 8
ERPLIBRE_REPO_NAME = "erplibre/erplibre" ERPLIBRE_REPO_NAME = "erplibre/erplibre"
ERPLIBRE_REPO_URL = "https://github.com/erplibre" ERPLIBRE_REPO_URL = "https://github.com/erplibre"
async def _run_git(*args, cwd=None, timeout=None):
"""Run a git command asynchronously.
Returns (stdout, stderr, returncode).
Raises asyncio.TimeoutError on timeout.
"""
process = await asyncio.create_subprocess_exec(
*args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=cwd,
)
try:
stdout, stderr = await asyncio.wait_for(
process.communicate(), timeout=timeout
)
return (
stdout.decode(),
stderr.decode(),
process.returncode,
)
except asyncio.TimeoutError:
process.kill()
await process.communicate()
raise
def get_config(): def get_config():
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, formatter_class=argparse.RawDescriptionHelpFormatter,
@ -107,6 +136,16 @@ Use --production-ready for /srv/git (requires root).
" uses git:// protocol (default: file)" " uses git:// protocol (default: file)"
), ),
) )
parser.add_argument(
"-j",
"--jobs",
type=int,
default=DEFAULT_JOBS,
help=(
"Parallel jobs for init/remote/push"
f" (default: {DEFAULT_JOBS})"
),
)
parser.add_argument( parser.add_argument(
"-v", "-v",
"--verbose", "--verbose",
@ -214,13 +253,12 @@ def get_erplibre_root_project(erplibre_root):
} }
def init_bare_repos(git_path, projects): # --- Async workers for init ---
"""Create bare repos for all projects in the manifest."""
os.makedirs(git_path, exist_ok=True)
created = 0
skipped = 0 async def _init_single_bare_repo(git_path, project, semaphore):
for project in projects: """Create a single bare repo (async worker)."""
async with semaphore:
repo_name = project["name"] repo_name = project["name"]
if not repo_name.endswith(".git"): if not repo_name.endswith(".git"):
repo_name += ".git" repo_name += ".git"
@ -228,23 +266,50 @@ def init_bare_repos(git_path, projects):
if os.path.exists(bare_path): if os.path.exists(bare_path):
_logger.info(f" Exists: {bare_path}") _logger.info(f" Exists: {bare_path}")
skipped += 1 return "skipped"
continue
_logger.info(f" Creating: {bare_path}") _logger.info(f" Creating: {bare_path}")
subprocess.run( _, err, rc = await _run_git(
["git", "init", "--bare", bare_path], "git", "init", "--bare", bare_path
check=True,
capture_output=True,
) )
if rc != 0:
_logger.warning(
f" Init failed: {bare_path}: {err.strip()}"
)
return "error"
# Enable git daemon export # Enable git daemon export
export_file = os.path.join(bare_path, "git-daemon-export-ok") export_file = os.path.join(
bare_path, "git-daemon-export-ok"
)
open(export_file, "w").close() open(export_file, "w").close()
created += 1 return "created"
print(f"Bare repos: {created} created, {skipped} skipped (already exist)")
async def init_bare_repos(git_path, projects, jobs):
"""Create bare repos for all projects (async)."""
os.makedirs(git_path, exist_ok=True)
semaphore = asyncio.Semaphore(jobs)
results = await asyncio.gather(
*[
_init_single_bare_repo(git_path, p, semaphore)
for p in projects
]
)
created = results.count("created")
skipped = results.count("skipped")
errors = results.count("error")
print(
f"Bare repos: {created} created,"
f" {skipped} skipped (already exist),"
f" {errors} errors"
)
# --- Async workers for remote ---
def build_remote_url(git_path, repo_name, port, remote_url_type): def build_remote_url(git_path, repo_name, port, remote_url_type):
@ -257,30 +322,23 @@ def build_remote_url(git_path, repo_name, port, remote_url_type):
return os.path.join(git_path, repo_name) return os.path.join(git_path, repo_name)
def add_remotes( async def _add_single_remote(
erplibre_root, erplibre_root,
git_path, git_path,
projects, project,
remote_name, remote_name,
port, port,
remote_url_type, remote_url_type,
semaphore,
): ):
"""Add local remote to each repo. """Add or update remote for a single repo (async)."""
async with semaphore:
remote_url_type='file': uses local path, push works repo_path = os.path.join(
without daemon running. erplibre_root, project["path"]
remote_url_type='daemon': uses git://localhost URL, )
requires daemon for push.
"""
added = 0
skipped = 0
errors = 0
for project in projects:
repo_path = os.path.join(erplibre_root, project["path"])
if not os.path.isdir(repo_path): if not os.path.isdir(repo_path):
_logger.warning(f" Not found: {repo_path}") _logger.warning(f" Not found: {repo_path}")
errors += 1 return "error"
continue
repo_name = project["name"] repo_name = project["name"]
if not repo_name.endswith(".git"): if not repo_name.endswith(".git"):
@ -291,106 +349,139 @@ def add_remotes(
) )
# Check if remote already exists # Check if remote already exists
result = subprocess.run( stdout, _, _ = await _run_git(
["git", "-C", repo_path, "remote"], "git", "-C", repo_path, "remote"
capture_output=True,
text=True,
) )
existing_remotes = result.stdout.strip().split("\n") existing_remotes = stdout.strip().split("\n")
if remote_name in existing_remotes: if remote_name in existing_remotes:
# Update URL if remote exists _, err, rc = await _run_git(
subprocess.run( "git",
[ "-C",
"git", repo_path,
"-C", "remote",
repo_path, "set-url",
"remote", remote_name,
"set-url", remote_url,
remote_name,
remote_url,
],
check=True,
capture_output=True,
) )
if rc != 0:
_logger.warning(
f" set-url failed for"
f" {project['path']}: {err.strip()}"
)
return "error"
_logger.info(f" Updated: {project['path']}") _logger.info(f" Updated: {project['path']}")
skipped += 1 return "updated"
else: else:
subprocess.run( _, err, rc = await _run_git(
[ "git",
"git", "-C",
"-C", repo_path,
repo_path, "remote",
"remote", "add",
"add", remote_name,
remote_name, remote_url,
remote_url,
],
check=True,
capture_output=True,
) )
if rc != 0:
_logger.warning(
f" add failed for"
f" {project['path']}: {err.strip()}"
)
return "error"
_logger.info(f" Added: {project['path']}") _logger.info(f" Added: {project['path']}")
added += 1 return "added"
print(f"Remotes: {added} added, {skipped} updated," f" {errors} errors")
def is_detached_head(repo_path): async def add_remotes(
"""Check if a repo is in detached HEAD state.""" erplibre_root,
result = subprocess.run( git_path,
["git", "-C", repo_path, "symbolic-ref", "HEAD"], projects,
capture_output=True, remote_name,
text=True, port,
remote_url_type,
jobs,
):
"""Add local remote to each repo (async).
remote_url_type='file': uses local path, push works
without daemon running.
remote_url_type='daemon': uses git://localhost URL,
requires daemon for push.
"""
semaphore = asyncio.Semaphore(jobs)
results = await asyncio.gather(
*[
_add_single_remote(
erplibre_root,
git_path,
p,
remote_name,
port,
remote_url_type,
semaphore,
)
for p in projects
]
)
added = results.count("added")
updated = results.count("updated")
errors = results.count("error")
print(
f"Remotes: {added} added, {updated} updated,"
f" {errors} errors"
) )
return result.returncode != 0
def checkout_manifest_branch(repo_path, revision): # --- Async workers for push ---
async def _is_detached_head(repo_path):
"""Check if a repo is in detached HEAD state."""
_, _, rc = await _run_git(
"git", "-C", repo_path, "symbolic-ref", "HEAD"
)
return rc != 0
async def _checkout_manifest_branch(repo_path, revision):
"""Checkout the branch from the manifest revision. """Checkout the branch from the manifest revision.
When Google Repo syncs, repos end up in detached HEAD When Google Repo syncs, repos end up in detached HEAD
on the exact commit. We create/checkout a local branch on the exact commit. We create/checkout a local branch
matching the manifest revision so git push works. matching the manifest revision so git push works.
""" """
# Extract branch name — revision can be branch = (
# "18.0", "18.0_dev", "ERPLibre/18.0", "main", etc. revision.split("/")[-1] if "/" in revision else revision
branch = revision.split("/")[-1] if "/" in revision else revision )
# Check if local branch already exists # Check if local branch already exists
result = subprocess.run( _, _, rc = await _run_git(
[ "git",
"-C",
repo_path,
"show-ref",
"--verify",
f"refs/heads/{branch}",
)
if rc == 0:
await _run_git(
"git", "-C", repo_path, "checkout", branch
)
else:
await _run_git(
"git", "git",
"-C", "-C",
repo_path, repo_path,
"show-ref", "checkout",
"--verify", "-b",
f"refs/heads/{branch}", branch,
],
capture_output=True,
)
if result.returncode == 0:
# Branch exists, checkout it
subprocess.run(
["git", "-C", repo_path, "checkout", branch],
capture_output=True,
)
else:
# Create branch from current HEAD
subprocess.run(
[
"git",
"-C",
repo_path,
"checkout",
"-b",
branch,
],
capture_output=True,
) )
return branch return branch
def update_bare_head(git_path, project): async def _update_bare_head(git_path, project):
"""Update the HEAD of the bare repo to point to the """Update the HEAD of the bare repo to point to the
manifest branch, so git clone checks out the right manifest branch, so git clone checks out the right
branch by default.""" branch by default."""
@ -404,75 +495,74 @@ def update_bare_head(git_path, project):
revision = project.get("revision", "") revision = project.get("revision", "")
if not revision: if not revision:
return return
branch = revision.split("/")[-1] if "/" in revision else revision branch = (
subprocess.run( revision.split("/")[-1] if "/" in revision else revision
[ )
"git", await _run_git(
"-C", "git",
bare_path, "-C",
"symbolic-ref", bare_path,
"HEAD", "symbolic-ref",
f"refs/heads/{branch}", "HEAD",
], f"refs/heads/{branch}",
capture_output=True,
) )
def push_to_local( async def _push_single_repo(
erplibre_root, erplibre_root,
git_path, git_path,
projects, project,
remote_name, remote_name,
push_all_branches, push_all_branches,
semaphore,
): ):
"""Push to local remote for each repo.""" """Push a single repo to local remote (async)."""
pushed = 0 async with semaphore:
errors = 0 repo_path = os.path.join(
checkouts = 0 erplibre_root, project["path"]
for project in projects: )
repo_path = os.path.join(erplibre_root, project["path"])
if not os.path.isdir(repo_path): if not os.path.isdir(repo_path):
_logger.warning(f" Not found: {repo_path}") _logger.warning(f" Not found: {repo_path}")
errors += 1 return "error", False
continue
# Unshallow if needed (clone-depth in manifest) # Unshallow if needed (clone-depth in manifest)
shallow_file = os.path.join(repo_path, ".git", "shallow") shallow_file = os.path.join(
repo_path, ".git", "shallow"
)
if os.path.exists(shallow_file): if os.path.exists(shallow_file):
_logger.info(f" Unshallowing {project['path']}...") _logger.info(
# Try each remote until unshallow succeeds f" Unshallowing {project['path']}..."
result = subprocess.run( )
["git", "-C", repo_path, "remote"], stdout, _, _ = await _run_git(
capture_output=True, "git", "-C", repo_path, "remote"
text=True,
) )
remotes = [ remotes = [
r r
for r in result.stdout.strip().split("\n") for r in stdout.strip().split("\n")
if r and r != remote_name if r and r != remote_name
] ]
# Try the manifest remote first
manifest_remote = project.get("remote", "") manifest_remote = project.get("remote", "")
if manifest_remote in remotes: if manifest_remote in remotes:
remotes.remove(manifest_remote) remotes.remove(manifest_remote)
remotes.insert(0, manifest_remote) remotes.insert(0, manifest_remote)
for try_remote in remotes: for try_remote in remotes:
result = subprocess.run( try:
[ await _run_git(
"git", "git",
"-C", "-C",
repo_path, repo_path,
"fetch", "fetch",
"--unshallow", "--unshallow",
try_remote, try_remote,
], timeout=300,
capture_output=True, )
text=True, except asyncio.TimeoutError:
timeout=300, continue
)
if not os.path.exists(shallow_file): if not os.path.exists(shallow_file):
_logger.info(f" Unshallowed via" f" {try_remote}") _logger.info(
f" Unshallowed via {try_remote}"
)
break break
else: else:
if os.path.exists(shallow_file): if os.path.exists(shallow_file):
@ -483,24 +573,27 @@ def push_to_local(
) )
# Handle detached HEAD: checkout manifest branch # Handle detached HEAD: checkout manifest branch
if is_detached_head(repo_path): did_checkout = False
if await _is_detached_head(repo_path):
revision = project.get("revision", "") revision = project.get("revision", "")
if revision: if revision:
branch = checkout_manifest_branch(repo_path, revision) branch = await _checkout_manifest_branch(
repo_path, revision
)
_logger.info( _logger.info(
f" Checkout {branch} for" f" Checkout {branch} for"
f" {project['path']}" f" {project['path']}"
" (was detached HEAD)" " (was detached HEAD)"
) )
checkouts += 1 did_checkout = True
else: else:
_logger.warning( _logger.warning(
f" Detached HEAD with no revision" f" Detached HEAD with no revision"
f" for {project['path']}, skipping" f" for {project['path']}, skipping"
) )
errors += 1 return "error", False
continue
# Push
try: try:
if push_all_branches: if push_all_branches:
cmd = [ cmd = [
@ -519,33 +612,66 @@ def push_to_local(
"push", "push",
remote_name, remote_name,
] ]
result = subprocess.run( _, err, rc = await _run_git(
cmd, *cmd, timeout=120
capture_output=True,
text=True,
timeout=120,
) )
if result.returncode != 0: if rc != 0:
_logger.warning( _logger.warning(
f" Push failed for" f" Push failed for"
f" {project['path']}:" f" {project['path']}:"
f" {result.stderr.strip()}" f" {err.strip()}"
) )
errors += 1 return "error", did_checkout
else: else:
update_bare_head(git_path, project) await _update_bare_head(git_path, project)
_logger.info(f" Pushed: {project['path']}") _logger.info(
pushed += 1 f" Pushed: {project['path']}"
except subprocess.TimeoutExpired: )
_logger.warning(f" Timeout pushing {project['path']}") return "pushed", did_checkout
errors += 1 except asyncio.TimeoutError:
_logger.warning(
f" Timeout pushing {project['path']}"
)
return "error", did_checkout
async def push_to_local(
erplibre_root,
git_path,
projects,
remote_name,
push_all_branches,
jobs,
):
"""Push to local remote for each repo (async)."""
semaphore = asyncio.Semaphore(jobs)
results = await asyncio.gather(
*[
_push_single_repo(
erplibre_root,
git_path,
p,
remote_name,
push_all_branches,
semaphore,
)
for p in projects
]
)
pushed = sum(1 for s, _ in results if s == "pushed")
errors = sum(1 for s, _ in results if s == "error")
checkouts = sum(1 for _, c in results if c)
print( print(
f"Push: {pushed} pushed, {checkouts} branch" f"Push: {pushed} pushed, {checkouts} branch"
f" checkouts, {errors} errors" f" checkouts, {errors} errors"
) )
# --- Serve (stays synchronous — long-running daemon) ---
def print_clone_commands(git_path, projects, port): def print_clone_commands(git_path, projects, port):
"""Print git clone commands for all available repos.""" """Print git clone commands for all available repos."""
print("=== Available repos to clone ===") print("=== Available repos to clone ===")
@ -561,7 +687,10 @@ def print_clone_commands(git_path, projects, port):
repo_name += ".git" repo_name += ".git"
bare_path = os.path.join(git_path, repo_name) bare_path = os.path.join(git_path, repo_name)
if os.path.isdir(bare_path): if os.path.isdir(bare_path):
print(f" git clone {base_url}/{repo_name}" f" {project['path']}") print(
f" git clone {base_url}/{repo_name}"
f" {project['path']}"
)
count += 1 count += 1
print(f"\nTotal: {count} repos available") print(f"\nTotal: {count} repos available")
print() print()
@ -588,6 +717,45 @@ def serve_git_daemon(git_path, projects, port):
execute.exec_command_live(cmd, source_erplibre=False) execute.exec_command_live(cmd, source_erplibre=False)
# --- Main ---
async def run_actions(config, erplibre_root, projects):
"""Run init/remote/push actions asynchronously."""
action = config.action
jobs = config.jobs
if action in ("init", "all"):
print("=== Creating bare repos ===")
await init_bare_repos(config.path, projects, jobs)
print()
if action in ("remote", "all"):
print("=== Adding remotes ===")
await add_remotes(
erplibre_root,
config.path,
projects,
config.remote_name,
config.port,
config.remote_url_type,
jobs,
)
print()
if action in ("push", "all"):
print("=== Pushing to local ===")
await push_to_local(
erplibre_root,
config.path,
projects,
config.remote_name,
config.push_all_branches,
jobs,
)
print()
def main(): def main():
config = get_config() config = get_config()
@ -621,6 +789,7 @@ def main():
else: else:
print("Mode: development (~/.git-server)") print("Mode: development (~/.git-server)")
print(f"Remote URL type: {config.remote_url_type}") print(f"Remote URL type: {config.remote_url_type}")
print(f"Parallel jobs: {config.jobs}")
print() print()
projects = parse_manifest(manifest_path) projects = parse_manifest(manifest_path)
@ -633,37 +802,11 @@ def main():
) )
print() print()
action = config.action # Run async actions (init, remote, push)
asyncio.run(run_actions(config, erplibre_root, projects))
if action in ("init", "all"): # Serve is synchronous (long-running daemon)
print("=== Creating bare repos ===") if config.action in ("serve", "all"):
init_bare_repos(config.path, projects)
print()
if action in ("remote", "all"):
print("=== Adding remotes ===")
add_remotes(
erplibre_root,
config.path,
projects,
config.remote_name,
config.port,
config.remote_url_type,
)
print()
if action in ("push", "all"):
print("=== Pushing to local ===")
push_to_local(
erplibre_root,
config.path,
projects,
config.remote_name,
config.push_all_branches,
)
print()
if action in ("serve", "all"):
print("=== Starting git daemon ===") print("=== Starting git daemon ===")
serve_git_daemon(config.path, projects, config.port) serve_git_daemon(config.path, projects, config.port)