#!/usr/bin/env python3 # © 2025-2026 TechnoLibre (http://www.technolibre.ca) # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import argparse import asyncio import logging import os import subprocess import sys import xml.etree.ElementTree as ET new_path = os.path.normpath( os.path.join(os.path.dirname(__file__), "..", "..") ) sys.path.append(new_path) from script.execute.execute import Execute _logger = logging.getLogger(__name__) DEFAULT_ERPLIBRE_PATH = os.path.normpath( os.path.join(os.path.dirname(__file__), "..", "..") ) DEFAULT_GIT_PATH = os.path.join(os.path.expanduser("~"), ".git-server") PRODUCTION_GIT_PATH = "/srv/git" DEFAULT_MANIFEST = ".repo/local_manifests/erplibre_manifest.xml" DEFAULT_REMOTE_NAME = "local" DEFAULT_PORT = 9418 DEFAULT_JOBS = 8 ERPLIBRE_REPO_NAME = "erplibre/erplibre" ERPLIBRE_REPO_URL = "https://github.com/erplibre" async def _run_git(*args, cwd=None, timeout=None): """Run a git command asynchronously. Returns (stdout, stderr, returncode). Raises asyncio.TimeoutError on timeout. """ process = await asyncio.create_subprocess_exec( *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=cwd, ) try: stdout, stderr = await asyncio.wait_for( process.communicate(), timeout=timeout ) return ( stdout.decode(), stderr.decode(), process.returncode, ) except asyncio.TimeoutError: process.kill() await process.communicate() raise def get_config(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description="""\ Deploy a local git server using git daemon and configure all repos from the ERPLibre manifest. Actions (in order): 1. init - Create bare repos in the git server path 2. remote - Add 'local' remote to each repo 3. push - Push all branches to the local remote 4. serve - Start git daemon By default, all actions are executed sequentially. The default path is ~/.git-server (user-level, no root needed). Use --production-ready for /srv/git (requires root). """, ) parser.add_argument( "-p", "--path", default=None, help=( "Path for git server bare repos" f" (default: {DEFAULT_GIT_PATH})" ), ) parser.add_argument( "--production-ready", action="store_true", help=( f"Use {PRODUCTION_GIT_PATH} as git server path" " (requires root privileges)" ), ) parser.add_argument( "-m", "--manifest", default=DEFAULT_MANIFEST, help="Manifest XML file" f" (default: {DEFAULT_MANIFEST})", ) parser.add_argument( "--remote-name", default=DEFAULT_REMOTE_NAME, help="Remote name to add" f" (default: {DEFAULT_REMOTE_NAME})", ) parser.add_argument( "--port", type=int, default=DEFAULT_PORT, help=f"Git daemon port (default: {DEFAULT_PORT})", ) parser.add_argument( "--action", choices=["init", "remote", "push", "serve", "all"], default="all", help="Action to perform (default: all)", ) parser.add_argument( "--erplibre-root", default=None, help="ERPLibre root directory (default: auto-detect)", ) parser.add_argument( "--push-all-branches", action="store_true", help="Push all local branches, not just the current one", ) parser.add_argument( "--remote-url-type", choices=["file", "daemon"], default="file", help=( "Remote URL type: 'file' uses local path" " (push works without daemon), 'daemon'" " uses git:// protocol (default: file)" ), ) parser.add_argument( "--unshallow", action="store_true", help=( "Fetch full history for shallow repos before" " push (slow for large repos like odoo)." " Default: push shallow for performance" ), ) parser.add_argument( "-j", "--jobs", type=int, default=DEFAULT_JOBS, help=( "Parallel jobs for init/remote/push" f" (default: {DEFAULT_JOBS})" ), ) parser.add_argument( "-v", "--verbose", action="store_true", help="Enable verbose output", ) args = parser.parse_args() # Resolve git server path if args.path and args.production_ready: parser.error("--path and --production-ready are mutually exclusive") if args.production_ready: args.path = PRODUCTION_GIT_PATH elif args.path is None: args.path = DEFAULT_GIT_PATH return args def check_path_permissions(path): """Check if we have write permissions on the target path. Try the path itself first, then its closest existing parent directory.""" check = path while check and not os.path.exists(check): check = os.path.dirname(check) if not check: check = "/" if os.access(check, os.W_OK): return True return False def require_permissions_or_exit(path): """Exit with instructions if we cannot write to path.""" print(f"Error: No write permission on {path}") print( "Please run this script with appropriate" " privileges:" f"\n sudo {sys.executable}" f" {' '.join(sys.argv)}" ) sys.exit(1) def parse_manifest(manifest_path): """Parse the manifest XML and return list of projects.""" tree = ET.parse(manifest_path) root = tree.getroot() remotes = {} for remote in root.findall("remote"): remotes[remote.get("name")] = remote.get("fetch") projects = [] for project in root.findall("project"): name = project.get("name") path = project.get("path") remote = project.get("remote") revision = project.get("revision") projects.append( { "name": name, "path": path, "remote": remote, "revision": revision, "fetch_url": remotes.get(remote, ""), } ) return projects def get_erplibre_root_project(erplibre_root): """Build a project entry for the ERPLibre root repo. The root repo is not in the manifest (it is managed by git directly, not Google Repo) but needs to be included in the local git server. """ result = subprocess.run( [ "git", "-C", erplibre_root, "rev-parse", "--abbrev-ref", "HEAD", ], capture_output=True, text=True, ) revision = result.stdout.strip() if result.returncode == 0 else "master" return { "name": ERPLIBRE_REPO_NAME, "path": ".", "remote": "origin", "revision": revision, "fetch_url": ERPLIBRE_REPO_URL, } # --- Async workers for init --- async def _init_single_bare_repo(git_path, project, semaphore): """Create a single bare repo (async worker).""" async with semaphore: repo_name = project["name"] if not repo_name.endswith(".git"): repo_name += ".git" bare_path = os.path.join(git_path, repo_name) if os.path.exists(bare_path): _logger.info(f" Exists: {bare_path}") return "skipped" _logger.info(f" Creating: {bare_path}") _, err, rc = await _run_git("git", "init", "--bare", bare_path) if rc != 0: _logger.warning(f" Init failed: {bare_path}: {err.strip()}") return "error" # Enable git daemon export export_file = os.path.join(bare_path, "git-daemon-export-ok") open(export_file, "w").close() # Allow pushing from shallow clones await _run_git( "git", "-C", bare_path, "config", "receive.shallowUpdate", "true", ) return "created" async def init_bare_repos(git_path, projects, jobs): """Create bare repos for all projects (async).""" os.makedirs(git_path, exist_ok=True) semaphore = asyncio.Semaphore(jobs) results = await asyncio.gather( *[_init_single_bare_repo(git_path, p, semaphore) for p in projects] ) created = results.count("created") skipped = results.count("skipped") errors = results.count("error") print( f"Bare repos: {created} created," f" {skipped} skipped (already exist)," f" {errors} errors" ) # --- Async workers for remote --- def build_remote_url(git_path, repo_name, port, remote_url_type): """Build the remote URL based on the type.""" if remote_url_type == "daemon": if port != DEFAULT_PORT: return f"git://localhost:{port}/{repo_name}" return f"git://localhost/{repo_name}" # Default: file path return os.path.join(git_path, repo_name) async def _add_single_remote( erplibre_root, git_path, project, remote_name, port, remote_url_type, semaphore, ): """Add or update remote for a single repo (async).""" async with semaphore: repo_path = os.path.join(erplibre_root, project["path"]) if not os.path.isdir(repo_path): _logger.warning(f" Not found: {repo_path}") return "error" repo_name = project["name"] if not repo_name.endswith(".git"): repo_name += ".git" remote_url = build_remote_url( git_path, repo_name, port, remote_url_type ) # Check if remote already exists stdout, _, _ = await _run_git("git", "-C", repo_path, "remote") existing_remotes = stdout.strip().split("\n") if remote_name in existing_remotes: _, err, rc = await _run_git( "git", "-C", repo_path, "remote", "set-url", remote_name, remote_url, ) if rc != 0: _logger.warning( f" set-url failed for" f" {project['path']}: {err.strip()}" ) return "error" _logger.info(f" Updated: {project['path']}") return "updated" else: _, err, rc = await _run_git( "git", "-C", repo_path, "remote", "add", remote_name, remote_url, ) if rc != 0: _logger.warning( f" add failed for" f" {project['path']}: {err.strip()}" ) return "error" _logger.info(f" Added: {project['path']}") return "added" async def add_remotes( erplibre_root, git_path, projects, remote_name, port, remote_url_type, jobs, ): """Add local remote to each repo (async). remote_url_type='file': uses local path, push works without daemon running. remote_url_type='daemon': uses git://localhost URL, requires daemon for push. """ semaphore = asyncio.Semaphore(jobs) results = await asyncio.gather( *[ _add_single_remote( erplibre_root, git_path, p, remote_name, port, remote_url_type, semaphore, ) for p in projects ] ) added = results.count("added") updated = results.count("updated") errors = results.count("error") print(f"Remotes: {added} added, {updated} updated," f" {errors} errors") # --- Async workers for push --- async def _is_detached_head(repo_path): """Check if a repo is in detached HEAD state.""" _, _, rc = await _run_git("git", "-C", repo_path, "symbolic-ref", "HEAD") return rc != 0 async def _checkout_manifest_branch(repo_path, revision): """Checkout the branch from the manifest revision. When Google Repo syncs, repos end up in detached HEAD on the exact commit. We create/checkout a local branch matching the manifest revision so git push works. """ branch = revision.split("/")[-1] if "/" in revision else revision # Check if local branch already exists _, _, rc = await _run_git( "git", "-C", repo_path, "show-ref", "--verify", f"refs/heads/{branch}", ) if rc == 0: await _run_git("git", "-C", repo_path, "checkout", branch) else: await _run_git( "git", "-C", repo_path, "checkout", "-b", branch, ) return branch async def _update_bare_head(git_path, project): """Update the HEAD of the bare repo to point to the manifest branch, so git clone checks out the right branch by default.""" repo_name = project["name"] if not repo_name.endswith(".git"): repo_name += ".git" bare_path = os.path.join(git_path, repo_name) if not os.path.isdir(bare_path): return revision = project.get("revision", "") if not revision: return branch = revision.split("/")[-1] if "/" in revision else revision await _run_git( "git", "-C", bare_path, "symbolic-ref", "HEAD", f"refs/heads/{branch}", ) async def _try_unshallow(repo_path, project, remote_name): """Try to unshallow a repo by fetching full history.""" shallow_file = os.path.join(repo_path, ".git", "shallow") _logger.info(f" Unshallowing {project['path']}...") stdout, _, _ = await _run_git("git", "-C", repo_path, "remote") remotes = [r for r in stdout.strip().split("\n") if r and r != remote_name] manifest_remote = project.get("remote", "") if manifest_remote in remotes: remotes.remove(manifest_remote) remotes.insert(0, manifest_remote) for try_remote in remotes: try: await _run_git( "git", "-C", repo_path, "fetch", "--unshallow", try_remote, timeout=300, ) except asyncio.TimeoutError: continue if not os.path.exists(shallow_file): _logger.info(f" Unshallowed via {try_remote}") return if os.path.exists(shallow_file): _logger.warning( f" Unshallow failed for" f" {project['path']}," " falling back to shallow push" ) async def _push_single_repo( erplibre_root, git_path, project, remote_name, push_all_branches, unshallow, semaphore, ): """Push a single repo to local remote (async).""" async with semaphore: repo_path = os.path.join(erplibre_root, project["path"]) if not os.path.isdir(repo_path): _logger.warning(f" Not found: {repo_path}") return "error", False # Handle shallow repos shallow_file = os.path.join(repo_path, ".git", "shallow") if os.path.exists(shallow_file): if unshallow: await _try_unshallow(repo_path, project, remote_name) else: # Enable shallow push on bare repo repo_name = project["name"] if not repo_name.endswith(".git"): repo_name += ".git" bare_path = os.path.join(git_path, repo_name) if os.path.isdir(bare_path): await _run_git( "git", "-C", bare_path, "config", "receive.shallowUpdate", "true", ) _logger.info(f" Shallow push for" f" {project['path']}") # Handle detached HEAD: checkout manifest branch did_checkout = False if await _is_detached_head(repo_path): revision = project.get("revision", "") if revision: branch = await _checkout_manifest_branch(repo_path, revision) _logger.info( f" Checkout {branch} for" f" {project['path']}" " (was detached HEAD)" ) did_checkout = True else: _logger.warning( f" Detached HEAD with no revision" f" for {project['path']}, skipping" ) return "error", False # Push try: if push_all_branches: cmd = [ "git", "-C", repo_path, "push", remote_name, "--all", ] else: cmd = [ "git", "-C", repo_path, "push", remote_name, ] _, err, rc = await _run_git(*cmd, timeout=120) if rc != 0: _logger.warning( f" Push failed for" f" {project['path']}:" f" {err.strip()}" ) return "error", did_checkout else: await _update_bare_head(git_path, project) _logger.info(f" Pushed: {project['path']}") return "pushed", did_checkout except asyncio.TimeoutError: _logger.warning(f" Timeout pushing {project['path']}") return "error", did_checkout async def push_to_local( erplibre_root, git_path, projects, remote_name, push_all_branches, unshallow, jobs, ): """Push to local remote for each repo (async).""" semaphore = asyncio.Semaphore(jobs) results = await asyncio.gather( *[ _push_single_repo( erplibre_root, git_path, p, remote_name, push_all_branches, unshallow, semaphore, ) for p in projects ] ) pushed = sum(1 for s, _ in results if s == "pushed") errors = sum(1 for s, _ in results if s == "error") checkouts = sum(1 for _, c in results if c) print( f"Push: {pushed} pushed, {checkouts} branch" f" checkouts, {errors} errors" ) # --- Serve (stays synchronous — long-running daemon) --- def print_clone_commands(git_path, projects, port): """Print git clone commands for all available repos.""" print("=== Available repos to clone ===") base_url = ( f"git://localhost:{port}" if port != DEFAULT_PORT else "git://localhost" ) lines = [] for project in projects: repo_name = project["name"] if not repo_name.endswith(".git"): repo_name += ".git" bare_path = os.path.join(git_path, repo_name) if os.path.isdir(bare_path): clone_path = project["path"] if clone_path == ".": clone_path = "erplibre" lines.append( f" git clone {base_url}/{repo_name}" f" {clone_path}" ) lines.sort() for line in lines: print(line) print(f"\nTotal: {len(lines)} repos available") print() def serve_git_daemon(git_path, projects, port): """Start git daemon to serve repos.""" print_clone_commands(git_path, projects, port) cmd = ( f"git daemon --reuseaddr" f" --base-path={git_path}" f" --port={port}" f" --export-all" f" --enable=receive-pack" f" {git_path}" ) print(f"Starting git daemon on port {port}...") print(f" Base path: {git_path}") print(f" URL: git://localhost:{port}/") print(" Press Ctrl+C to stop") execute = Execute() execute.exec_command_live(cmd, source_erplibre=False) # --- Main --- async def run_actions(config, erplibre_root, projects): """Run init/remote/push actions asynchronously.""" action = config.action jobs = config.jobs if action in ("init", "all"): print("=== Creating bare repos ===") await init_bare_repos(config.path, projects, jobs) print() if action in ("remote", "all"): print("=== Adding remotes ===") await add_remotes( erplibre_root, config.path, projects, config.remote_name, config.port, config.remote_url_type, jobs, ) print() if action in ("push", "all"): print("=== Pushing to local ===") await push_to_local( erplibre_root, config.path, projects, config.remote_name, config.push_all_branches, config.unshallow, jobs, ) print() def main(): config = get_config() log_level = logging.DEBUG if config.verbose else logging.INFO logging.basicConfig( level=log_level, format="%(levelname)s: %(message)s", ) # Check write permissions on target path if not check_path_permissions(config.path): require_permissions_or_exit(config.path) # Detect ERPLibre root if config.erplibre_root: erplibre_root = os.path.abspath(config.erplibre_root) else: erplibre_root = DEFAULT_ERPLIBRE_PATH manifest_path = os.path.join(erplibre_root, config.manifest) if not os.path.exists(manifest_path): print(f"Error: Manifest not found: {manifest_path}") sys.exit(1) print(f"ERPLibre root: {erplibre_root}") print(f"Manifest: {manifest_path}") print(f"Git server path: {config.path}") print(f"Remote name: {config.remote_name}") if config.production_ready: print("Mode: production (/srv/git)") else: print("Mode: development (~/.git-server)") print(f"Remote URL type: {config.remote_url_type}") print(f"Parallel jobs: {config.jobs}") print() projects = parse_manifest(manifest_path) erplibre_project = get_erplibre_root_project(erplibre_root) projects.append(erplibre_project) print( f"Found {len(projects)} projects" f" ({len(projects) - 1} from manifest" f" + erplibre root)" ) print() # Run async actions (init, remote, push) asyncio.run(run_actions(config, erplibre_root, projects)) # Serve is synchronous (long-running daemon) if config.action in ("serve", "all"): print("=== Starting git daemon ===") serve_git_daemon(config.path, projects, config.port) if __name__ == "__main__": main()