[IMP] script: shallow push by default in git server
Repos like odoo and OpenUpgrade blocked for 5+ minutes trying to unshallow from GitHub (~3GB). Push shallow by default with receive.shallowUpdate on bare repos. Add --unshallow flag to opt into full history fetch. Generated by Claude Code 2.1.72 model claude-sonnet-4-6 Co-Authored-By: Mathieu Benoit <mathben@technolibre.ca>
This commit is contained in:
parent
24bcb3a7d3
commit
84a021258e
1 changed files with 92 additions and 38 deletions
|
|
@ -136,6 +136,15 @@ Use --production-ready for /srv/git (requires root).
|
||||||
" uses git:// protocol (default: file)"
|
" uses git:// protocol (default: file)"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--unshallow",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"Fetch full history for shallow repos before"
|
||||||
|
" push (slow for large repos like odoo)."
|
||||||
|
" Default: push shallow for performance"
|
||||||
|
),
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-j",
|
"-j",
|
||||||
"--jobs",
|
"--jobs",
|
||||||
|
|
@ -284,6 +293,16 @@ async def _init_single_bare_repo(git_path, project, semaphore):
|
||||||
)
|
)
|
||||||
open(export_file, "w").close()
|
open(export_file, "w").close()
|
||||||
|
|
||||||
|
# Allow pushing from shallow clones
|
||||||
|
await _run_git(
|
||||||
|
"git",
|
||||||
|
"-C",
|
||||||
|
bare_path,
|
||||||
|
"config",
|
||||||
|
"receive.shallowUpdate",
|
||||||
|
"true",
|
||||||
|
)
|
||||||
|
|
||||||
return "created"
|
return "created"
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -508,12 +527,60 @@ async def _update_bare_head(git_path, project):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _try_unshallow(repo_path, project, remote_name):
|
||||||
|
"""Try to unshallow a repo by fetching full history."""
|
||||||
|
shallow_file = os.path.join(
|
||||||
|
repo_path, ".git", "shallow"
|
||||||
|
)
|
||||||
|
_logger.info(
|
||||||
|
f" Unshallowing {project['path']}..."
|
||||||
|
)
|
||||||
|
stdout, _, _ = await _run_git(
|
||||||
|
"git", "-C", repo_path, "remote"
|
||||||
|
)
|
||||||
|
remotes = [
|
||||||
|
r
|
||||||
|
for r in stdout.strip().split("\n")
|
||||||
|
if r and r != remote_name
|
||||||
|
]
|
||||||
|
manifest_remote = project.get("remote", "")
|
||||||
|
if manifest_remote in remotes:
|
||||||
|
remotes.remove(manifest_remote)
|
||||||
|
remotes.insert(0, manifest_remote)
|
||||||
|
|
||||||
|
for try_remote in remotes:
|
||||||
|
try:
|
||||||
|
await _run_git(
|
||||||
|
"git",
|
||||||
|
"-C",
|
||||||
|
repo_path,
|
||||||
|
"fetch",
|
||||||
|
"--unshallow",
|
||||||
|
try_remote,
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
continue
|
||||||
|
if not os.path.exists(shallow_file):
|
||||||
|
_logger.info(
|
||||||
|
f" Unshallowed via {try_remote}"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if os.path.exists(shallow_file):
|
||||||
|
_logger.warning(
|
||||||
|
f" Unshallow failed for"
|
||||||
|
f" {project['path']},"
|
||||||
|
" falling back to shallow push"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _push_single_repo(
|
async def _push_single_repo(
|
||||||
erplibre_root,
|
erplibre_root,
|
||||||
git_path,
|
git_path,
|
||||||
project,
|
project,
|
||||||
remote_name,
|
remote_name,
|
||||||
push_all_branches,
|
push_all_branches,
|
||||||
|
unshallow,
|
||||||
semaphore,
|
semaphore,
|
||||||
):
|
):
|
||||||
"""Push a single repo to local remote (async)."""
|
"""Push a single repo to local remote (async)."""
|
||||||
|
|
@ -525,52 +592,36 @@ async def _push_single_repo(
|
||||||
_logger.warning(f" Not found: {repo_path}")
|
_logger.warning(f" Not found: {repo_path}")
|
||||||
return "error", False
|
return "error", False
|
||||||
|
|
||||||
# Unshallow if needed (clone-depth in manifest)
|
# Handle shallow repos
|
||||||
shallow_file = os.path.join(
|
shallow_file = os.path.join(
|
||||||
repo_path, ".git", "shallow"
|
repo_path, ".git", "shallow"
|
||||||
)
|
)
|
||||||
if os.path.exists(shallow_file):
|
if os.path.exists(shallow_file):
|
||||||
_logger.info(
|
if unshallow:
|
||||||
f" Unshallowing {project['path']}..."
|
await _try_unshallow(
|
||||||
)
|
repo_path, project, remote_name
|
||||||
stdout, _, _ = await _run_git(
|
)
|
||||||
"git", "-C", repo_path, "remote"
|
else:
|
||||||
)
|
# Enable shallow push on bare repo
|
||||||
remotes = [
|
repo_name = project["name"]
|
||||||
r
|
if not repo_name.endswith(".git"):
|
||||||
for r in stdout.strip().split("\n")
|
repo_name += ".git"
|
||||||
if r and r != remote_name
|
bare_path = os.path.join(
|
||||||
]
|
git_path, repo_name
|
||||||
manifest_remote = project.get("remote", "")
|
)
|
||||||
if manifest_remote in remotes:
|
if os.path.isdir(bare_path):
|
||||||
remotes.remove(manifest_remote)
|
|
||||||
remotes.insert(0, manifest_remote)
|
|
||||||
|
|
||||||
for try_remote in remotes:
|
|
||||||
try:
|
|
||||||
await _run_git(
|
await _run_git(
|
||||||
"git",
|
"git",
|
||||||
"-C",
|
"-C",
|
||||||
repo_path,
|
bare_path,
|
||||||
"fetch",
|
"config",
|
||||||
"--unshallow",
|
"receive.shallowUpdate",
|
||||||
try_remote,
|
"true",
|
||||||
timeout=300,
|
|
||||||
)
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
continue
|
|
||||||
if not os.path.exists(shallow_file):
|
|
||||||
_logger.info(
|
|
||||||
f" Unshallowed via {try_remote}"
|
|
||||||
)
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
if os.path.exists(shallow_file):
|
|
||||||
_logger.warning(
|
|
||||||
f" Unshallow failed for"
|
|
||||||
f" {project['path']},"
|
|
||||||
" push may fail"
|
|
||||||
)
|
)
|
||||||
|
_logger.info(
|
||||||
|
f" Shallow push for"
|
||||||
|
f" {project['path']}"
|
||||||
|
)
|
||||||
|
|
||||||
# Handle detached HEAD: checkout manifest branch
|
# Handle detached HEAD: checkout manifest branch
|
||||||
did_checkout = False
|
did_checkout = False
|
||||||
|
|
@ -641,6 +692,7 @@ async def push_to_local(
|
||||||
projects,
|
projects,
|
||||||
remote_name,
|
remote_name,
|
||||||
push_all_branches,
|
push_all_branches,
|
||||||
|
unshallow,
|
||||||
jobs,
|
jobs,
|
||||||
):
|
):
|
||||||
"""Push to local remote for each repo (async)."""
|
"""Push to local remote for each repo (async)."""
|
||||||
|
|
@ -654,6 +706,7 @@ async def push_to_local(
|
||||||
p,
|
p,
|
||||||
remote_name,
|
remote_name,
|
||||||
push_all_branches,
|
push_all_branches,
|
||||||
|
unshallow,
|
||||||
semaphore,
|
semaphore,
|
||||||
)
|
)
|
||||||
for p in projects
|
for p in projects
|
||||||
|
|
@ -751,6 +804,7 @@ async def run_actions(config, erplibre_root, projects):
|
||||||
projects,
|
projects,
|
||||||
config.remote_name,
|
config.remote_name,
|
||||||
config.push_all_branches,
|
config.push_all_branches,
|
||||||
|
config.unshallow,
|
||||||
jobs,
|
jobs,
|
||||||
)
|
)
|
||||||
print()
|
print()
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue