diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index d6442e8..cf83014 100755 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -10,6 +10,7 @@ on: jobs: sync: runs-on: ubuntu-latest + timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -26,4 +27,4 @@ jobs: GITEETOCKEN: ${{ secrets.GITEETOCKEN }} run: | cd sync-github-to-gitee - python3 sync.py "$GITHUBUSER" "$GITHUBTOKEN" "$GITEEUSER" "$GITEETOCKEN" + python3 -u sync.py "$GITHUBUSER" "$GITHUBTOKEN" "$GITEEUSER" "$GITEETOCKEN" diff --git a/sync-github-to-gitee/sync.py b/sync-github-to-gitee/sync.py index 00276a5..2718c57 100644 --- a/sync-github-to-gitee/sync.py +++ b/sync-github-to-gitee/sync.py @@ -7,19 +7,23 @@ python3 sync.py 流程: - 1. 通过 GitHub REST API 分页获取 GXDE-OS 组织下的全部仓库列表(带 token 失败时回退匿名) + 1. 通过 GitHub REST API(Bearer token 认证)分页获取 GXDE-OS 组织下的全部仓库 2. 多线程 clone --bare 拉取 GitHub 仓库 3. 逐个 push --mirror 推送到 Gitee,失败自动重试并在结束时汇总报告 注意: - Gitee 侧的同名仓库需要预先存在,push --mirror 会覆盖其分支与标签 - GitHub 只接受 Personal Access Token 作为密码(需具备组织仓库的读取权限) + - GitHub 匿名 API 只有 60 次/小时且按出口 IP 共享,故不做匿名回退: + token 缺失或失效时直接报错退出,避免静默同步到不完整的仓库列表 """ import json import os +import shutil import subprocess import sys import time +import urllib.error import urllib.parse import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed @@ -29,41 +33,53 @@ cloneDir = os.path.join(programPath, "git-clone") orgName = "GXDE-OS" perPage = 100 threadMax = 8 +retryMax = 3 +gitTimeout = 600 +# git 自带低速检测:连续 60 秒低于 1KB/s 即中止,避免跨境网络长时间挂起 +gitOpts = ["git", "-c", "http.lowSpeedLimit=1000", "-c", "http.lowSpeedTime=60"] + + +def log(msg): + # Actions 下 stdout 默认全缓冲,必须 flush 才能实时看到进度 + print(msg, flush=True) def usage(): - print("用法: python3 sync.py ") + log("用法: python3 sync.py ") sys.exit(1) def apiGet(path, params, token): - """调用 GitHub REST API 的单个请求""" + """调用 GitHub REST API 的单个请求,失败时给出可读的状态码与响应体""" params["per_page"] = str(perPage) url = "https://api.github.com" + path + "?" + urllib.parse.urlencode(params) headers = { "User-Agent": "GXDE-Sync/1.0", "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", + "Authorization": "Bearer " + token, } - if token: - headers["Authorization"] = "Bearer " + token req = urllib.request.Request(url, headers=headers) - with urllib.request.urlopen(req, timeout=30) as resp: - return json.loads(resp.read().decode("utf-8")) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", "replace")[:500] + raise RuntimeError(f"HTTP {e.code} {body}") from e + except urllib.error.URLError as e: + raise RuntimeError(f"网络错误: {e.reason}") from e def getGithubRepos(token): - """分页获取 GitHub 组织下全部仓库的 name(即 URL 路径中的仓库名) - - 带 token 时 type=all 可包含私有仓库;匿名请求只返回公开仓库。 - GitHub 的 name 可直接拼接 clone/push URL,无需像 Gitee 那样取 path 字段。 - """ + """分页获取 GitHub 组织下全部仓库的 name(即 URL 路径中的仓库名)""" repoList = [] page = 1 while True: - params = {"page": str(page), "type": "all"} - data = apiGet(f"/orgs/{orgName}/repos", params, token) - if not isinstance(data, list) or len(data) == 0: + data = apiGet(f"/orgs/{orgName}/repos", {"page": str(page), "type": "all"}, token) + if not isinstance(data, list): + # 例如限流时返回 {"message": "API rate limit exceeded ..."} + raise RuntimeError(f"返回内容不是仓库列表: {str(data)[:500]}") + if not data: break repoList.extend([i["name"] for i in data]) if len(data) < perPage: @@ -72,54 +88,74 @@ def getGithubRepos(token): return repoList -def runGit(args, cwd=None): - return subprocess.run(args, cwd=cwd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, text=True) +def runGit(args, timeout=gitTimeout): + """执行 git 命令;超时后终止子进程并返回失败结果,避免 job 永久挂起""" + try: + return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=timeout) + except subprocess.TimeoutExpired as e: + out = e.output or "" + if isinstance(out, bytes): + out = out.decode("utf-8", "replace") + return subprocess.CompletedProcess(args, 1, out + f"\n[超时] 超过 {timeout} 秒已终止") + + +def hideSecrets(out, pullUrl, pushUrl): + """git 报错会带上含账号密码的 URL,打印前先脱敏""" + return (out or "").replace(pullUrl, "/").replace(pushUrl, "/") def syncRepo(name, pullUrl, pushUrl): repoDir = os.path.join(cloneDir, name + ".git") try: - # 1. 从 GitHub clone --bare(重试 3 次) + # 1. 从 GitHub clone --bare(重试 retryMax 次) ok = False - lastOut = "" - for attempt in range(3): - result = runGit(["git", "clone", "--bare", pullUrl + name, repoDir]) + for attempt in range(1, retryMax + 1): + shutil.rmtree(repoDir, ignore_errors=True) # 清掉上次失败的残留目录 + log(f"[....] clone ({attempt}/{retryMax}): {name}") + result = runGit(gitOpts + ["clone", "--bare", pullUrl + name, repoDir]) if result.returncode == 0: ok = True break - lastOut = result.stdout + log(f"[WARN] clone 失败 ({attempt}/{retryMax}): {name}\n" + f"{hideSecrets(result.stdout, pullUrl, pushUrl)}") time.sleep(5) if not ok: - print(f"[FAIL] clone 失败: {name}\n{lastOut}") + log(f"[FAIL] clone 放弃: {name}") return False - # 2. push --mirror 到 Gitee(重试 3 次) + # 2. push --mirror 到 Gitee(重试 retryMax 次) ok = False - lastOut = "" - for attempt in range(3): - result = runGit(["git", "-C", repoDir, "push", "--mirror", pushUrl + name]) + for attempt in range(1, retryMax + 1): + log(f"[....] push ({attempt}/{retryMax}): {name}") + result = runGit(gitOpts + ["-C", repoDir, "push", "--mirror", pushUrl + name]) if result.returncode == 0: ok = True break - lastOut = result.stdout + log(f"[WARN] push 失败 ({attempt}/{retryMax}): {name}\n" + f"{hideSecrets(result.stdout, pullUrl, pushUrl)}") time.sleep(5) if not ok: - print(f"[FAIL] push 失败: {name}\n{lastOut}") + log(f"[FAIL] push 放弃: {name}") return False - print(f"[ OK ] 同步完成: {name}") + log(f"[ OK ] 同步完成: {name}") return True finally: - # 清理临时目录 - runGit(["rm", "-rf", repoDir]) + shutil.rmtree(repoDir, ignore_errors=True) def main(): + os.environ["GIT_TERMINAL_PROMPT"] = "0" # 凭证失效时不要等交互式输入 + if len(sys.argv) < 5: usage() - githubUser, githubToken = sys.argv[1], sys.argv[2] - giteeUser, giteeToken = sys.argv[3], sys.argv[4] + githubUser, githubToken, giteeUser, giteeToken = sys.argv[1:5] + for label, value in (("github用户名", githubUser), ("githubToken", githubToken), + ("gitee用户名", giteeUser), ("gitee密码或Token", giteeToken)): + if not value: + log(f"[FAIL] 参数 {label} 为空,请检查 workflow 中的 secrets 配置") + sys.exit(1) githubAuth = urllib.parse.quote(githubUser, safe="") + ":" + urllib.parse.quote(githubToken, safe="") giteeAuth = urllib.parse.quote(giteeUser, safe="") + ":" + urllib.parse.quote(giteeToken, safe="") @@ -128,29 +164,36 @@ def main(): os.makedirs(cloneDir, exist_ok=True) - # 获取仓库列表:优先带 token(可含私有仓库),失败则回退到匿名(仅公开仓库) try: repoList = getGithubRepos(githubToken) - except Exception: - repoList = getGithubRepos("") - print(f"共获取到 {len(repoList)} 个仓库") + except Exception as e: + log(f"[FAIL] 获取 GitHub 仓库列表失败: {e}") + log(" 请确认 GITHUBTOKEN 有效未过期,且对 GXDE-OS 组织有读取权限") + sys.exit(1) + log(f"共获取到 {len(repoList)} 个仓库") if not repoList: - print("未能获取仓库列表,退出") + log("[FAIL] 仓库列表为空,退出") sys.exit(1) failed = [] with ThreadPoolExecutor(max_workers=threadMax) as pool: futures = {pool.submit(syncRepo, n, pullUrl, pushUrl): n for n in repoList} for future in as_completed(futures): - if not future.result(): - failed.append(futures[future]) + name = futures[future] + try: + ok = future.result() + except Exception as e: + log(f"[FAIL] 同步异常: {name}: {e}") + ok = False + if not ok: + failed.append(name) - print("=" * 50) - print(f"同步完成: 成功 {len(repoList) - len(failed)} / {len(repoList)}") + log("=" * 50) + log(f"同步完成: 成功 {len(repoList) - len(failed)} / {len(repoList)}") if failed: - print("以下仓库同步失败:") + log("以下仓库同步失败:") for name in failed: - print(" -", name) + log(" - " + name) sys.exit(1)