diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index cf83014..d6442e8 100755 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -10,7 +10,6 @@ on: jobs: sync: runs-on: ubuntu-latest - timeout-minutes: 90 steps: - uses: actions/checkout@v4 @@ -27,4 +26,4 @@ jobs: GITEETOCKEN: ${{ secrets.GITEETOCKEN }} run: | cd sync-github-to-gitee - python3 -u sync.py "$GITHUBUSER" "$GITHUBTOKEN" "$GITEEUSER" "$GITEETOCKEN" + python3 sync.py "$GITHUBUSER" "$GITHUBTOKEN" "$GITEEUSER" "$GITEETOCKEN" diff --git a/sync-github-to-gitee/sync.py b/sync-github-to-gitee/sync.py index 2718c57..00276a5 100644 --- a/sync-github-to-gitee/sync.py +++ b/sync-github-to-gitee/sync.py @@ -7,23 +7,19 @@ python3 sync.py 流程: - 1. 通过 GitHub REST API（Bearer token 认证）分页获取 GXDE-OS 组织下的全部仓库 + 1. 通过 GitHub REST API 分页获取 GXDE-OS 组织下的全部仓库列表（带 token 失败时回退匿名） 2. 多线程 clone --bare 拉取 GitHub 仓库 3. 逐个 push --mirror 推送到 Gitee，失败自动重试并在结束时汇总报告 注意: - Gitee 侧的同名仓库需要预先存在，push --mirror 会覆盖其分支与标签 - GitHub 只接受 Personal Access Token 作为密码（需具备组织仓库的读取权限） - - GitHub 匿名 API 只有 60 次/小时且按出口 IP 共享，故不做匿名回退： - token 缺失或失效时直接报错退出，避免静默同步到不完整的仓库列表 """ import json import os -import shutil import subprocess import sys import time -import urllib.error import urllib.parse import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed @@ -33,53 +29,41 @@ cloneDir = os.path.join(programPath, "git-clone") orgName = "GXDE-OS" perPage = 100 threadMax = 8 -retryMax = 3 -gitTimeout = 600 -# git 自带低速检测：连续 60 秒低于 1KB/s 即中止，避免跨境网络长时间挂起 -gitOpts = ["git", "-c", "http.lowSpeedLimit=1000", "-c", "http.lowSpeedTime=60"] - - -def log(msg): - # Actions 下 stdout 默认全缓冲，必须 flush 才能实时看到进度 - print(msg, flush=True) def usage(): - log("用法: python3 sync.py ") + print("用法: python3 sync.py ") sys.exit(1) def apiGet(path, params, token): - """调用 GitHub REST API 的单个请求，失败时给出可读的状态码与响应体""" + """调用 GitHub REST API 的单个请求""" params["per_page"] = str(perPage) url = "https://api.github.com" + path + "?" + urllib.parse.urlencode(params) headers = { "User-Agent": "GXDE-Sync/1.0", "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", - "Authorization": "Bearer " + token, } + if token: + headers["Authorization"] = "Bearer " + token req = urllib.request.Request(url, headers=headers) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - return json.loads(resp.read().decode("utf-8")) - except urllib.error.HTTPError as e: - body = e.read().decode("utf-8", "replace")[:500] - raise RuntimeError(f"HTTP {e.code} {body}") from e - except urllib.error.URLError as e: - raise RuntimeError(f"网络错误: {e.reason}") from e + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) def getGithubRepos(token): - """分页获取 GitHub 组织下全部仓库的 name（即 URL 路径中的仓库名）""" + """分页获取 GitHub 组织下全部仓库的 name（即 URL 路径中的仓库名） + + 带 token 时 type=all 可包含私有仓库；匿名请求只返回公开仓库。 + GitHub 的 name 可直接拼接 clone/push URL，无需像 Gitee 那样取 path 字段。 + """ repoList = [] page = 1 while True: - data = apiGet(f"/orgs/{orgName}/repos", {"page": str(page), "type": "all"}, token) - if not isinstance(data, list): - # 例如限流时返回 {"message": "API rate limit exceeded ..."} - raise RuntimeError(f"返回内容不是仓库列表: {str(data)[:500]}") - if not data: + params = {"page": str(page), "type": "all"} + data = apiGet(f"/orgs/{orgName}/repos", params, token) + if not isinstance(data, list) or len(data) == 0: break repoList.extend([i["name"] for i in data]) if len(data) < perPage: @@ -88,74 +72,54 @@ def getGithubRepos(token): return repoList -def runGit(args, timeout=gitTimeout): - """执行 git 命令；超时后终止子进程并返回失败结果，避免 job 永久挂起""" - try: - return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, timeout=timeout) - except subprocess.TimeoutExpired as e: - out = e.output or "" - if isinstance(out, bytes): - out = out.decode("utf-8", "replace") - return subprocess.CompletedProcess(args, 1, out + f"\n[超时] 超过 {timeout} 秒已终止") - - -def hideSecrets(out, pullUrl, pushUrl): - """git 报错会带上含账号密码的 URL，打印前先脱敏""" - return (out or "").replace(pullUrl, "/").replace(pushUrl, "/") +def runGit(args, cwd=None): + return subprocess.run(args, cwd=cwd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) def syncRepo(name, pullUrl, pushUrl): repoDir = os.path.join(cloneDir, name + ".git") try: - # 1. 从 GitHub clone --bare（重试 retryMax 次） + # 1. 从 GitHub clone --bare（重试 3 次） ok = False - for attempt in range(1, retryMax + 1): - shutil.rmtree(repoDir, ignore_errors=True) # 清掉上次失败的残留目录 - log(f"[....] clone ({attempt}/{retryMax}): {name}") - result = runGit(gitOpts + ["clone", "--bare", pullUrl + name, repoDir]) + lastOut = "" + for attempt in range(3): + result = runGit(["git", "clone", "--bare", pullUrl + name, repoDir]) if result.returncode == 0: ok = True break - log(f"[WARN] clone 失败 ({attempt}/{retryMax}): {name}\n" - f"{hideSecrets(result.stdout, pullUrl, pushUrl)}") + lastOut = result.stdout time.sleep(5) if not ok: - log(f"[FAIL] clone 放弃: {name}") + print(f"[FAIL] clone 失败: {name}\n{lastOut}") return False - # 2. push --mirror 到 Gitee（重试 retryMax 次） + # 2. push --mirror 到 Gitee（重试 3 次） ok = False - for attempt in range(1, retryMax + 1): - log(f"[....] push ({attempt}/{retryMax}): {name}") - result = runGit(gitOpts + ["-C", repoDir, "push", "--mirror", pushUrl + name]) + lastOut = "" + for attempt in range(3): + result = runGit(["git", "-C", repoDir, "push", "--mirror", pushUrl + name]) if result.returncode == 0: ok = True break - log(f"[WARN] push 失败 ({attempt}/{retryMax}): {name}\n" - f"{hideSecrets(result.stdout, pullUrl, pushUrl)}") + lastOut = result.stdout time.sleep(5) if not ok: - log(f"[FAIL] push 放弃: {name}") + print(f"[FAIL] push 失败: {name}\n{lastOut}") return False - log(f"[ OK ] 同步完成: {name}") + print(f"[ OK ] 同步完成: {name}") return True finally: - shutil.rmtree(repoDir, ignore_errors=True) + # 清理临时目录 + runGit(["rm", "-rf", repoDir]) def main(): - os.environ["GIT_TERMINAL_PROMPT"] = "0" # 凭证失效时不要等交互式输入 - if len(sys.argv) < 5: usage() - githubUser, githubToken, giteeUser, giteeToken = sys.argv[1:5] - for label, value in (("github用户名", githubUser), ("githubToken", githubToken), - ("gitee用户名", giteeUser), ("gitee密码或Token", giteeToken)): - if not value: - log(f"[FAIL] 参数 {label} 为空，请检查 workflow 中的 secrets 配置") - sys.exit(1) + githubUser, githubToken = sys.argv[1], sys.argv[2] + giteeUser, giteeToken = sys.argv[3], sys.argv[4] githubAuth = urllib.parse.quote(githubUser, safe="") + ":" + urllib.parse.quote(githubToken, safe="") giteeAuth = urllib.parse.quote(giteeUser, safe="") + ":" + urllib.parse.quote(giteeToken, safe="") @@ -164,36 +128,29 @@ def main(): os.makedirs(cloneDir, exist_ok=True) + # 获取仓库列表：优先带 token（可含私有仓库），失败则回退到匿名（仅公开仓库） try: repoList = getGithubRepos(githubToken) - except Exception as e: - log(f"[FAIL] 获取 GitHub 仓库列表失败: {e}") - log(" 请确认 GITHUBTOKEN 有效未过期，且对 GXDE-OS 组织有读取权限") - sys.exit(1) - log(f"共获取到 {len(repoList)} 个仓库") + except Exception: + repoList = getGithubRepos("") + print(f"共获取到 {len(repoList)} 个仓库") if not repoList: - log("[FAIL] 仓库列表为空，退出") + print("未能获取仓库列表，退出") sys.exit(1) failed = [] with ThreadPoolExecutor(max_workers=threadMax) as pool: futures = {pool.submit(syncRepo, n, pullUrl, pushUrl): n for n in repoList} for future in as_completed(futures): - name = futures[future] - try: - ok = future.result() - except Exception as e: - log(f"[FAIL] 同步异常: {name}: {e}") - ok = False - if not ok: - failed.append(name) + if not future.result(): + failed.append(futures[future]) - log("=" * 50) - log(f"同步完成: 成功 {len(repoList) - len(failed)} / {len(repoList)}") + print("=" * 50) + print(f"同步完成: 成功 {len(repoList) - len(failed)} / {len(repoList)}") if failed: - log("以下仓库同步失败:") + print("以下仓库同步失败:") for name in failed: - log(" - " + name) + print(" -", name) sys.exit(1)