Revert "添加同步脚本超时时间以避免卡死"

This reverts commit 7262370817.

Signed-off-by: gfdgd_xi <3025613752@qq.com>
This commit is contained in:
2026-11-09 22:53:02 +08:00
parent 2fe440700e
commit 8312e04ea3
2 changed files with 47 additions and 91 deletions
+1 -2
View File
@@ -10,7 +10,6 @@ on:
jobs: jobs:
sync: sync:
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 90
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
@@ -27,4 +26,4 @@ jobs:
GITEETOCKEN: ${{ secrets.GITEETOCKEN }} GITEETOCKEN: ${{ secrets.GITEETOCKEN }}
run: | run: |
cd sync-github-to-gitee cd sync-github-to-gitee
python3 -u sync.py "$GITHUBUSER" "$GITHUBTOKEN" "$GITEEUSER" "$GITEETOCKEN" python3 sync.py "$GITHUBUSER" "$GITHUBTOKEN" "$GITEEUSER" "$GITEETOCKEN"
+46 -89
View File
@@ -7,23 +7,19 @@
python3 sync.py <github用户名> <githubToken> <gitee用户名> <gitee密码或Token> python3 sync.py <github用户名> <githubToken> <gitee用户名> <gitee密码或Token>
流程: 流程:
1. 通过 GitHub REST API(Bearer token 认证)分页获取 GXDE-OS 组织下的全部仓库 1. 通过 GitHub REST API 分页获取 GXDE-OS 组织下的全部仓库列表(带 token 失败时回退匿名)
2. 多线程 clone --bare 拉取 GitHub 仓库 2. 多线程 clone --bare 拉取 GitHub 仓库
3. 逐个 push --mirror 推送到 Gitee,失败自动重试并在结束时汇总报告 3. 逐个 push --mirror 推送到 Gitee,失败自动重试并在结束时汇总报告
注意: 注意:
- Gitee 侧的同名仓库需要预先存在,push --mirror 会覆盖其分支与标签 - Gitee 侧的同名仓库需要预先存在,push --mirror 会覆盖其分支与标签
- GitHub 只接受 Personal Access Token 作为密码(需具备组织仓库的读取权限) - GitHub 只接受 Personal Access Token 作为密码(需具备组织仓库的读取权限)
- GitHub 匿名 API 只有 60 次/小时且按出口 IP 共享,故不做匿名回退:
token 缺失或失效时直接报错退出,避免静默同步到不完整的仓库列表
""" """
import json import json
import os import os
import shutil
import subprocess import subprocess
import sys import sys
import time import time
import urllib.error
import urllib.parse import urllib.parse
import urllib.request import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -33,53 +29,41 @@ cloneDir = os.path.join(programPath, "git-clone")
orgName = "GXDE-OS" orgName = "GXDE-OS"
perPage = 100 perPage = 100
threadMax = 8 threadMax = 8
retryMax = 3
gitTimeout = 600
# git 自带低速检测:连续 60 秒低于 1KB/s 即中止,避免跨境网络长时间挂起
gitOpts = ["git", "-c", "http.lowSpeedLimit=1000", "-c", "http.lowSpeedTime=60"]
def log(msg):
# Actions 下 stdout 默认全缓冲,必须 flush 才能实时看到进度
print(msg, flush=True)
def usage(): def usage():
log("用法: python3 sync.py <github用户名> <githubToken> <gitee用户名> <gitee密码或Token>") print("用法: python3 sync.py <github用户名> <githubToken> <gitee用户名> <gitee密码或Token>")
sys.exit(1) sys.exit(1)
def apiGet(path, params, token): def apiGet(path, params, token):
"""调用 GitHub REST API 的单个请求,失败时给出可读的状态码与响应体""" """调用 GitHub REST API 的单个请求"""
params["per_page"] = str(perPage) params["per_page"] = str(perPage)
url = "https://api.github.com" + path + "?" + urllib.parse.urlencode(params) url = "https://api.github.com" + path + "?" + urllib.parse.urlencode(params)
headers = { headers = {
"User-Agent": "GXDE-Sync/1.0", "User-Agent": "GXDE-Sync/1.0",
"Accept": "application/vnd.github+json", "Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28", "X-GitHub-Api-Version": "2022-11-28",
"Authorization": "Bearer " + token,
} }
if token:
headers["Authorization"] = "Bearer " + token
req = urllib.request.Request(url, headers=headers) req = urllib.request.Request(url, headers=headers)
try: with urllib.request.urlopen(req, timeout=30) as resp:
with urllib.request.urlopen(req, timeout=30) as resp: return json.loads(resp.read().decode("utf-8"))
return json.loads(resp.read().decode("utf-8"))
except urllib.error.HTTPError as e:
body = e.read().decode("utf-8", "replace")[:500]
raise RuntimeError(f"HTTP {e.code} {body}") from e
except urllib.error.URLError as e:
raise RuntimeError(f"网络错误: {e.reason}") from e
def getGithubRepos(token): def getGithubRepos(token):
"""分页获取 GitHub 组织下全部仓库的 name(即 URL 路径中的仓库名)""" """分页获取 GitHub 组织下全部仓库的 name(即 URL 路径中的仓库名)
带 token 时 type=all 可包含私有仓库;匿名请求只返回公开仓库。
GitHub 的 name 可直接拼接 clone/push URL,无需像 Gitee 那样取 path 字段。
"""
repoList = [] repoList = []
page = 1 page = 1
while True: while True:
data = apiGet(f"/orgs/{orgName}/repos", {"page": str(page), "type": "all"}, token) params = {"page": str(page), "type": "all"}
if not isinstance(data, list): data = apiGet(f"/orgs/{orgName}/repos", params, token)
# 例如限流时返回 {"message": "API rate limit exceeded ..."} if not isinstance(data, list) or len(data) == 0:
raise RuntimeError(f"返回内容不是仓库列表: {str(data)[:500]}")
if not data:
break break
repoList.extend([i["name"] for i in data]) repoList.extend([i["name"] for i in data])
if len(data) < perPage: if len(data) < perPage:
@@ -88,74 +72,54 @@ def getGithubRepos(token):
return repoList return repoList
def runGit(args, timeout=gitTimeout): def runGit(args, cwd=None):
"""执行 git 命令;超时后终止子进程并返回失败结果,避免 job 永久挂起""" return subprocess.run(args, cwd=cwd, stdout=subprocess.PIPE,
try: stderr=subprocess.STDOUT, text=True)
return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, timeout=timeout)
except subprocess.TimeoutExpired as e:
out = e.output or ""
if isinstance(out, bytes):
out = out.decode("utf-8", "replace")
return subprocess.CompletedProcess(args, 1, out + f"\n[超时] 超过 {timeout} 秒已终止")
def hideSecrets(out, pullUrl, pushUrl):
"""git 报错会带上含账号密码的 URL,打印前先脱敏"""
return (out or "").replace(pullUrl, "<github>/").replace(pushUrl, "<gitee>/")
def syncRepo(name, pullUrl, pushUrl): def syncRepo(name, pullUrl, pushUrl):
repoDir = os.path.join(cloneDir, name + ".git") repoDir = os.path.join(cloneDir, name + ".git")
try: try:
# 1. 从 GitHub clone --bare(重试 retryMax 次) # 1. 从 GitHub clone --bare(重试 3 次)
ok = False ok = False
for attempt in range(1, retryMax + 1): lastOut = ""
shutil.rmtree(repoDir, ignore_errors=True) # 清掉上次失败的残留目录 for attempt in range(3):
log(f"[....] clone ({attempt}/{retryMax}): {name}") result = runGit(["git", "clone", "--bare", pullUrl + name, repoDir])
result = runGit(gitOpts + ["clone", "--bare", pullUrl + name, repoDir])
if result.returncode == 0: if result.returncode == 0:
ok = True ok = True
break break
log(f"[WARN] clone 失败 ({attempt}/{retryMax}): {name}\n" lastOut = result.stdout
f"{hideSecrets(result.stdout, pullUrl, pushUrl)}")
time.sleep(5) time.sleep(5)
if not ok: if not ok:
log(f"[FAIL] clone 放弃: {name}") print(f"[FAIL] clone 失败: {name}\n{lastOut}")
return False return False
# 2. push --mirror 到 Gitee(重试 retryMax 次) # 2. push --mirror 到 Gitee(重试 3 次)
ok = False ok = False
for attempt in range(1, retryMax + 1): lastOut = ""
log(f"[....] push ({attempt}/{retryMax}): {name}") for attempt in range(3):
result = runGit(gitOpts + ["-C", repoDir, "push", "--mirror", pushUrl + name]) result = runGit(["git", "-C", repoDir, "push", "--mirror", pushUrl + name])
if result.returncode == 0: if result.returncode == 0:
ok = True ok = True
break break
log(f"[WARN] push 失败 ({attempt}/{retryMax}): {name}\n" lastOut = result.stdout
f"{hideSecrets(result.stdout, pullUrl, pushUrl)}")
time.sleep(5) time.sleep(5)
if not ok: if not ok:
log(f"[FAIL] push 放弃: {name}") print(f"[FAIL] push 失败: {name}\n{lastOut}")
return False return False
log(f"[ OK ] 同步完成: {name}") print(f"[ OK ] 同步完成: {name}")
return True return True
finally: finally:
shutil.rmtree(repoDir, ignore_errors=True) # 清理临时目录
runGit(["rm", "-rf", repoDir])
def main(): def main():
os.environ["GIT_TERMINAL_PROMPT"] = "0" # 凭证失效时不要等交互式输入
if len(sys.argv) < 5: if len(sys.argv) < 5:
usage() usage()
githubUser, githubToken, giteeUser, giteeToken = sys.argv[1:5] githubUser, githubToken = sys.argv[1], sys.argv[2]
for label, value in (("github用户名", githubUser), ("githubToken", githubToken), giteeUser, giteeToken = sys.argv[3], sys.argv[4]
("gitee用户名", giteeUser), ("gitee密码或Token", giteeToken)):
if not value:
log(f"[FAIL] 参数 {label} 为空,请检查 workflow 中的 secrets 配置")
sys.exit(1)
githubAuth = urllib.parse.quote(githubUser, safe="") + ":" + urllib.parse.quote(githubToken, safe="") githubAuth = urllib.parse.quote(githubUser, safe="") + ":" + urllib.parse.quote(githubToken, safe="")
giteeAuth = urllib.parse.quote(giteeUser, safe="") + ":" + urllib.parse.quote(giteeToken, safe="") giteeAuth = urllib.parse.quote(giteeUser, safe="") + ":" + urllib.parse.quote(giteeToken, safe="")
@@ -164,36 +128,29 @@ def main():
os.makedirs(cloneDir, exist_ok=True) os.makedirs(cloneDir, exist_ok=True)
# 获取仓库列表:优先带 token(可含私有仓库),失败则回退到匿名(仅公开仓库)
try: try:
repoList = getGithubRepos(githubToken) repoList = getGithubRepos(githubToken)
except Exception as e: except Exception:
log(f"[FAIL] 获取 GitHub 仓库列表失败: {e}") repoList = getGithubRepos("")
log(" 请确认 GITHUBTOKEN 有效未过期,且对 GXDE-OS 组织有读取权限") print(f"共获取到 {len(repoList)} 个仓库")
sys.exit(1)
log(f"共获取到 {len(repoList)} 个仓库")
if not repoList: if not repoList:
log("[FAIL] 仓库列表为空,退出") print("未能获取仓库列表,退出")
sys.exit(1) sys.exit(1)
failed = [] failed = []
with ThreadPoolExecutor(max_workers=threadMax) as pool: with ThreadPoolExecutor(max_workers=threadMax) as pool:
futures = {pool.submit(syncRepo, n, pullUrl, pushUrl): n for n in repoList} futures = {pool.submit(syncRepo, n, pullUrl, pushUrl): n for n in repoList}
for future in as_completed(futures): for future in as_completed(futures):
name = futures[future] if not future.result():
try: failed.append(futures[future])
ok = future.result()
except Exception as e:
log(f"[FAIL] 同步异常: {name}: {e}")
ok = False
if not ok:
failed.append(name)
log("=" * 50) print("=" * 50)
log(f"同步完成: 成功 {len(repoList) - len(failed)} / {len(repoList)}") print(f"同步完成: 成功 {len(repoList) - len(failed)} / {len(repoList)}")
if failed: if failed:
log("以下仓库同步失败:") print("以下仓库同步失败:")
for name in failed: for name in failed:
log(" - " + name) print(" -", name)
sys.exit(1) sys.exit(1)