ci(github workflow): simplify sync workflow and rewrite sync script

1. 移除多余的第三方同步目标和依赖包 2. 将原bash脚本重写为python脚本，新增多线程同步、失败重试和同步报告 3. 精简工作流的环境变量和执行参数
2026-08-09 22:04:15 +08:00 · 2026-08-03 19:16:34 +08:00
parent eebd5d7817
commit a6f6c26fb9
2 changed files with 133 additions and 70 deletions
@@ -16,19 +16,15 @@ jobs:
      - name: Config
        run: |
          sudo apt update
-          sudo apt install git python3-requests -y
+          sudo apt install git -y

      - name: Push
        env: 
          GITHUBTOKEN: ${{ secrets.GITHUBTOKEN }}
          GITHUBUSER: ${{ secrets.GITHUBUSER }}
-          GITCODETOKEN: ${{ secrets.GITCODETOKEN }}
-          GITCODEUSER: ${{ secrets.GITCODEUSER }}
-          ATOMGITTOKEN: ${{ secrets.ATOMGITTOKEN }}
-          ATOMGITUSER: ${{ secrets.ATOMGITUSER }}
-          ATOMGITEEUSER: ${{ secrets.GITEEUSER }}
-          ATOMGITEETOCKEN: ${{ secrets.GITEETOCKEN }}
+          GITEEUSER: ${{ secrets.GITEEUSER }}
+          GITEETOCKEN: ${{ secrets.GITEETOCKEN }}
        run: |
          cd sync-gitee-to-github
-          python3 sync.py $ATOMGITEEUSER $ATOMGITEETOCKEN $GITHUBUSER $GITHUBTOKEN $GITCODEUSER $GITCODETOKEN $ATOMGITUSER $ATOMGITTOKEN 
+          python3 sync.py $GITEEUSER $GITEETOCKEN $GITHUBUSER $GITHUBTOKEN

@@ -1,74 +1,141 @@
-#!/bin/bash
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+将 Gitee 上的 GXDE-OS 组织仓库镜像同步到 GitHub（GXDE-OS 组织）。
+
+用法:
+    python3 sync.py <gitee用户名> <gitee密码或Token> <github用户名> <githubToken>
+
+流程:
+    1. 通过 Gitee API v5 分页获取 GXDE-OS 组织下的全部仓库列表（带 token 失败时回退匿名）
+    2. 多线程 clone --bare 拉取 Gitee 仓库
+    3. 逐个 push --mirror 推送到 GitHub，失败自动重试并在结束时汇总报告
+"""
+import json
 import os
+import subprocess
 import sys
 import time
-import requests
-import threading
+import urllib.parse
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed

-programPath = os.path.split(os.path.realpath(__file__))[0]  # 返回 string
-dataHeaders = {
-    "accept": "application/json"
-}
-giteeUser = ""
-giteePassword = ""
-if len(sys.argv) > 3:
-    giteeUser = sys.argv[1]
-    giteePassword = sys.argv[2]
-pullUrl = f"https://{giteeUser}:{giteePassword}@gitee.com/GXDE-OS/"
-pushUrl = [
-    "https://@USER@:@PASS@@github.com/GXDE-OS/",
-    "https://@USER@:@PASS@@gitcode.com/GXDE-OS/",
-    "https://@USER@:@PASS@@atomgit.com/GXDE-OS/"
-]
+programPath = os.path.split(os.path.realpath(__file__))[0]
+cloneDir = os.path.join(programPath, "git-clone")
+orgName = "GXDE-OS"
+perPage = 100
+threadMax = 8

-if (len(sys.argv) < 1 + len(pushUrl) * 2):
-    print(f"请输入用于推送的账户和密码（需要 {len(pushUrl)} 组以推送不同的平台，提供顺序如下）")
-    for i in pushUrl:
-        print(f"    {i}")
-    exit(1)

-# 读取参数
-userName = []
-password = []
+def usage():
+    print("用法: python3 sync.py <gitee用户名> <gitee密码或Token> <github用户名> <githubToken>")
+    sys.exit(1)

-for i in range(1 + 2, len(sys.argv)):
-    # 输入账户
-    if (i % 2 == 1):
-        userName.append(sys.argv[i])
-    # 输入密码
-    if (i % 2 == 0):
-        password.append(sys.argv[i])

-def sync(i):
-    global thread
-    # 拉取代码进行同步
-    if (not os.path.exists(f"{programPath}/git-clone")):
-        os.makedirs(f"{programPath}/git-clone")
-    print(f"======================> {pullUrl}/{i}")
-    os.system(f"cd '{programPath}/git-clone' ; git clone --bare '{pullUrl}/{i}'")
-    # 推送
-    for j in range(len(pushUrl)):
-        # git remote set-url origin 
-        gitUrl = pushUrl[j].replace("@USER@", userName[j]).replace("@PASS@", password[j]) + "/" + i
-        os.system(f"cd '{programPath}/git-clone/{i}.git' ; git remote set-url origin '{gitUrl}'")
-        os.system(f"cd '{programPath}/git-clone/{i}.git' ; git push --mirror")
-    # 移除临时文件
-    os.system(f"rm -rf '{programPath}/git-clone/{i}.git'")
-    thread -= 1
+def apiGet(path, params):
+    """调用 Gitee API v5 的单个请求"""
+    params["per_page"] = str(perPage)
+    url = "https://gitee.com/api/v5" + path + "?" + urllib.parse.urlencode(params)
+    req = urllib.request.Request(url, headers={"User-Agent": "GXDE-Sync/1.0"})
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        return json.loads(resp.read().decode("utf-8"))

-data = requests.get("https://api.github.com/users/GXDE-OS/repos", headers=dataHeaders).json()
-repoList = []
-# 获取仓库列表
-for i in data:
-    repoList.append(i["name"])
-# 多线程处理以提升速度
-thread = 0
-threadMax = 32
-for i in repoList:
+
+def getGiteeRepos(token):
+    """分页获取 Gitee 组织下全部仓库名"""
+    repoList = []
+    page = 1
    while True:
-        time.sleep(0.1)
-        if (thread < threadMax):
-            threading.Thread(target=sync, args=[i]).start()
-            thread += 1
+        params = {"page": str(page)}
+        if token:
+            params["access_token"] = token
+        data = apiGet(f"/orgs/{orgName}/repos", params)
+        if not isinstance(data, list) or len(data) == 0:
            break
+        repoList.extend([i["name"] for i in data])
+        page += 1
+    return repoList

+
+def runGit(args, cwd=None):
+    return subprocess.run(args, cwd=cwd, stdout=subprocess.PIPE,
+                          stderr=subprocess.STDOUT, text=True)
+
+
+def syncRepo(name, pullUrl, pushUrl):
+    repoDir = os.path.join(cloneDir, name + ".git")
+    try:
+        # 1. 从 Gitee clone --bare（重试 3 次）
+        ok = False
+        lastOut = ""
+        for attempt in range(3):
+            result = runGit(["git", "clone", "--bare", pullUrl + name, repoDir])
+            if result.returncode == 0:
+                ok = True
+                break
+            lastOut = result.stdout
+            time.sleep(5)
+        if not ok:
+            print(f"[FAIL] clone 失败: {name}\n{lastOut}")
+            return False
+
+        # 2. push --mirror 到 GitHub（重试 3 次）
+        ok = False
+        lastOut = ""
+        for attempt in range(3):
+            result = runGit(["git", "-C", repoDir, "push", "--mirror", pushUrl + name])
+            if result.returncode == 0:
+                ok = True
+                break
+            lastOut = result.stdout
+            time.sleep(5)
+        if not ok:
+            print(f"[FAIL] push 失败: {name}\n{lastOut}")
+            return False
+
+        print(f"[ OK ] 同步完成: {name}")
+        return True
+    finally:
+        # 清理临时目录
+        runGit(["rm", "-rf", repoDir])
+
+
+def main():
+    if len(sys.argv) < 5:
+        usage()
+    giteeUser, giteeToken = sys.argv[1], sys.argv[2]
+    githubUser, githubToken = sys.argv[3], sys.argv[4]
+
+    pullUrl = f"https://{urllib.parse.quote(giteeUser)}:{urllib.parse.quote(giteeToken)}@gitee.com/{orgName}/"
+    pushUrl = f"https://{urllib.parse.quote(githubUser)}:{urllib.parse.quote(githubToken)}@github.com/{orgName}/"
+
+    os.makedirs(cloneDir, exist_ok=True)
+
+    # 获取仓库列表：优先带 token，失败则回退到匿名（仅公开仓库）
+    try:
+        repoList = getGiteeRepos(giteeToken)
+    except Exception:
+        repoList = getGiteeRepos("")
+    print(f"共获取到 {len(repoList)} 个仓库")
+    if not repoList:
+        print("未能获取仓库列表，退出")
+        sys.exit(1)
+
+    failed = []
+    with ThreadPoolExecutor(max_workers=threadMax) as pool:
+        futures = {pool.submit(syncRepo, n, pullUrl, pushUrl): n for n in repoList}
+        for future in as_completed(futures):
+            if not future.result():
+                failed.append(futures[future])
+
+    print("=" * 50)
+    print(f"同步完成: 成功 {len(repoList) - len(failed)} / {len(repoList)}")
+    if failed:
+        print("以下仓库同步失败:")
+        for name in failed:
+            print("  -", name)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()