diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 09c3c43..da3d4c2 100755 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -16,19 +16,15 @@ jobs: - name: Config run: | sudo apt update - sudo apt install git python3-requests -y + sudo apt install git -y - name: Push env: GITHUBTOKEN: ${{ secrets.GITHUBTOKEN }} GITHUBUSER: ${{ secrets.GITHUBUSER }} - GITCODETOKEN: ${{ secrets.GITCODETOKEN }} - GITCODEUSER: ${{ secrets.GITCODEUSER }} - ATOMGITTOKEN: ${{ secrets.ATOMGITTOKEN }} - ATOMGITUSER: ${{ secrets.ATOMGITUSER }} - ATOMGITEEUSER: ${{ secrets.GITEEUSER }} - ATOMGITEETOCKEN: ${{ secrets.GITEETOCKEN }} + GITEEUSER: ${{ secrets.GITEEUSER }} + GITEETOCKEN: ${{ secrets.GITEETOCKEN }} run: | cd sync-gitee-to-github - python3 sync.py $ATOMGITEEUSER $ATOMGITEETOCKEN $GITHUBUSER $GITHUBTOKEN $GITCODEUSER $GITCODETOKEN $ATOMGITUSER $ATOMGITTOKEN + python3 sync.py $GITEEUSER $GITEETOCKEN $GITHUBUSER $GITHUBTOKEN diff --git a/sync-gitee-to-github/sync.py b/sync-gitee-to-github/sync.py index 2d29257..73018b5 100644 --- a/sync-gitee-to-github/sync.py +++ b/sync-gitee-to-github/sync.py @@ -1,74 +1,141 @@ -#!/bin/bash +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +将 Gitee 上的 GXDE-OS 组织仓库镜像同步到 GitHub(GXDE-OS 组织)。 + +用法: + python3 sync.py + +流程: + 1. 通过 Gitee API v5 分页获取 GXDE-OS 组织下的全部仓库列表(带 token 失败时回退匿名) + 2. 多线程 clone --bare 拉取 Gitee 仓库 + 3. 逐个 push --mirror 推送到 GitHub,失败自动重试并在结束时汇总报告 +""" +import json import os +import subprocess import sys import time -import requests -import threading +import urllib.parse +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed -programPath = os.path.split(os.path.realpath(__file__))[0] # 返回 string -dataHeaders = { - "accept": "application/json" -} -giteeUser = "" -giteePassword = "" -if len(sys.argv) > 3: - giteeUser = sys.argv[1] - giteePassword = sys.argv[2] -pullUrl = f"https://{giteeUser}:{giteePassword}@gitee.com/GXDE-OS/" -pushUrl = [ - "https://@USER@:@PASS@@github.com/GXDE-OS/", - "https://@USER@:@PASS@@gitcode.com/GXDE-OS/", - "https://@USER@:@PASS@@atomgit.com/GXDE-OS/" -] +programPath = os.path.split(os.path.realpath(__file__))[0] +cloneDir = os.path.join(programPath, "git-clone") +orgName = "GXDE-OS" +perPage = 100 +threadMax = 8 -if (len(sys.argv) < 1 + len(pushUrl) * 2): - print(f"请输入用于推送的账户和密码(需要 {len(pushUrl)} 组以推送不同的平台,提供顺序如下)") - for i in pushUrl: - print(f" {i}") - exit(1) -# 读取参数 -userName = [] -password = [] +def usage(): + print("用法: python3 sync.py ") + sys.exit(1) -for i in range(1 + 2, len(sys.argv)): - # 输入账户 - if (i % 2 == 1): - userName.append(sys.argv[i]) - # 输入密码 - if (i % 2 == 0): - password.append(sys.argv[i]) -def sync(i): - global thread - # 拉取代码进行同步 - if (not os.path.exists(f"{programPath}/git-clone")): - os.makedirs(f"{programPath}/git-clone") - print(f"======================> {pullUrl}/{i}") - os.system(f"cd '{programPath}/git-clone' ; git clone --bare '{pullUrl}/{i}'") - # 推送 - for j in range(len(pushUrl)): - # git remote set-url origin - gitUrl = pushUrl[j].replace("@USER@", userName[j]).replace("@PASS@", password[j]) + "/" + i - os.system(f"cd '{programPath}/git-clone/{i}.git' ; git remote set-url origin '{gitUrl}'") - os.system(f"cd '{programPath}/git-clone/{i}.git' ; git push --mirror") - # 移除临时文件 - os.system(f"rm -rf '{programPath}/git-clone/{i}.git'") - thread -= 1 +def apiGet(path, params): + """调用 Gitee API v5 的单个请求""" + params["per_page"] = str(perPage) + url = "https://gitee.com/api/v5" + path + "?" + urllib.parse.urlencode(params) + req = urllib.request.Request(url, headers={"User-Agent": "GXDE-Sync/1.0"}) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) -data = requests.get("https://api.github.com/users/GXDE-OS/repos", headers=dataHeaders).json() -repoList = [] -# 获取仓库列表 -for i in data: - repoList.append(i["name"]) -# 多线程处理以提升速度 -thread = 0 -threadMax = 32 -for i in repoList: + +def getGiteeRepos(token): + """分页获取 Gitee 组织下全部仓库名""" + repoList = [] + page = 1 while True: - time.sleep(0.1) - if (thread < threadMax): - threading.Thread(target=sync, args=[i]).start() - thread += 1 + params = {"page": str(page)} + if token: + params["access_token"] = token + data = apiGet(f"/orgs/{orgName}/repos", params) + if not isinstance(data, list) or len(data) == 0: break + repoList.extend([i["name"] for i in data]) + page += 1 + return repoList + +def runGit(args, cwd=None): + return subprocess.run(args, cwd=cwd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + + +def syncRepo(name, pullUrl, pushUrl): + repoDir = os.path.join(cloneDir, name + ".git") + try: + # 1. 从 Gitee clone --bare(重试 3 次) + ok = False + lastOut = "" + for attempt in range(3): + result = runGit(["git", "clone", "--bare", pullUrl + name, repoDir]) + if result.returncode == 0: + ok = True + break + lastOut = result.stdout + time.sleep(5) + if not ok: + print(f"[FAIL] clone 失败: {name}\n{lastOut}") + return False + + # 2. push --mirror 到 GitHub(重试 3 次) + ok = False + lastOut = "" + for attempt in range(3): + result = runGit(["git", "-C", repoDir, "push", "--mirror", pushUrl + name]) + if result.returncode == 0: + ok = True + break + lastOut = result.stdout + time.sleep(5) + if not ok: + print(f"[FAIL] push 失败: {name}\n{lastOut}") + return False + + print(f"[ OK ] 同步完成: {name}") + return True + finally: + # 清理临时目录 + runGit(["rm", "-rf", repoDir]) + + +def main(): + if len(sys.argv) < 5: + usage() + giteeUser, giteeToken = sys.argv[1], sys.argv[2] + githubUser, githubToken = sys.argv[3], sys.argv[4] + + pullUrl = f"https://{urllib.parse.quote(giteeUser)}:{urllib.parse.quote(giteeToken)}@gitee.com/{orgName}/" + pushUrl = f"https://{urllib.parse.quote(githubUser)}:{urllib.parse.quote(githubToken)}@github.com/{orgName}/" + + os.makedirs(cloneDir, exist_ok=True) + + # 获取仓库列表:优先带 token,失败则回退到匿名(仅公开仓库) + try: + repoList = getGiteeRepos(giteeToken) + except Exception: + repoList = getGiteeRepos("") + print(f"共获取到 {len(repoList)} 个仓库") + if not repoList: + print("未能获取仓库列表,退出") + sys.exit(1) + + failed = [] + with ThreadPoolExecutor(max_workers=threadMax) as pool: + futures = {pool.submit(syncRepo, n, pullUrl, pushUrl): n for n in repoList} + for future in as_completed(futures): + if not future.result(): + failed.append(futures[future]) + + print("=" * 50) + print(f"同步完成: 成功 {len(repoList) - len(failed)} / {len(repoList)}") + if failed: + print("以下仓库同步失败:") + for name in failed: + print(" -", name) + sys.exit(1) + + +if __name__ == "__main__": + main()