spark-store-project/spark-store
15
0
Fork 0
mirror of https://gitee.com/spark-store-project/spark-store synced 2025-09-05 10:52:22 +08:00
Code Issues 1 Packages Projects Releases Wiki Activity

!52 使用aria2替换原本的多线程下载

Browse Source 
* update debian/control.
* fix: readme
* change: 切换到 aria2
* Download: 初步完成对 axel 的适配工作
* README: 英文修正 ubuntu22 的依赖问题
* Merge remote-tracking branch 'upstream/master'
* widget: 检查cdn状况在下载开始前检测,不堵塞ui线程
* Merge remote-tracking branch 'upstream/master'
* readme: 修正 ubuntu 的编译说明
* download: 检查软件源的有效性
This commit is contained in:
Pluto 2022-06-30 15:26:13 +00:00 committed by shenmo
parent 766dc8b88a
commit 5f9599c47d
11 changed files with 176 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -51,3 +51,4 @@ debian/*.debhelper*
debian/files
debian/*.substvars
debian/spark-store

9
README.md
View File

@ -28,12 +28,19 @@ If you are using Debian11/Ubuntu 20.04, you will need extra dependency package.
#### Compile and developement
For Deepin V20/UOS 21/ Ubuntu 22.04/ Debian 11
For Deepin V20/UOS 21/ Debian 11
```shell
sudo apt install git qt5-default debhelper pkg-config qtchooser libqt5core5a libqt5gui5 libqt5widgets5 libqt5network5 libqt5concurrent5 libdtkcore-dev libdtkgui-dev libdtkwidget-dev qttools5-private-dev libnotify-dev qtwebengine5-dev
```
Ubuntu 22.04
```shell
sudo apt install git qtbase5-dev debhelper pkg-config qtchooser libqt5core5a libqt5gui5 libqt5widgets5 libqt5network5 libqt5concurrent5 libdtkcore-dev libdtkgui-dev libdtkwidget-dev qttools5-private-dev libnotify-dev qtwebengine5-dev
```
Then
```shell

21
debian/changelog vendored
View File

@ -1,18 +1,13 @@
spark-store (3.2) stable; urgency=medium
新增：ssinstall现在会在没有apt-fast的时候自动安装
新增：ss-apt-fast现在会在没有apt-fast的时候自动安装
修改：删除ssinstall中无用的 || dpkg -P $1
新增：ss-apt-fast会先下载云上的conf以确保mirror是最新的
修复：去除wget指令
-- shenmo <shenmo@spark-app.store> Mon, 17 Jan 2022 00:00:00 +0800
spark-store (3.1.3) stable; urgency=medium
* Now uses aria2 to download softwares form all mirrors
* 新增：ssinstall现在会在没有apt-fast的时候自动安装
* 新增：ss-apt-fast现在会在没有apt-fast的时候自动安装
* 修改：删除ssinstall中无用的 || dpkg -P $1
* 新增：ss-apt-fast会先下载云上的conf以确保mirror是最新的
* 修复：去除wget指令
-- shenmo <shenmo@spark-app.store> Fri, 30 Jan 2022 00:00:00 +0800
spark-store (3.1.2) stable; urgency=medium

3
debian/control vendored
View File

@ -35,7 +35,8 @@ Depends:${shlibs:Depends}, ${misc:Depends},
libnotify4,
curl,
dde-qt5integration,
bubblewrap
bubblewrap,
aria2
Description: Spark Store
A community powered app store, based on DTK.
Recommends: apt-fast

4
debian/rules vendored
View File

@ -17,11 +17,12 @@ override_dh_auto_clean:
override_dh_auto_configure:
mkdir -p $(CURDIR)/build
dh_auto_configure MAKEFLAGS=-j$(JOBS) -- spark-store-project.pro \
-spec linux-g++ CONFIG+=qtquickcompiler \
-o $(CURDIR)/build/
override_dh_auto_build:
make -C $(CURDIR)/build -j$(JOBS)
@ -29,6 +30,7 @@ override_dh_auto_install:
make -C $(CURDIR)/build install \
INSTALL_ROOT=$(CURDIR)/debian/spark-store
# Ignore the dpkg-shlibdeps: warning (it uses none of the library's symbols)
# Qt Mutidedia lib will ref to network libraray.
override_dh_shlibdeps:

13
src/downloadlist.cpp
View File

@ -92,19 +92,6 @@ QString downloadlist::getName()
void downloadlist::readyInstall()
{
if(ui->progressBar->value() != ui->progressBar->maximum() && !close)
{
ui->progressBar->hide();
ui->pushButton_install->show();
ui->pushButton_2->hide();
Widget::sendNotification(tr("Failed to download %1").arg(ui->label->text()), 5000,
"/tmp/spark-store/icon_" + QString::number(num).toUtf8() + ".png");
ui->label_2->setText(tr("Download Failed，Check Your Connection"));
ui->pushButton_install->setEnabled(false);
return;
}
if(!close)
{
ui->progressBar->hide();

331
src/downloadworker.cpp
View File

@ -1,93 +1,15 @@
#include "downloadworker.h"
#include <QEventLoop>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QThread>
#include <QProcess>
#include <QRegularExpression>
#include <QFileInfo>
#include <QDir>
DownloadWorker::DownloadWorker(QObject *parent)
{
Q_UNUSED(parent)
}
void DownloadWorker::setIdentifier(int identifier)
{
this->identifier = identifier;
}
void DownloadWorker::setParamter(const QString &url, QPair<qint64, qint64> range, QFile *file)
{
this->url = url;
this->startPos = range.first;
this->endPos = range.second;
this->file = file;
}
qint64 DownloadWorker::getReceivedPos()
{
return receivedPos;
}
void DownloadWorker::doWork()
{
mgr = new QNetworkAccessManager(this);
QNetworkRequest request;
request.setUrl(url);
request.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
request.setRawHeader("Range", QString("bytes=%1-%2").arg(startPos).arg(endPos).toLocal8Bit());
reply = mgr->get(request);
qDebug() << "开始下载数据：" << QString(" %1~%2 -> writePos Start %3").arg(startPos).arg(endPos).arg(receivedPos);
connect(reply, static_cast<void(QNetworkReply::*)(QNetworkReply::NetworkError) > (&QNetworkReply::error),
[this](QNetworkReply::NetworkError error)
{
if(error != QNetworkReply::NoError)
{
qDebug() << "出错了：" << reply->errorString();
}
});
connect(reply, &QNetworkReply::finished, mgr, &QNetworkAccessManager::deleteLater);
connect(reply, &QNetworkReply::readyRead, this, &DownloadWorker::dataReady);
connect(reply, &QNetworkReply::finished, this, &DownloadWorker::slotFinish);
connect(reply, &QNetworkReply::downloadProgress, this, &DownloadWorker::handleProcess);
// connect(reply, &QNetworkReply::finished, reply, &QNetworkReply::deleteLater);
connect(reply, &QNetworkReply::finished, this, &DownloadWorker::doStop);
}
void DownloadWorker::doStop()
{
if (reply) {
reply->disconnect();
reply->aboutToClose();
reply->deleteLater();
reply = nullptr;
}
}
void DownloadWorker::dataReady()
{
QByteArray data = reply->readAll();
file->seek(startPos + receivedPos);
file->write(data);
receivedPos += data.size();
}
void DownloadWorker::slotFinish()
{
file->flush();
qDebug() << "数据块下载完毕：" << QString(" %1~%2 -> writePos Start %3").arg(startPos).arg(endPos).arg(receivedPos);
emit workFinished();
}
void DownloadWorker::handleProcess(qint64, qint64)
{
emit this->downloadProcess();
}
#include <QElapsedTimer>
#include <QtConcurrent>
DownloadController::DownloadController(QObject *parent)
{
@ -97,28 +19,7 @@ DownloadController::DownloadController(QObject *parent)
domains.clear();
domains.append("d.store.deepinos.org.cn");
QFile serverList(QDir::homePath().toUtf8() + "/.config/spark-store/server.list");
if(serverList.open(QFile::ReadOnly))
{
QStringList list = QString(serverList.readAll()).trimmed().split("\n");
qDebug() << list << list.size();
for (int i = 0; i < list.size(); i++) {
if (list.at(i).contains("镜像源 Download only") && i + 1 < list.size()) {
for (int j = i + 1; j < list.size(); j++) {
system("curl -I -s --connect-timeout 5 " + list.at(j).toUtf8()
+ "/dcs-repo.gpg-key.asc -w %{http_code} |tail -n1 > /tmp/spark-store/cdnStatus.txt");
QFile cdnStatus("/tmp/spark-store/cdnStatus.txt");
if(cdnStatus.open(QFile::ReadOnly) && QString(cdnStatus.readAll()).toUtf8()=="200"){
qDebug() << list.at(j);
domains.append(list.at(j));
}
}
break;
}
}
}
qDebug() << domains << domains.size();
/*
domains = {
@ -132,38 +33,41 @@ DownloadController::DownloadController(QObject *parent)
this->threadNum = domains.size();
}
DownloadController::~DownloadController()
{
if(workers.size() > 0)
{
for(int i = 0; i < workers.size(); i++)
{
workers.at(i)->doStop();
workers.at(i)->disconnect();
workers.at(i)->deleteLater();
}
workers.clear();
}
}
void DownloadController::setFilename(QString filename)
{
this->filename = filename;
}
void DownloadController::setThreadNum(int threadNum)
void timeSleeper(int time)
{
this->threadNum = threadNum;
QElapsedTimer t1;
t1.start();
while(t1.elapsed()<time);
return;
}
//int checkPID(QString pidCommand){
// system(pidCommand.toUtf8());
// timeSleeper(10);
// QFile downloadStatus("/tmp/spark-store/downloadStatus.txt");
// downloadStatus.open(QFile::ReadOnly);
// auto temp = QString(downloadStatus.readAll()).toUtf8();
// downloadStatus.close();
// if (temp!=""){
// return 1;
// }
// return 0;
//}
/**
* @brief 开始下载
*/
void DownloadController::startDownload(const QString &url)
{
finish = 0;
// 下载任务等分，计算每个线程的下载数据
// 获取下载任务信息
fileSize = getFileSize(url);
if(fileSize == 0)
{
@ -171,55 +75,120 @@ void DownloadController::startDownload(const QString &url)
return;
}
qint64 segmentSize = fileSize / threadNum;
ranges.resize(threadNum);
QVector<qint64> receivedBytes;
receivedBytes.resize(threadNum);
for(int i = 0; i < threadNum; i++)
{
ranges[i].first = i * segmentSize;
ranges[i].second = i * segmentSize + segmentSize - 1;
receivedBytes[i] = 0;
}
ranges[threadNum - 1].second = fileSize; // 余数部分加入最后一个
// 打开文件
QDir tmpdir("/tmp/spark-store");
file = new QFile;
file->setFileName(tmpdir.absoluteFilePath(filename));
QtConcurrent::run([=](){
QFile serverList(QDir::homePath().toUtf8() + "/.config/spark-store/server.list");
if(serverList.open(QFile::ReadOnly))
{
QStringList list = QString(serverList.readAll()).trimmed().split("\n");
qDebug() << list << list.size();
if(file->exists())
{
file->remove();
}
for (int i = 0; i < list.size(); i++) {
if (list.at(i).contains("镜像源 Download only") && i + 1 < list.size()) {
for (int j = i + 1; j < list.size(); j++) {
system("curl -I -s --connect-timeout 5 https://" + list.at(j).toUtf8()
+ "/dcs-repo.gpg-key.asc -w %{http_code} |tail -n1 > /tmp/spark-store/cdnStatus.txt");
QFile cdnStatus("/tmp/spark-store/cdnStatus.txt");
if(cdnStatus.open(QFile::ReadOnly) && QString(cdnStatus.readAll()).toUtf8()=="200"){
qDebug() << list.at(j);
domains.append(list.at(j));
}
}
break;
}
}
}
qDebug() << domains << domains.size();
QDir tmpdir("/tmp/spark-store/");
QString aria2Command = "-d";
QString aria2Urls = "";
QString aria2Verbose = "--summary-interval=1";
QString aria2Threads = "-s " + QString::number(domains.size());
QStringList command;
QString downloadDir = "/tmp/spark-store/";
for(int i = 0; i < domains.size(); i ++)
{
command.append(replaceDomain(url, domains.at(i)).toUtf8());
aria2Urls += replaceDomain(url, domains.at(i));
aria2Urls += " ";
}
if(!file->open(QIODevice::WriteOnly))
{
delete file;
file = nullptr;
emit errorOccur(file->errorString());
return;
}
qint64 downloadSizeRecord = 0;
QString speedInfo = "";
QString percentInfo = "";
command.append(aria2Command.toUtf8());
command.append(downloadDir.toUtf8());
command.append(aria2Verbose.toUtf8());
command.append(aria2Threads.toUtf8());
qDebug() << command;
auto cmd = new QProcess();
cmd->setProcessChannelMode(QProcess::MergedChannels);
cmd->setProgram("aria2c");
cmd->setArguments(command);
cmd->start();
cmd->waitForStarted(); //等待启动完成
file->resize(fileSize);
QObject::connect(cmd,&QProcess::readyReadStandardOutput,
[&](){
//通过读取输出计算下载速度
QFileInfo info(tmpdir.absoluteFilePath(filename));
QString message = cmd->readAllStandardOutput().data();
message = message.replace(" ","").replace("\n","").replace("-","");
message = message.replace("*","").replace("=","");
QStringList list;
qint64 downloadSize = 0;
int downloadSizePlace1 = message.indexOf("(");
int downloadSizePlace2 = message.indexOf(")");
int speedPlace1 = message.indexOf("DL:");
int speedPlace2 = message.indexOf("ETA");
if (downloadSizePlace1 != -1 && downloadSizePlace2 != -1){
percentInfo = message.mid(downloadSizePlace1+1, downloadSizePlace2-downloadSizePlace1-1).replace("%","");
if (percentInfo != "s"){
int percentInfoNumber = percentInfo.toUInt();
downloadSize = (percentInfoNumber+1) * fileSize / 100;
}
}
if (speedPlace1 != -1 && speedPlace2 != -1){
speedInfo = message.mid(speedPlace1+3, speedPlace2-speedPlace1-3);
speedInfo += "/s";
// 创建下载线程
workers.clear();
for(int i = 0; i < ranges.size(); i++)
{
qDebug() << QString("第%1个下载请求：%2-%3").arg(i).arg(ranges.at(i).first).arg(ranges.at(i).second);
auto worker = new DownloadWorker(this);
auto range = ranges.at(i);
QString chunkUrl = replaceDomain(url, domains.at(i));
worker->setIdentifier(i);
worker->setParamter(chunkUrl, range, file);
workers.append(worker);
}
qDebug() << percentInfo << speedInfo;
if(downloadSize >= downloadSizeRecord)
{
downloadSizeRecord = downloadSize;
}
if(percentInfo=="OK"){
finished = true;
emit downloadProcess("", fileSize, fileSize);
qDebug() <<"finished:"<< finished;
}
else{
emit downloadProcess(speedInfo, downloadSizeRecord, fileSize);
}
connect(worker, &DownloadWorker::downloadProcess, this, &DownloadController::handleProcess);
connect(worker, &DownloadWorker::workFinished, this, &DownloadController::chunkDownloadFinish);
worker->doWork();
}
});
QObject::connect(cmd,&QProcess::readyReadStandardError,
[&](){
emit errorOccur(cmd->readAllStandardError().data());
return;
});
auto pidNumber = cmd->processId();
this->pidNumber = pidNumber;
int statusSum = 0;
while(statusSum > -20)
{
auto status = cmd->waitForFinished() - 1;
statusSum += status;
}
emit downloadFinished();
});
}
/**
@ -227,44 +196,14 @@ void DownloadController::startDownload(const QString &url)
*/
void DownloadController::stopDownload()
{
for(int i = 0; i < workers.size(); i++)
{
workers.at(i)->doStop();
workers.at(i)->disconnect();
workers.at(i)->deleteLater();
}
workers.clear();
qDebug() << "文件下载路径：" << QFileInfo(file->fileName()).absoluteFilePath();
// 实现下载进程退出
QString killCmd = QString("kill -9 %1").arg(pidNumber);
system(killCmd.toUtf8());
qDebug()<<"kill aria2!";
file->flush();
file->close();
delete file;
file = nullptr;
}
void DownloadController::handleProcess()
{
qint64 bytesReceived = 0;
for(int i = 0; i < workers.size(); i++)
{
bytesReceived += workers.at(i)->getReceivedPos();
}
qDebug() << QString("下载进度 %1-%2").arg(bytesReceived).arg(fileSize);
emit downloadProcess(bytesReceived, fileSize);
}
void DownloadController::chunkDownloadFinish()
{
finish++;
qDebug() << QString("已下载了%1块，共%2块！！！").arg(finish).arg(threadNum);
if(finish == threadNum)
{
stopDownload();
emit downloadFinished();
}
}
qint64 DownloadController::getFileSize(const QString& url)
{
QEventLoop event;

48
src/downloadworker.h
View File

@ -6,52 +6,14 @@
#include <QFile>
#include <QNetworkReply>
class DownloadWorker : public QObject
{
Q_OBJECT
public:
explicit DownloadWorker(QObject *parent = nullptr);
void setIdentifier(int identifier);
void setParamter(const QString &url, QPair<qint64, qint64> range, QFile *flle);
qint64 getReceivedPos();
public slots:
void doWork();
void doStop();
void dataReady();
void slotFinish();
void handleProcess(qint64, qint64);
private:
int identifier;
QString url;
qint64 startPos;
qint64 endPos;
qint64 receivedPos = 0;
QNetworkReply *reply;
QNetworkAccessManager *mgr;
QFile *file;
signals:
void resultReady(int identifier, QByteArray data);
void testSignals();
void workFinished();
void downloadProcess();
};
class DownloadController : public QObject
{
Q_OBJECT
public:
explicit DownloadController(QObject *parent = nullptr);
~DownloadController();
void setFilename(QString filename);
void setThreadNum(int threadNum);
void startDownload(const QString &url);
void stopDownload();
qint64 getFileSize(const QString& url);
@ -59,21 +21,19 @@ public:
private:
int threadNum;
int pidNumber;
QString filename;
qint64 fileSize;
QVector<QPair<qint64, qint64>> ranges;
QFile *file;
QList<DownloadWorker*> workers;
int finish = 0;
bool finished = false;
QVector<QString> domains;
public slots:
void handleProcess();
void chunkDownloadFinish();
signals:
void errorOccur(const QString& msg);
void downloadProcess(qint64, qint64);
void downloadProcess(QString, qint64, qint64);
void downloadFinished();
};

2
src/main.cpp
View File

@ -30,7 +30,7 @@ int main(int argc, char *argv[])
DAboutDialog dialog;
a.setAboutDialog(&dialog);
dialog.setLicense(QObject::tr("We publish this program under GPL V3"));
dialog.setVersion(DApplication::buildVersion("Version 3.2"));
dialog.setVersion(DApplication::buildVersion("Version 3.1.3"));
dialog.setProductIcon(QIcon::fromTheme("spark-store")); // 设置Logo
dialog.setProductName(QLabel::tr("Spark Store"));
dialog.setDescription(

21
src/widget.cpp
View File

@ -108,10 +108,9 @@ Widget::Widget(DBlurEffectWidget *parent) :
download_speed.start();
connect(&download_speed,&QTimer::timeout,[=]()
{
if(isdownload)
if(isdownload && theSpeed == "")
{
size1 = download_size;
QString theSpeed;
double bspeed;
bspeed = size1 - size2;
if(bspeed < 1024)
@ -129,10 +128,12 @@ Widget::Widget(DBlurEffectWidget *parent) :
else
{
theSpeed = QString::number(0.01 * int(100 * (bspeed / (1024 * 1024 * 1024)))) + "GB/s";
}
download_list[nowDownload - 1].setSpeed(theSpeed);
}
size2 = download_size;
}
if(isdownload){
download_list[nowDownload - 1].setSpeed(theSpeed);
}
});
notify_init(tr("Spark\\ Store").toLocal8Bit());
@ -788,15 +789,15 @@ void Widget::httpReadyRead()
}
}
void Widget::updateDataReadProgress(qint64 bytesRead, qint64 totalBytes)
void Widget::updateDataReadProgress(QString speedInfo, qint64 bytesRead, qint64 totalBytes)
{
if(totalBytes <= 0)
{
return;
}
theSpeed = speedInfo;
download_list[nowDownload - 1].setMax(10000); // 最大值
download_list[nowDownload - 1].setValue((bytesRead * 10000) / totalBytes); // 当前值
download_list[nowDownload - 1].setValue(int(bytesRead * 100 / totalBytes) * 100); // 当前值
download_size = bytesRead;
if(download_list[nowDownload - 1].close)
{
@ -955,7 +956,9 @@ void Widget::httpFinished() // 完成下载
download_list[nowDownload - 1].free = true;
if(nowDownload < allDownload)
{
// 如果有排队则下载下一个
qDebug() << "切换下一个下载...";
nowDownload += 1;
while(download_list[nowDownload - 1].close)
{
@ -1039,7 +1042,7 @@ void Widget::on_pushButton_updateServer_clicked()
ui->comboBox_server->clear();
QFile::remove(QDir::homePath().toUtf8() + "/.config/spark-store/server.list");
system("curl -o " + QDir::homePath().toUtf8() + "/.config/spark-store/server.list https://d.store.deepinos.org.cn/store/server.list");
system("curl -o " + QDir::homePath().toUtf8() + "/.config/spark-store/server.list https://d.store.deepinos.org.cn/store/server-and-mirror.list");
std::fstream server;
server.open(QDir::homePath().toUtf8() + "/.config/spark-store/server.list", std::ios::in);
std::string lineTmp;
@ -1325,7 +1328,7 @@ void Widget::on_webEngineView_urlChanged(const QUrl &arg1)
ui->label_appname->clear();
ui->pushButton_download->setEnabled(false);
ui->stackedWidget->setCurrentIndex(2);
qDebug() << "https://demo-one-vert.vercel.app/" + type_name + "/" + pname;
qDebug() << "https://d.store.deepinos.org.cn/" + type_name + "/" + pname;
qDebug() << "链接地址：" << arg1;
/*

3
src/widget.h
View File

@ -69,7 +69,7 @@ private slots:
void httpFinished();
void httpReadyRead();
void updateDataReadProgress(qint64,qint64);
void updateDataReadProgress(QString, qint64, qint64);
// SpkAppInfoLoaderThread的槽函数
void sltAppinfoResetUi();
@ -147,6 +147,7 @@ private:
QFuture<void> load;
QFutureWatcher<void> watchScreenshotLoad;
QTimer download_speed;
QString theSpeed;
QString type_name;
QColor main_color;
int foot;