spark-store-project/spark-qt-research
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新多线程下载代码

Browse Source
This commit is contained in:
metanoia1989 2021-02-16 00:25:54 +08:00
parent 5ad594a028
commit 8379d7922a
9 changed files with 347 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
MultiplethreadDownload/MultiplethreadDownload.pro
View File

@ -9,13 +9,13 @@ CONFIG += c++17
#DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000 # disables all the APIs deprecated before Qt 6.0.0
SOURCES += \
filedownloadworker.cpp \
downloadworker.cpp \
main.cpp \
utils.cpp \
widget.cpp
HEADERS += \
filedownloadworker.h \
downloadworker.h \
utils.h \
widget.h

33
MultiplethreadDownload/REAMDE.md
View File

@ -11,4 +11,35 @@
开始下载数据： " 0~22484391 -> writePos Start 0"
开始下载数据： " 22484392~44968783 -> writePos Start 22484392"
开始下载数据： " 44968784~67453175 -> writePos Start 44968784"
开始下载数据： " 67453176~89937570 -> writePos Start 67453176
开始下载数据： " 67453176~89937570 -> writePos Start 67453176
# 星火商店多线程下载说明
现在的这个多线程下载已经可以了，但是暂停恢复不太好控制，所以还是用 QThread 来做暂停控制，即使线程数超过CPU
核数问题也不大，因为下载并不是CPU密集操作，完全没问题的。总体的精细的线程池控制后续技术强大了再来封装好了。
先做好这个功能，后续再来研究DTK库，各个组件的使用，已经如何自己封装漂亮的控件，摆脱DTK的依赖。
再后面，写一个Qt开发Linux应用的教程，先建立起星火商店单独的开发者博客，专门用来发技术文章。
星火商店的几个源服务器，源列表写在
```sh
$ cat /etc/apt/sources.list.d/sparkstore.list
deb [by-hash=force] https://sucdn.jerrywang.top /
```
服务器源列表 http://dcstore.shenmo.tech/store/server.list
```js
https://sucdn.jerrywang.top/
国内推荐 China Lines
https://sucdn.jerrywang.top/
https://store.deepinos.org.cn/
国外推荐 Global Lines
开发者模式 Dev only
http://localhost:8080/
用户贡献 Community lines
```
也就是目前的就两个服务器了
* https://sucdn.jerrywang.top/
* https://store.deepinos.org.cn/

171
MultiplethreadDownload/downloadworker.cpp Normal file
View File

@ -0,0 +1,171 @@
#include "downloadworker.h"
#include <QIODevice>
#include <QEventLoop>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QDebug>
DownloadWorker::DownloadWorker(QObject *parent)
{
}
void DownloadWorker::setIdentifier(int identifier)
{
this->identifier = identifier;
}
void DownloadWorker::setParamter(const QString &url, QPair<qint64, qint64> range)
{
this->url = url;
this->range = range;
}
void DownloadWorker::doWork()
{
QNetworkAccessManager *mgr = new QNetworkAccessManager(this);
QNetworkRequest request;
request.setUrl(url);
request.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
request.setRawHeader("Range", QString("bytes=%1-%2").arg(range.first)
.arg(range.second).toLocal8Bit());
// QNetworkReply *reply = mgr->get(request);
// QNetworkReply *reply = mgr->get(request);
reply = mgr->get(request);
qint64 writePos = range.first;
qDebug() << "开始下载数据：" << QString(" %1~%2 -> writePos Start %3")
.arg(range.first).arg(range.second).arg(writePos);
// connect(reply, &QNetworkReply::readyRead, [reply, this](){
//// qDebug() << "测试是否触发，午安啦啦啦";
// QByteArray data = reply->readAll();
// qDebug() << "获取到了数据" << " " << data.size();
// emit resultReady(identifier, data);
// });
// connect(reply, &QNetworkReply::errorOccurred, [reply](QNetworkReply::NetworkError error){
// if (error != QNetworkReply::NoError) {
// qDebug() << "出错了：" << reply->errorString();
// }
// });
connect(reply, &QNetworkReply::finished, mgr, &QNetworkAccessManager::deleteLater);
connect(reply, &QNetworkReply::finished, reply, &QNetworkReply::deleteLater);
connect(reply, &QNetworkReply::readyRead, this, &DownloadWorker::dataReady);
}
void DownloadWorker::dataReady()
{
QByteArray data = reply->readAll();
qDebug() << "获取到了数据" << " " << data.size();
emit resultReady(identifier, data);
auto controller = qobject_cast<DownloadController*>(parent());
QMetaObject::invokeMethod(controller, "handleResults", Qt::DirectConnection,
Q_ARG(int, identifier), Q_ARG(QByteArray, data));
// QMetaObject::invokeMethod(controller, "handleResults", Qt::QueuedConnection,
// Q_ARG(int, identifier), Q_ARG(QByteArray, data));
}
DownloadController::DownloadController(QObject *parent)
{
this->threadNum = 4;
// 注册信号槽传递的数据类型
qRegisterMetaType<QPair<qint64, qint64>>("QPair<qint64, qint64>");
}
DownloadController::~DownloadController()
{
}
void DownloadController::setFilename(QString filename)
{
this->filename = filename;
}
void DownloadController::setThreadNum(int threadNum)
{
this->threadNum = threadNum;
}
/**
* @brief 开始下载
*/
void DownloadController::startDownload(const QString &url, qint64 fileSize)
{
// 下载任务等分，计算每个线程的下载数据
qint64 segmentSize = fileSize / threadNum;
ranges.resize(threadNum);
// QVector<qint64> writePosList;
writePosList.resize(threadNum);
for (int i = 0; i < threadNum; i++) {
ranges[i].first = i * segmentSize;
ranges[i].second = i * segmentSize + segmentSize - 1;
writePosList[i] = ranges[i].first;
}
ranges[threadNum-1].second = fileSize; // 余数部分加入最后一个
qDebug() << QString("查看分段下载：%1 %2").arg(ranges.size()).arg(writePosList.size());
// return;
// 打开文件
// QFile file;
file.setFileName(filename);
if (file.exists())
file.remove();
if (!file.open(QIODevice::WriteOnly)) {
emit errorOccur(file.errorString());
return;
}
file.resize(fileSize);
// 创建下载线程
// qint64 bytesReceived = 0;
for(int i = 0; i < ranges.size(); i++) {
qDebug() << QString("第%1个线程：%2-%3").arg(i).arg(ranges.at(i).first).arg(ranges.at(i).second);
auto worker = new DownloadWorker(this);
auto range = ranges.at(i);
worker->setIdentifier(i);
worker->setParamter(url, range);
connect(worker, &DownloadWorker::resultReady, this, &DownloadController::handleResults);
worker->doWork();
}
}
/**
* @brief 暂停下载
*/
void DownloadController::paruseDownload()
{
}
/**
* @brief 停止下载
*/
void DownloadController::stopDownload()
{
}
/**
* @brief 写入响应数据
* @param identifier 线程编号
* @param data 请求数据
*/
void DownloadController::handleResults(int identifier, QByteArray data)
{
qDebug() << QString("测试哈哈哈: %1 --- %2").arg(writePosList.size()).arg(identifier);
QMutexLocker lock(&mutex);
qint64 writePos = writePosList.at(identifier);
file.seek(writePos);
file.write(data);
qDebug() << QString("%1, %2, %3").arg(writePos).arg(bytesReceived).arg(data.size());
writePos += data.size();
writePosList.replace(identifier, writePos);
bytesReceived += data.size();
qDebug() << "已经下载了数据: " << bytesReceived;
emit receivedProcess(bytesReceived);
}

65
MultiplethreadDownload/downloadworker.h Normal file
View File

@ -0,0 +1,65 @@
#ifndef DOWNLOADWORKER_H
#define DOWNLOADWORKER_H
#include <QObject>
#include <QList>
#include <QFile>
#include <QNetworkReply>
#include <QMutex>
class DownloadWorker : public QObject
{
Q_OBJECT
public:
explicit DownloadWorker(QObject *parent = nullptr);
void setIdentifier(int identifier);
void setParamter(const QString &url, QPair<qint64, qint64> range);
public slots:
void doWork();
void dataReady();
signals:
void resultReady(int identifier, QByteArray data);
void testSignals();
private:
int identifier;
QString url;
QPair<qint64, qint64> range;
QNetworkReply *reply;
};
class DownloadController : public QObject
{
Q_OBJECT
public:
explicit DownloadController(QObject *parent = nullptr);
~DownloadController();
void setFilename(QString filename);
void setThreadNum(int threadNum);
void startDownload(const QString &url, qint64 fileSize);
void paruseDownload();
void stopDownload();
public slots:
void handleResults(int identifier, QByteArray data);
// void handleTest();
signals:
void operate(const QString &url, const QPair<qint64, qint64> &downloadRange);
void errorOccur(const QString& msg);
void receivedProcess(qint64 number);
private:
int threadNum;
QString filename;
QVector<QPair<qint64, qint64>> ranges;
QFile file;
qint64 bytesReceived;
QVector<qint64> writePosList;
QMutex mutex;
};
#endif // FILEDOWNLOADWORKER_H

6
MultiplethreadDownload/filedownloadworker.cpp
View File

@ -1,6 +0,0 @@
#include "filedownloadworker.h"
FileDownloadWorker::FileDownloadWorker(QObject *parent) : QObject(parent)
{
}

16
MultiplethreadDownload/filedownloadworker.h
View File

@ -1,16 +0,0 @@
#ifndef FILEDOWNLOADWORKER_H
#define FILEDOWNLOADWORKER_H
#include <QObject>
class FileDownloadWorker : public QObject
{
Q_OBJECT
public:
explicit FileDownloadWorker(QObject *parent = nullptr);
signals:
};
#endif // FILEDOWNLOADWORKER_H

127
MultiplethreadDownload/widget.cpp
View File

@ -14,6 +14,7 @@
#include <QFuture>
#include <QtConcurrent/QtConcurrent>
#include <QMutex>
#include <downloadworker.h>
Widget::Widget(QWidget *parent)
: QWidget(parent)
@ -121,6 +122,66 @@ void Widget::singleDownload(const QString& url, const QString& filename)
ui->downloadBtn->setEnabled(true);
}
/**
* @brief 点击开始下载
*/
void Widget::on_downloadBtn_clicked()
{
QString url = ui->urlInput->text().trimmed();
if (url.isEmpty()) {
return showError(tr("请求链接不允许为空！"));
return ;
}
// 选择保存路径
QString savePath = ui->savePathInput->text().trimmed();
if (savePath.isEmpty()) {
savePath = QFileDialog::getExistingDirectory();
if (savePath.isEmpty()) return;
} else {
if(!QDir(savePath).exists()) {
return showError(tr("路径不存在！"));
}
}
ui->downloadBtn->setEnabled(false);
ui->downProgressBar->setValue(0);
qint64 fileSize = getFileSize(url);
QString sizeText = fileSize == 0 ? "未知大小" : Utils::sizeFormat(fileSize);
ui->filesizeLabel->setText(sizeText);
int process_num = ui->threadCountSpinbox->text().toInt();
QDir::setCurrent(savePath);
QString filename = QFileInfo(url).fileName();
if (fileSize == 0 || process_num == 1) {
singleDownload(url, filename);
} else {
multiDownloadWithQThread(url, fileSize, filename, process_num);
}
}
/**
* @brief 设置保存下载文件的路径
*/
void Widget::on_brwoserPathBtn_clicked()
{
QString savePath = QFileDialog::getExistingDirectory();
if (!savePath.isEmpty()) {
ui->savePathInput->setText(savePath);
}
}
void Widget::download_progress_change(qint64 bytesReceived, qint64 bytesTotal)
{
if (bytesTotal <= 0)
return;
ui->downProgressBar->setMaximum(10000); // 最大值
ui->downProgressBar->setValue((bytesReceived * 10000) / bytesTotal); // 当前值
ui->downProgressBar->setTextVisible(true);
}
/**
* @brief 多线程下载
* @param url
@ -128,7 +189,7 @@ void Widget::singleDownload(const QString& url, const QString& filename)
* @param filename
* @param threadCount
*/
void Widget::multiDownload(const QString &url, qint64 fileSize, const QString &filename, int threadCount)
void Widget::multiDownloadWithQtConcurrent(const QString &url, qint64 fileSize, const QString &filename, int threadCount)
{
QFile file(filename);
if (file.exists())
@ -192,61 +253,13 @@ void Widget::multiDownload(const QString &url, qint64 fileSize, const QString &f
ui->downloadBtn->setEnabled(true);
}
/**
* @brief 点击开始下载
*/
void Widget::on_downloadBtn_clicked()
void Widget::multiDownloadWithQThread(const QString& url, qint64 fileSize, const QString& filename, int threadCount)
{
QString url = ui->urlInput->text().trimmed();
if (url.isEmpty()) {
return showError(tr("请求链接不允许为空！"));
return ;
}
// 选择保存路径
QString savePath = ui->savePathInput->text().trimmed();
if (savePath.isEmpty()) {
savePath = QFileDialog::getExistingDirectory();
if (savePath.isEmpty()) return;
} else {
if(!QDir(savePath).exists()) {
return showError(tr("路径不存在！"));
}
}
ui->downloadBtn->setEnabled(false);
ui->downProgressBar->setValue(0);
qint64 fileSize = getFileSize(url);
QString sizeText = fileSize == 0 ? "未知大小" : Utils::sizeFormat(fileSize);
ui->filesizeLabel->setText(sizeText);
int process_num = ui->threadCountSpinbox->text().toInt();
QDir::setCurrent(savePath);
QString filename = QFileInfo(url).fileName();
if (fileSize == 0 || process_num == 1) {
singleDownload(url, filename);
} else {
multiDownload(url, fileSize, filename, process_num);
}
}
/**
* @brief 设置保存下载文件的路径
*/
void Widget::on_brwoserPathBtn_clicked()
{
QString savePath = QFileDialog::getExistingDirectory();
if (!savePath.isEmpty()) {
ui->savePathInput->setText(savePath);
}
}
void Widget::download_progress_change(qint64 bytesReceived, qint64 bytesTotal)
{
if (bytesTotal <= 0)
return;
ui->downProgressBar->setMaximum(10000); // 最大值
ui->downProgressBar->setValue((bytesReceived * 10000) / bytesTotal); // 当前值
ui->downProgressBar->setTextVisible(true);
DownloadController download;
download.setFilename(filename);
download.startDownload(url, fileSize);
connect(&download, &DownloadController::receivedProcess, [fileSize, this](int receivedNum){
int percent = receivedNum / fileSize;
ui->downProgressBar->setValue(percent);
});
}

3
MultiplethreadDownload/widget.h
View File

@ -20,7 +20,8 @@ protected:
void showError(const QString& msg);
qint64 getFileSize(const QString& url);
void singleDownload(const QString& url, const QString& filename);
void multiDownload(const QString& url, qint64 fileSize, const QString& filename, int threadCount);
void multiDownloadWithQtConcurrent(const QString& url, qint64 fileSize, const QString& filename, int threadCount);
void multiDownloadWithQThread(const QString& url, qint64 fileSize, const QString& filename, int threadCount);
private slots:

6
MultiplethreadDownload/widget.ui
View File

@ -42,7 +42,11 @@
</widget>
</item>
<item row="0" column="1">
<widget class="QLineEdit" name="urlInput"/>
<widget class="QLineEdit" name="urlInput">
<property name="text">
<string>http://sucdn.jerrywang.top/store/network/microsoft-edge-dev/microsoft-edge-dev_89.0.731.0-1_amd64.deb </string>
</property>
</widget>
</item>
<item row="0" column="2">
<widget class="QPushButton" name="downloadBtn">