更新多线程下载代码

MultiplethreadDownload/MultiplethreadDownload.pro

@@ -9,13 +9,13 @@ CONFIG += c++17
 #DEFINES += QT_DISABLE_DEPRECATED_BEFORE=0x060000    # disables all the APIs deprecated before Qt 6.0.0
 
 SOURCES += \
-    filedownloadworker.cpp \
+    downloadworker.cpp \
     main.cpp \
     utils.cpp \
     widget.cpp
 
 HEADERS += \
-    filedownloadworker.h \
+    downloadworker.h \
     utils.h \
     widget.h
 

MultiplethreadDownload/REAMDE.md

@@ -11,4 +11,35 @@
 开始下载数据： " 0~22484391 -> writePos Start 0"
 开始下载数据： " 22484392~44968783 -> writePos Start 22484392"
 开始下载数据： " 44968784~67453175 -> writePos Start 44968784"
-开始下载数据： " 67453176~89937570 -> writePos Start 67453176
+开始下载数据： " 67453176~89937570 -> writePos Start 67453176
+
+
+# 星火商店多线程下载说明
+现在的这个多线程下载已经可以了，但是暂停恢复不太好控制，所以还是用 QThread 来做暂停控制，即使线程数超过CPU
+核数问题也不大，因为下载并不是CPU密集操作，完全没问题的。总体的精细的线程池控制后续技术强大了再来封装好了。
+
+先做好这个功能，后续再来研究DTK库，各个组件的使用，已经如何自己封装漂亮的控件，摆脱DTK的依赖。
+再后面，写一个Qt开发Linux应用的教程，先建立起星火商店单独的开发者博客，专门用来发技术文章。 
+
+星火商店的几个源服务器，源列表写在 
+```sh
+$ cat /etc/apt/sources.list.d/sparkstore.list 
+deb [by-hash=force] https://sucdn.jerrywang.top /
+```
+
+服务器源列表 http://dcstore.shenmo.tech/store/server.list   
+```js
+https://sucdn.jerrywang.top/
+国内推荐 China Lines
+https://sucdn.jerrywang.top/
+https://store.deepinos.org.cn/
+国外推荐 Global Lines
+开发者模式 Dev only
+http://localhost:8080/
+用户贡献 Community lines
+```
+
+也就是目前的就两个服务器了 
+* https://sucdn.jerrywang.top/
+* https://store.deepinos.org.cn/
+

MultiplethreadDownload/downloadworker.cpp

@@ -0,0 +1,171 @@
+#include "downloadworker.h"
+#include <QIODevice>
+#include <QEventLoop>
+#include <QNetworkAccessManager>
+#include <QNetworkRequest>
+#include <QNetworkReply>
+#include <QDebug>
+
+DownloadWorker::DownloadWorker(QObject *parent)
+{
+
+}
+
+void DownloadWorker::setIdentifier(int identifier)
+{
+    this->identifier = identifier;
+}
+
+void DownloadWorker::setParamter(const QString &url, QPair<qint64, qint64> range)
+{
+    this->url = url;
+    this->range = range;
+}
+
+void DownloadWorker::doWork()
+{
+        QNetworkAccessManager *mgr = new QNetworkAccessManager(this);
+        QNetworkRequest request;
+        request.setUrl(url);
+        request.setAttribute(QNetworkRequest::FollowRedirectsAttribute, true);
+        request.setRawHeader("Range", QString("bytes=%1-%2").arg(range.first)
+                    .arg(range.second).toLocal8Bit());
+//        QNetworkReply *reply = mgr->get(request);
+//        QNetworkReply *reply = mgr->get(request);
+        reply = mgr->get(request);
+        qint64 writePos = range.first;
+        qDebug() << "开始下载数据：" << QString(" %1~%2 -> writePos Start %3")
+            .arg(range.first).arg(range.second).arg(writePos);
+//        connect(reply, &QNetworkReply::readyRead, [reply, this](){
+////            qDebug() << "测试是否触发，午安啦啦啦";
+//            QByteArray data = reply->readAll();
+//            qDebug() << "获取到了数据" << " " << data.size();
+//            emit resultReady(identifier, data);
+//        });
+//        connect(reply, &QNetworkReply::errorOccurred, [reply](QNetworkReply::NetworkError error){
+//            if (error != QNetworkReply::NoError) {
+//                qDebug() << "出错了：" << reply->errorString();
+//            }
+//        });
+        connect(reply, &QNetworkReply::finished, mgr, &QNetworkAccessManager::deleteLater);
+        connect(reply, &QNetworkReply::finished, reply, &QNetworkReply::deleteLater);
+        connect(reply, &QNetworkReply::readyRead, this, &DownloadWorker::dataReady);
+}
+
+void DownloadWorker::dataReady()
+{
+            QByteArray data = reply->readAll();
+            qDebug() << "获取到了数据" << " " << data.size();
+            emit resultReady(identifier, data);
+            auto controller = qobject_cast<DownloadController*>(parent());
+            QMetaObject::invokeMethod(controller, "handleResults", Qt::DirectConnection,
+                                      Q_ARG(int, identifier), Q_ARG(QByteArray, data));
+//            QMetaObject::invokeMethod(controller, "handleResults", Qt::QueuedConnection,
+//                                      Q_ARG(int, identifier), Q_ARG(QByteArray, data));
+}
+
+
+DownloadController::DownloadController(QObject *parent)
+{
+    this->threadNum = 4;
+
+    // 注册信号槽传递的数据类型
+    qRegisterMetaType<QPair<qint64, qint64>>("QPair<qint64, qint64>");
+}
+
+DownloadController::~DownloadController()
+{
+
+}
+
+
+void DownloadController::setFilename(QString filename)
+{
+    this->filename = filename;
+}
+
+void DownloadController::setThreadNum(int threadNum)
+{
+    this->threadNum = threadNum;
+}
+
+/**
+ * @brief 开始下载
+ */
+void DownloadController::startDownload(const QString &url, qint64 fileSize)
+{
+    // 下载任务等分，计算每个线程的下载数据
+    qint64 segmentSize = fileSize / threadNum;
+    ranges.resize(threadNum);
+//    QVector<qint64> writePosList;
+    writePosList.resize(threadNum);
+    for (int i = 0; i < threadNum; i++) {
+        ranges[i].first = i * segmentSize;
+        ranges[i].second = i * segmentSize + segmentSize - 1;
+        writePosList[i]  = ranges[i].first;
+    }
+    ranges[threadNum-1].second = fileSize; // 余数部分加入最后一个
+
+    qDebug() << QString("查看分段下载：%1 %2").arg(ranges.size()).arg(writePosList.size());
+//    return;
+    // 打开文件
+//    QFile file;
+    file.setFileName(filename);
+    if (file.exists())
+        file.remove();
+    if (!file.open(QIODevice::WriteOnly)) {
+        emit errorOccur(file.errorString());
+        return;
+    }
+    file.resize(fileSize);
+
+    // 创建下载线程
+//    qint64 bytesReceived = 0;
+    for(int i = 0; i < ranges.size(); i++) {
+        qDebug() << QString("第%1个线程：%2-%3").arg(i).arg(ranges.at(i).first).arg(ranges.at(i).second);
+        auto worker = new DownloadWorker(this);
+        auto range = ranges.at(i);
+        worker->setIdentifier(i);
+        worker->setParamter(url, range);
+        connect(worker, &DownloadWorker::resultReady, this, &DownloadController::handleResults);
+        worker->doWork();
+    }
+}
+
+/**
+ * @brief 暂停下载
+ */
+void DownloadController::paruseDownload()
+{
+
+}
+
+/**
+ * @brief 停止下载
+ */
+void DownloadController::stopDownload()
+{
+
+}
+
+/**
+ * @brief 写入响应数据
+ * @param identifier 线程编号
+ * @param data 请求数据
+ */
+void DownloadController::handleResults(int identifier, QByteArray data)
+{
+    qDebug() << QString("测试哈哈哈: %1 --- %2").arg(writePosList.size()).arg(identifier);
+    QMutexLocker lock(&mutex);
+    qint64 writePos = writePosList.at(identifier);
+    file.seek(writePos);
+    file.write(data);
+    qDebug() << QString("%1, %2, %3").arg(writePos).arg(bytesReceived).arg(data.size());
+    writePos += data.size();
+    writePosList.replace(identifier, writePos);
+    bytesReceived += data.size();
+    qDebug() << "已经下载了数据: " << bytesReceived;
+    emit receivedProcess(bytesReceived);
+}
+
+

MultiplethreadDownload/downloadworker.h

@@ -0,0 +1,65 @@
+#ifndef DOWNLOADWORKER_H
+#define DOWNLOADWORKER_H
+
+#include <QObject>
+#include <QList>
+#include <QFile>
+#include <QNetworkReply>
+#include <QMutex>
+
+class DownloadWorker : public QObject
+{
+    Q_OBJECT
+public:
+    explicit DownloadWorker(QObject *parent = nullptr);
+    void setIdentifier(int identifier);
+    void setParamter(const QString &url, QPair<qint64, qint64> range);
+
+public slots:
+    void doWork();
+    void dataReady();
+
+signals:
+    void resultReady(int identifier, QByteArray data);
+    void testSignals();
+
+
+private:
+    int identifier;
+    QString url;
+    QPair<qint64, qint64> range;
+    QNetworkReply *reply;
+};
+
+class DownloadController : public QObject
+{
+    Q_OBJECT
+public:
+    explicit DownloadController(QObject *parent = nullptr);
+    ~DownloadController();
+    void setFilename(QString filename);
+    void setThreadNum(int threadNum);
+    void startDownload(const QString &url, qint64 fileSize);
+    void paruseDownload();
+    void stopDownload();
+
+public slots:    
+    void handleResults(int identifier, QByteArray data);
+//    void handleTest();
+
+signals:
+    void operate(const QString &url, const QPair<qint64, qint64> &downloadRange);
+    void errorOccur(const QString& msg);
+    void receivedProcess(qint64 number);
+
+private:
+    int threadNum;
+    QString filename;
+    QVector<QPair<qint64, qint64>> ranges;
+    QFile file;
+    qint64 bytesReceived;
+    QVector<qint64> writePosList;
+    QMutex mutex;
+};
+
+#endif // FILEDOWNLOADWORKER_H

MultiplethreadDownload/filedownloadworker.cpp

@@ -1,6 +0,0 @@
-#include "filedownloadworker.h"
-
-FileDownloadWorker::FileDownloadWorker(QObject *parent) : QObject(parent)
-{
-
-}

MultiplethreadDownload/filedownloadworker.h

@@ -1,16 +0,0 @@
-#ifndef FILEDOWNLOADWORKER_H
-#define FILEDOWNLOADWORKER_H
-
-#include <QObject>
-
-class FileDownloadWorker : public QObject
-{
-    Q_OBJECT
-public:
-    explicit FileDownloadWorker(QObject *parent = nullptr);
-
-signals:
-
-};
-
-#endif // FILEDOWNLOADWORKER_H

MultiplethreadDownload/widget.cpp

@@ -14,6 +14,7 @@
 #include <QFuture>
 #include <QtConcurrent/QtConcurrent>
 #include <QMutex>
+#include <downloadworker.h>
 
 Widget::Widget(QWidget *parent)
     : QWidget(parent)
@@ -121,6 +122,66 @@ void Widget::singleDownload(const QString& url, const QString& filename)
     ui->downloadBtn->setEnabled(true);
 }
 
+
+/**
+ * @brief  点击开始下载
+ */
+void Widget::on_downloadBtn_clicked()
+{
+    QString url = ui->urlInput->text().trimmed();
+    if (url.isEmpty()) {
+        return showError(tr("请求链接不允许为空！"));
+        return ;
+    }
+
+    // 选择保存路径
+    QString savePath = ui->savePathInput->text().trimmed();
+    if (savePath.isEmpty()) {
+        savePath = QFileDialog::getExistingDirectory();
+        if (savePath.isEmpty()) return;
+    } else {
+        if(!QDir(savePath).exists()) {
+            return showError(tr("路径不存在！"));
+        }
+    }
+
+    ui->downloadBtn->setEnabled(false);
+    ui->downProgressBar->setValue(0);
+    qint64 fileSize = getFileSize(url);
+    QString sizeText = fileSize == 0 ? "未知大小" : Utils::sizeFormat(fileSize);
+    ui->filesizeLabel->setText(sizeText);
+
+    int process_num = ui->threadCountSpinbox->text().toInt();
+
+    QDir::setCurrent(savePath);
+    QString filename = QFileInfo(url).fileName();
+    if (fileSize == 0 || process_num == 1) {
+        singleDownload(url, filename);
+    } else {
+        multiDownloadWithQThread(url, fileSize, filename, process_num);
+    }
+}
+
+/**
+ * @brief 设置保存下载文件的路径
+ */
+void Widget::on_brwoserPathBtn_clicked()
+{
+    QString savePath = QFileDialog::getExistingDirectory();
+    if (!savePath.isEmpty()) {
+        ui->savePathInput->setText(savePath);
+    }
+}
+
+void Widget::download_progress_change(qint64 bytesReceived, qint64 bytesTotal)
+{
+    if (bytesTotal <= 0)
+        return;
+    ui->downProgressBar->setMaximum(10000); // 最大值
+    ui->downProgressBar->setValue((bytesReceived * 10000) / bytesTotal); // 当前值
+    ui->downProgressBar->setTextVisible(true);
+}
+
 /**
  * @brief 多线程下载
  * @param url
@@ -128,7 +189,7 @@ void Widget::singleDownload(const QString& url, const QString& filename)
  * @param filename
  * @param threadCount
  */
-void Widget::multiDownload(const QString &url, qint64 fileSize, const QString &filename, int threadCount)
+void Widget::multiDownloadWithQtConcurrent(const QString &url, qint64 fileSize, const QString &filename, int threadCount)
 {
     QFile file(filename);
     if (file.exists())
@@ -192,61 +253,13 @@ void Widget::multiDownload(const QString &url, qint64 fileSize, const QString &f
     ui->downloadBtn->setEnabled(true);
 }
 
-/**
+void Widget::multiDownloadWithQThread(const QString& url, qint64 fileSize, const QString& filename, int threadCount)
- * @brief  点击开始下载
- */
-void Widget::on_downloadBtn_clicked()
 {
-    QString url = ui->urlInput->text().trimmed();
+    DownloadController download;
-    if (url.isEmpty()) {
+    download.setFilename(filename);
-        return showError(tr("请求链接不允许为空！"));
+    download.startDownload(url, fileSize);
-        return ;
+    connect(&download, &DownloadController::receivedProcess, [fileSize, this](int receivedNum){
-    }
+        int percent = receivedNum / fileSize;
-
+        ui->downProgressBar->setValue(percent);
-    // 选择保存路径
+    });
-    QString savePath = ui->savePathInput->text().trimmed();
-    if (savePath.isEmpty()) {
-        savePath = QFileDialog::getExistingDirectory();
-        if (savePath.isEmpty()) return;
-    } else {
-        if(!QDir(savePath).exists()) {
-            return showError(tr("路径不存在！"));
-        }
-    }
-
-    ui->downloadBtn->setEnabled(false);
-    ui->downProgressBar->setValue(0);
-    qint64 fileSize = getFileSize(url);
-    QString sizeText = fileSize == 0 ? "未知大小" : Utils::sizeFormat(fileSize);
-    ui->filesizeLabel->setText(sizeText);
-
-    int process_num = ui->threadCountSpinbox->text().toInt();
-
-    QDir::setCurrent(savePath);
-    QString filename = QFileInfo(url).fileName();
-    if (fileSize == 0 || process_num == 1) {
-        singleDownload(url, filename);
-    } else {
-        multiDownload(url, fileSize, filename, process_num);
-    }
-}
-
-/**
- * @brief 设置保存下载文件的路径
- */
-void Widget::on_brwoserPathBtn_clicked()
-{
-    QString savePath = QFileDialog::getExistingDirectory();
-    if (!savePath.isEmpty()) {
-        ui->savePathInput->setText(savePath);
-    }
-}
-
-void Widget::download_progress_change(qint64 bytesReceived, qint64 bytesTotal)
-{
-    if (bytesTotal <= 0)
-        return;
-    ui->downProgressBar->setMaximum(10000); // 最大值
-    ui->downProgressBar->setValue((bytesReceived * 10000) / bytesTotal); // 当前值
-    ui->downProgressBar->setTextVisible(true);
 }

MultiplethreadDownload/widget.h

@@ -20,7 +20,8 @@ protected:
     void showError(const QString& msg);
     qint64 getFileSize(const QString& url);
     void singleDownload(const QString& url, const QString& filename);
-    void multiDownload(const QString& url, qint64 fileSize, const QString& filename, int threadCount);
+    void multiDownloadWithQtConcurrent(const QString& url, qint64 fileSize, const QString& filename, int threadCount);
+    void multiDownloadWithQThread(const QString& url, qint64 fileSize, const QString& filename, int threadCount);
 
 
 private slots:

MultiplethreadDownload/widget.ui

@@ -42,7 +42,11 @@
       </widget>
      </item>
      <item row="0" column="1">
-      <widget class="QLineEdit" name="urlInput"/>
+      <widget class="QLineEdit" name="urlInput">
+       <property name="text">
+        <string>http://sucdn.jerrywang.top/store/network/microsoft-edge-dev/microsoft-edge-dev_89.0.731.0-1_amd64.deb </string>
+       </property>
+      </widget>
      </item>
      <item row="0" column="2">
       <widget class="QPushButton" name="downloadBtn">