Avoid holding encoded resume data in memory

Now it the encoded resume data will be streamed to file instead of a
temporary buffer holding the whole of it.
This commit is contained in:
Chocobo1 2020-04-04 12:50:11 +08:00
parent fe81633403
commit d8401c76f5
No known key found for this signature in database
GPG key ID: 210D9C873253A68C
5 changed files with 44 additions and 18 deletions

View file

@ -28,11 +28,15 @@
#include "resumedatasavingmanager.h" #include "resumedatasavingmanager.h"
#include <libtorrent/bencode.hpp>
#include <libtorrent/entry.hpp>
#include <QByteArray> #include <QByteArray>
#include <QSaveFile> #include <QSaveFile>
#include "base/logger.h" #include "base/logger.h"
#include "base/utils/fs.h" #include "base/utils/fs.h"
#include "base/utils/io.h"
ResumeDataSavingManager::ResumeDataSavingManager(const QString &resumeFolderPath) ResumeDataSavingManager::ResumeDataSavingManager(const QString &resumeFolderPath)
: m_resumeDataDir(resumeFolderPath) : m_resumeDataDir(resumeFolderPath)
@ -50,6 +54,24 @@ void ResumeDataSavingManager::save(const QString &filename, const QByteArray &da
} }
} }
void ResumeDataSavingManager::save(const QString &filename, const std::shared_ptr<lt::entry> &data) const
{
const QString filepath = m_resumeDataDir.absoluteFilePath(filename);
QSaveFile file {filepath};
if (!file.open(QIODevice::WriteOnly)) {
LogMsg(tr("Couldn't save data to '%1'. Error: %2")
.arg(filepath, file.errorString()), Log::CRITICAL);
return;
}
lt::bencode(Utils::IO::FileDeviceOutputIterator {file}, *data);
if ((file.error() != QFileDevice::NoError) || !file.commit()) {
LogMsg(tr("Couldn't save data to '%1'. Error: %2")
.arg(filepath, file.errorString()), Log::CRITICAL);
}
}
void ResumeDataSavingManager::remove(const QString &filename) const void ResumeDataSavingManager::remove(const QString &filename) const
{ {
const QString filepath = m_resumeDataDir.absoluteFilePath(filename); const QString filepath = m_resumeDataDir.absoluteFilePath(filename);

View file

@ -28,6 +28,10 @@
#pragma once #pragma once
#include <memory>
#include <libtorrent/fwd.hpp>
#include <QDir> #include <QDir>
#include <QObject> #include <QObject>
@ -43,8 +47,9 @@ public:
public slots: public slots:
void save(const QString &filename, const QByteArray &data) const; void save(const QString &filename, const QByteArray &data) const;
void save(const QString &filename, const std::shared_ptr<lt::entry> &data) const;
void remove(const QString &filename) const; void remove(const QString &filename) const;
private: private:
QDir m_resumeDataDir; const QDir m_resumeDataDir;
}; };

View file

@ -3989,25 +3989,20 @@ void Session::handleTorrentFinished(TorrentHandle *const torrent)
emit allTorrentsFinished(); emit allTorrentsFinished();
} }
void Session::handleTorrentResumeDataReady(TorrentHandle *const torrent, const lt::entry &data) void Session::handleTorrentResumeDataReady(TorrentHandle *const torrent, const std::shared_ptr<lt::entry> &data)
{ {
--m_numResumeData; --m_numResumeData;
// Separated thread is used for the blocking IO which results in slow processing of many torrents. // Separated thread is used for the blocking IO which results in slow processing of many torrents.
// Encoding data in parallel while doing IO saves time. Copying lt::entry objects around // Copying lt::entry objects around isn't cheap.
// isn't cheap too.
QByteArray out;
out.reserve(1024 * 1024); // most fastresume file sizes are under 1 MB
lt::bencode(std::back_inserter(out), data);
const QString filename = QString::fromLatin1("%1.fastresume").arg(torrent->hash()); const QString filename = QString::fromLatin1("%1.fastresume").arg(torrent->hash());
#if (QT_VERSION >= QT_VERSION_CHECK(5, 10, 0)) #if (QT_VERSION >= QT_VERSION_CHECK(5, 10, 0))
QMetaObject::invokeMethod(m_resumeDataSavingManager QMetaObject::invokeMethod(m_resumeDataSavingManager
, [this, filename, out]() { m_resumeDataSavingManager->save(filename, out); }); , [this, filename, data]() { m_resumeDataSavingManager->save(filename, data); });
#else #else
QMetaObject::invokeMethod(m_resumeDataSavingManager, "save", QMetaObject::invokeMethod(m_resumeDataSavingManager, "save"
Q_ARG(QString, filename), Q_ARG(QByteArray, out)); , Q_ARG(QString, filename), Q_ARG(std::shared_ptr<lt::entry>, data));
#endif #endif
} }

View file

@ -30,6 +30,7 @@
#ifndef BITTORRENT_SESSION_H #ifndef BITTORRENT_SESSION_H
#define BITTORRENT_SESSION_H #define BITTORRENT_SESSION_H
#include <memory>
#include <vector> #include <vector>
#include <libtorrent/fwd.hpp> #include <libtorrent/fwd.hpp>
@ -459,7 +460,7 @@ namespace BitTorrent
void handleTorrentTrackersChanged(TorrentHandle *const torrent); void handleTorrentTrackersChanged(TorrentHandle *const torrent);
void handleTorrentUrlSeedsAdded(TorrentHandle *const torrent, const QVector<QUrl> &newUrlSeeds); void handleTorrentUrlSeedsAdded(TorrentHandle *const torrent, const QVector<QUrl> &newUrlSeeds);
void handleTorrentUrlSeedsRemoved(TorrentHandle *const torrent, const QVector<QUrl> &urlSeeds); void handleTorrentUrlSeedsRemoved(TorrentHandle *const torrent, const QVector<QUrl> &urlSeeds);
void handleTorrentResumeDataReady(TorrentHandle *const torrent, const lt::entry &data); void handleTorrentResumeDataReady(TorrentHandle *const torrent, const std::shared_ptr<lt::entry> &data);
void handleTorrentResumeDataFailed(TorrentHandle *const torrent); void handleTorrentResumeDataFailed(TorrentHandle *const torrent);
void handleTorrentTrackerReply(TorrentHandle *const torrent, const QString &trackerUrl); void handleTorrentTrackerReply(TorrentHandle *const torrent, const QString &trackerUrl);
void handleTorrentTrackerWarning(TorrentHandle *const torrent, const QString &trackerUrl); void handleTorrentTrackerWarning(TorrentHandle *const torrent, const QString &trackerUrl);

View file

@ -30,6 +30,7 @@
#include "torrenthandle.h" #include "torrenthandle.h"
#include <algorithm> #include <algorithm>
#include <memory>
#include <type_traits> #include <type_traits>
#ifdef Q_OS_WIN #ifdef Q_OS_WIN
@ -1647,14 +1648,16 @@ void TorrentHandle::handleSaveResumeDataAlert(const lt::save_resume_data_alert *
{ {
#if (LIBTORRENT_VERSION_NUM < 10200) #if (LIBTORRENT_VERSION_NUM < 10200)
const bool useDummyResumeData = !(p && p->resume_data); const bool useDummyResumeData = !(p && p->resume_data);
lt::entry dummyEntry; auto resumeDataPtr = std::make_shared<lt::entry>(useDummyResumeData
? lt::entry {}
lt::entry &resumeData = useDummyResumeData ? dummyEntry : *(p->resume_data); : *(p->resume_data));
#else #else
const bool useDummyResumeData = !p; const bool useDummyResumeData = !p;
auto resumeDataPtr = std::make_shared<lt::entry>(useDummyResumeData
lt::entry resumeData = useDummyResumeData ? lt::entry() : lt::write_resume_data(p->params); ? lt::entry {}
: lt::write_resume_data(p->params));
#endif #endif
lt::entry &resumeData = *resumeDataPtr;
updateStatus(); updateStatus();
@ -1697,7 +1700,7 @@ void TorrentHandle::handleSaveResumeDataAlert(const lt::save_resume_data_alert *
resumeData["auto_managed"] = false; resumeData["auto_managed"] = false;
} }
m_session->handleTorrentResumeDataReady(this, resumeData); m_session->handleTorrentResumeDataReady(this, resumeDataPtr);
} }
void TorrentHandle::handleSaveResumeDataFailedAlert(const lt::save_resume_data_failed_alert *p) void TorrentHandle::handleSaveResumeDataFailedAlert(const lt::save_resume_data_failed_alert *p)