I run 2.0.6.102_canary_2022-04-06 on Windows 10
I’m doing backup of 3TB of photos and video files from external HDD to STORJ.
There are 70k files in total with sizes from 1MB to 3GB.
Backup goes very slow, with upload speed (as shown in progress bar) only 700kb/s despite my broadband connection allows up to 20mb/s so it takes the whole day to backup only 100-200gb.
To compare I have another STORJ backup of single 5GB file and it uploads with 3mb/s, so, I think, large number of files somehow significantly degrade upload speed.
During backup progress memory consumption by duplicati process constantly increases and after 24 hours of work it eats 20Gb of RAM.
When click (X) on progess bar and stop backing up, RAM usage is not reduced. Logs show that everything is normal and backup is successfully done.
Is there any solution to prevent memory leak and to increase the upload speed ?
Options:
Log File for last day’s backup:
{
"DeletedFiles": 0,
"DeletedFolders": 0,
"ModifiedFiles": 0,
"ExaminedFiles": 22914,
"OpenedFiles": 22912,
"AddedFiles": 136,
"SizeOfModifiedFiles": 0,
"SizeOfAddedFiles": 63483535895,
"SizeOfExaminedFiles": 880166329300,
"SizeOfOpenedFiles": 879181625010,
"NotProcessedFiles": 0,
"AddedFolders": 0,
"TooLargeFiles": 0,
"FilesWithError": 0,
"ModifiedFolders": 0,
"ModifiedSymlinks": 0,
"AddedSymlinks": 0,
"DeletedSymlinks": 0,
"PartialBackup": true,
"Dryrun": false,
"MainOperation": "Backup",
"CompactResults": null,
"VacuumResults": null,
"DeleteResults": {
"DeletedSetsActualLength": 0,
"DeletedSets": [],
"Dryrun": false,
"MainOperation": "Delete",
"CompactResults": null,
"ParsedResult": "Success",
"Version": "2.0.6.102 (2.0.6.102_canary_2022-04-06)",
"EndTime": "2022-06-12T09:46:17.8531579Z",
"BeginTime": "2022-06-12T09:44:57.6658186Z",
"Duration": "00:01:20.1873393",
"MessagesActualLength": 0,
"WarningsActualLength": 0,
"ErrorsActualLength": 0,
"Messages": null,
"Warnings": null,
"Errors": null,
"BackendStatistics": {
"RemoteCalls": 1896,
"BytesUploaded": 63521407486,
"BytesDownloaded": 0,
"FilesUploaded": 1894,
"FilesDownloaded": 0,
"FilesDeleted": 0,
"FoldersCreated": 0,
"RetryAttempts": 0,
"UnknownFileSize": 0,
"UnknownFileCount": 0,
"KnownFileCount": 26239,
"KnownFileSize": 880674578083,
"LastBackupDate": "2022-06-11T13:40:11+02:00",
"BackupListCount": 3,
"TotalQuotaSpace": 0,
"FreeQuotaSpace": 0,
"AssignedQuotaSpace": -1,
"ReportedQuotaError": false,
"ReportedQuotaWarning": false,
"MainOperation": "Backup",
"ParsedResult": "Success",
"Version": "2.0.6.102 (2.0.6.102_canary_2022-04-06)",
"EndTime": "0001-01-01T00:00:00",
"BeginTime": "2022-06-11T11:40:11.7348369Z",
"Duration": "00:00:00",
"MessagesActualLength": 0,
"WarningsActualLength": 0,
"ErrorsActualLength": 0,
"Messages": null,
"Warnings": null,
"Errors": null
}
},
"RepairResults": null,
"TestResults": null,
"ParsedResult": "Warning",
"Version": "2.0.6.102 (2.0.6.102_canary_2022-04-06)",
"EndTime": "2022-06-12T09:46:58.0074845Z",
"BeginTime": "2022-06-11T11:40:11.7338368Z",
"Duration": "22:06:46.2736477",
"MessagesActualLength": 3797,
"WarningsActualLength": 1,
"ErrorsActualLength": 0,
"Messages": [
"2022-06-11 13:40:11 +02 - [Information-Duplicati.Library.Main.Controller-StartingOperation]: The operation Backup has started",
"2022-06-11 13:40:14 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: List - Started: ()",
"2022-06-11 13:40:37 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: List - Completed: (23.77 KB)",
"2022-06-11 13:40:38 +02 - [Information-Duplicati.Library.Main.Operation.FilelistProcessor-KeepIncompleteFile]: keeping protected incomplete remote file listed as Temporary: duplicati-20220609T121930Z.dlist.zip.aes",
"2022-06-11 13:40:38 +02 - [Information-Duplicati.Library.Main.Operation.Backup.UploadSyntheticFilelist-PreviousBackupFilelistUpload]: Uploading filelist from previous interrupted backup",
"2022-06-11 16:30:13 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Started: duplicati-b24f520fd2a0547b9bedce0f914b0fcc7.dblock.zip.aes (63.97 MB)",
"2022-06-11 16:30:37 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Completed: duplicati-b24f520fd2a0547b9bedce0f914b0fcc7.dblock.zip.aes (63.97 MB)",
"2022-06-11 16:31:05 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Started: duplicati-if8aa2af56e8f40389d3018ca9cf65035.dindex.zip.aes (45.54 KB)",
"2022-06-11 16:31:06 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Completed: duplicati-if8aa2af56e8f40389d3018ca9cf65035.dindex.zip.aes (45.54 KB)",
"2022-06-11 16:31:56 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Started: duplicati-bdfa457288e0e486785cd777fc9674b8c.dblock.zip.aes (63.93 MB)",
"2022-06-11 16:32:28 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Completed: duplicati-bdfa457288e0e486785cd777fc9674b8c.dblock.zip.aes (63.93 MB)",
"2022-06-11 16:33:15 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Started: duplicati-ie0eaa0088aef4a598a8eb56d906b1d47.dindex.zip.aes (45.54 KB)",
"2022-06-11 16:33:16 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Completed: duplicati-ie0eaa0088aef4a598a8eb56d906b1d47.dindex.zip.aes (45.54 KB)",
"2022-06-11 16:34:04 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Started: duplicati-b92059cbd5de9474494214b92281ac30f.dblock.zip.aes (63.97 MB)",
"2022-06-11 16:34:34 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Completed: duplicati-b92059cbd5de9474494214b92281ac30f.dblock.zip.aes (63.97 MB)",
"2022-06-11 16:35:23 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Started: duplicati-i5c1f791a58eb40469237e3e1b0287d3b.dindex.zip.aes (45.54 KB)",
"2022-06-11 16:35:24 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Completed: duplicati-i5c1f791a58eb40469237e3e1b0287d3b.dindex.zip.aes (45.54 KB)",
"2022-06-11 16:36:18 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Started: duplicati-bade4312b37c94479bed7e39e3a3d6a3f.dblock.zip.aes (63.97 MB)",
"2022-06-11 16:36:48 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Completed: duplicati-bade4312b37c94479bed7e39e3a3d6a3f.dblock.zip.aes (63.97 MB)",
"2022-06-11 16:37:42 +02 - [Information-Duplicati.Library.Main.BasicResults-BackendEvent]: Backend event: Put - Started: duplicati-i5e326e87f23342fbacfe0306f430cdb3.dindex.zip.aes (45.62 KB)"
],
"Warnings": [
"2022-06-12 11:43:36 +02 - [Warning-Duplicati.Library.Main.Operation.BackupHandler-CancellationRequested]: Cancellation was requested by user."
],
"Errors": [],
"BackendStatistics": {
"RemoteCalls": 1896,
"BytesUploaded": 63521407486,
"BytesDownloaded": 0,
"FilesUploaded": 1894,
"FilesDownloaded": 0,
"FilesDeleted": 0,
"FoldersCreated": 0,
"RetryAttempts": 0,
"UnknownFileSize": 0,
"UnknownFileCount": 0,
"KnownFileCount": 26239,
"KnownFileSize": 880674578083,
"LastBackupDate": "2022-06-11T13:40:11+02:00",
"BackupListCount": 3,
"TotalQuotaSpace": 0,
"FreeQuotaSpace": 0,
"AssignedQuotaSpace": -1,
"ReportedQuotaError": false,
"ReportedQuotaWarning": false,
"MainOperation": "Backup",
"ParsedResult": "Success",
"Version": "2.0.6.102 (2.0.6.102_canary_2022-04-06)",
"EndTime": "0001-01-01T00:00:00",
"BeginTime": "2022-06-11T11:40:11.7348369Z",
"Duration": "00:00:00",
"MessagesActualLength": 0,
"WarningsActualLength": 0,
"ErrorsActualLength": 0,
"Messages": null,
"Warnings": null,
"Errors": null
}
}