Pre-beta: BackupBook now includes downloading pdf. This replaces the need for throttling pdf downloads

This commit is contained in:
Robert McRackan 2019-11-13 09:49:23 -05:00
parent 88d49acdad
commit e69df2abbc
9 changed files with 152 additions and 105 deletions

View File

@ -1,4 +1,5 @@
using System; using System;
using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using DataLayer; using DataLayer;
using Dinah.Core.ErrorHandling; using Dinah.Core.ErrorHandling;
@ -7,7 +8,7 @@ using FileManager;
namespace FileLiberator namespace FileLiberator
{ {
/// <summary> /// <summary>
/// Download DRM book and decrypt audiobook files. /// Download DRM book and decrypt audiobook files
/// ///
/// Processes: /// Processes:
/// Download: download aax file: the DRM encrypted audiobook /// Download: download aax file: the DRM encrypted audiobook
@ -20,11 +21,12 @@ namespace FileLiberator
public event EventHandler<string> StatusUpdate; public event EventHandler<string> StatusUpdate;
public event EventHandler<string> Completed; public event EventHandler<string> Completed;
public DownloadBook Download { get; } = new DownloadBook(); public DownloadBook DownloadBook { get; } = new DownloadBook();
public DecryptBook Decrypt { get; } = new DecryptBook(); public DecryptBook DecryptBook { get; } = new DecryptBook();
public DownloadPdf DownloadPdf { get; } = new DownloadPdf();
// ValidateAsync() doesn't need UI context // ValidateAsync() doesn't need UI context
public async Task<bool> ValidateAsync(LibraryBook libraryBook) public async Task<bool> ValidateAsync(LibraryBook libraryBook)
=> await validateAsync_ConfigureAwaitFalse(libraryBook.Book.AudibleProductId).ConfigureAwait(false); => await validateAsync_ConfigureAwaitFalse(libraryBook.Book.AudibleProductId).ConfigureAwait(false);
private async Task<bool> validateAsync_ConfigureAwaitFalse(string productId) private async Task<bool> validateAsync_ConfigureAwaitFalse(string productId)
=> !await AudibleFileStorage.Audio.ExistsAsync(productId); => !await AudibleFileStorage.Audio.ExistsAsync(productId);
@ -33,22 +35,42 @@ namespace FileLiberator
// often does a lot with forms in the UI context // often does a lot with forms in the UI context
public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook) public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook)
{ {
var displayMessage = $"[{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}"; var productId = libraryBook.Book.AudibleProductId;
var displayMessage = $"[{productId}] {libraryBook.Book.Title}";
Begin?.Invoke(this, displayMessage); Begin?.Invoke(this, displayMessage);
try try
{ {
var aaxExists = await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId); {
if (!aaxExists) var statusHandler = await processAsync(libraryBook, AudibleFileStorage.AAX, DownloadBook);
await Download.ProcessAsync(libraryBook); if (statusHandler.Any())
return statusHandler;
}
return await Decrypt.ProcessAsync(libraryBook); {
} var statusHandler = await processAsync(libraryBook, AudibleFileStorage.Audio, DecryptBook);
finally if (statusHandler.Any())
return statusHandler;
}
{
var statusHandler = await processAsync(libraryBook, AudibleFileStorage.PDF, DownloadPdf);
if (statusHandler.Any())
return statusHandler;
}
return new StatusHandler();
}
finally
{ {
Completed?.Invoke(this, displayMessage); Completed?.Invoke(this, displayMessage);
} }
} }
}
private static async Task<StatusHandler> processAsync(LibraryBook libraryBook, AudibleFileStorage afs, IProcessable processable)
=> !await afs.ExistsAsync(libraryBook.Book.AudibleProductId)
? await processable.ProcessAsync(libraryBook)
: new StatusHandler();
}
} }

View File

@ -12,7 +12,7 @@ using FileManager;
namespace FileLiberator namespace FileLiberator
{ {
/// <summary> /// <summary>
/// Download DRM book and decrypt audiobook files. /// Decrypt audiobook files
/// ///
/// Processes: /// Processes:
/// Download: download aax file: the DRM encrypted audiobook /// Download: download aax file: the DRM encrypted audiobook

View File

@ -1,14 +1,14 @@
using System; using System;
using System.IO; using System.IO;
using System.Threading.Tasks; using System.Threading.Tasks;
using FileManager;
using DataLayer; using DataLayer;
using Dinah.Core.ErrorHandling; using Dinah.Core.ErrorHandling;
using FileManager;
namespace FileLiberator namespace FileLiberator
{ {
/// <summary> /// <summary>
/// Download DRM book and decrypt audiobook files. /// Download DRM book
/// ///
/// Processes: /// Processes:
/// Download: download aax file: the DRM encrypted audiobook /// Download: download aax file: the DRM encrypted audiobook

View File

@ -9,7 +9,7 @@ using FileManager;
namespace FileLiberator namespace FileLiberator
{ {
public class DownloadPdf : DownloadableBase public class DownloadPdf : DownloadableBase
{ {
static DownloadPdf() static DownloadPdf()
{ {
@ -18,71 +18,68 @@ namespace FileLiberator
} }
public override async Task<bool> ValidateAsync(LibraryBook libraryBook) public override async Task<bool> ValidateAsync(LibraryBook libraryBook)
{ {
var product = libraryBook.Book; var product = libraryBook.Book;
if (!product.Supplements.Any()) if (!product.Supplements.Any())
return false; return false;
return !await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId); return !await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);
} }
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook) public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
{ {
var product = libraryBook.Book; var product = libraryBook.Book;
if (product == null) if (product == null)
return new StatusHandler { "Book not found" }; return new StatusHandler { "Book not found" };
var urls = product.Supplements.Select(d => d.Url).ToList(); var urls = product.Supplements.Select(d => d.Url).ToList();
if (urls.Count == 0) if (urls.Count == 0)
return new StatusHandler { "PDF download url not found" }; return new StatusHandler { "PDF download url not found" };
// sanity check // sanity check
if (urls.Count > 1) if (urls.Count > 1)
throw new Exception("Multiple PDF downloads are not currently supported. typically indicates an error"); throw new Exception("Multiple PDF downloads are not currently supported. Typically indicates an error");
var url = urls.Single(); var destinationDir = await getDestinationDirectoryAsync(product.AudibleProductId);
if (destinationDir == null)
return new StatusHandler { "Destination directory not found for PDF download" };
var destinationDir = await getDestinationDirectory(product.AudibleProductId); var url = urls.Single();
if (destinationDir == null) var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url));
return new StatusHandler { "Destination directory not found for PDF download" }; await performDownloadAsync(url, destinationFilename);
var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url)); var statusHandler = new StatusHandler();
var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);
if (!exists)
statusHandler.AddError("Downloaded PDF cannot be found");
return statusHandler;
}
using var webClient = GetWebClient(destinationFilename); private async Task<string> getDestinationDirectoryAsync(string productId)
await webClient.DownloadFileTaskAsync(url, destinationFilename); {
// if audio file exists, get it's dir
var audioFile = await AudibleFileStorage.Audio.GetAsync(productId);
if (audioFile != null)
return Path.GetDirectoryName(audioFile);
var statusHandler = new StatusHandler(); // else return base Book dir
var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId); return AudibleFileStorage.PDF.StorageDirectory;
if (!exists) }
statusHandler.AddError("Downloaded PDF cannot be found");
return statusHandler;
}
private async Task<string> getDestinationDirectory(string productId)
{
// if audio file exists, get it's dir
var audioFile = await AudibleFileStorage.Audio.GetAsync(productId);
if (audioFile != null)
return Path.GetDirectoryName(audioFile);
// else return base Book dir
return AudibleFileStorage.PDF.StorageDirectory;
}
// other user agents from my chrome. from: https://www.whoishostingthis.com/tools/user-agent/ // other user agents from my chrome. from: https://www.whoishostingthis.com/tools/user-agent/
private static string[] userAgents { get; } = new[] private static string[] userAgents { get; } = new[]
{ {
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
}; };
private WebClient GetWebClient(string downloadMessage)
private async Task performDownloadAsync(string url, string destinationFilename)
{ {
var webClient = new WebClient(); using var webClient = new WebClient();
var userAgentIndex = new Random().Next(0, userAgents.Length); // upper bound is exclusive var userAgentIndex = new Random().Next(0, userAgents.Length); // upper bound is exclusive
webClient.Headers["User-Agent"] = userAgents[userAgentIndex]; webClient.Headers["User-Agent"] = userAgents[userAgentIndex];
@ -93,13 +90,13 @@ namespace FileLiberator
webClient.Headers["Accept-Language"] = "en-US,en;q=0.9"; webClient.Headers["Accept-Language"] = "en-US,en;q=0.9";
webClient.DownloadProgressChanged += (s, e) => Invoke_DownloadProgressChanged(s, new Dinah.Core.Net.Http.DownloadProgress { BytesReceived = e.BytesReceived, ProgressPercentage = e.ProgressPercentage, TotalBytesToReceive = e.TotalBytesToReceive }); webClient.DownloadProgressChanged += (s, e) => Invoke_DownloadProgressChanged(s, new Dinah.Core.Net.Http.DownloadProgress { BytesReceived = e.BytesReceived, ProgressPercentage = e.ProgressPercentage, TotalBytesToReceive = e.TotalBytesToReceive });
webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}"); webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}"); webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}"); webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
Invoke_DownloadBegin(downloadMessage); Invoke_DownloadBegin(destinationFilename);
return webClient; await webClient.DownloadFileTaskAsync(url, destinationFilename);
} }
} }
} }

View File

@ -29,8 +29,10 @@ namespace FileManager
[".aac"] = FileType.Audio, [".aac"] = FileType.Audio,
[".mp4"] = FileType.Audio, [".mp4"] = FileType.Audio,
[".m4a"] = FileType.Audio, [".m4a"] = FileType.Audio,
[".ogg"] = FileType.Audio,
[".flac"] = FileType.Audio,
[".aax"] = FileType.AAX, [".aax"] = FileType.AAX,
[".pdf"] = FileType.PDF, [".pdf"] = FileType.PDF,
[".zip"] = FileType.PDF, [".zip"] = FileType.PDF,

View File

@ -23,8 +23,8 @@ namespace LibationWinForm.BookLiberation
return; return;
var backupBook = new BackupBook(); var backupBook = new BackupBook();
backupBook.Download.Completed += SetBackupCountsAsync; backupBook.DownloadBook.Completed += SetBackupCountsAsync;
backupBook.Decrypt.Completed += SetBackupCountsAsync; backupBook.DecryptBook.Completed += SetBackupCountsAsync;
await ProcessValidateLibraryBookAsync(backupBook, libraryBook); await ProcessValidateLibraryBookAsync(backupBook, libraryBook);
} }
@ -55,8 +55,8 @@ namespace LibationWinForm.BookLiberation
async Task BackupFirstBookAsync() async Task BackupFirstBookAsync()
{ {
var backupBook = ProcessorAutomationController.GetWiredUpBackupBook(); var backupBook = ProcessorAutomationController.GetWiredUpBackupBook();
backupBook.Download.Completed += SetBackupCountsAsync; backupBook.DownloadBook.Completed += SetBackupCountsAsync;
backupBook.Decrypt.Completed += SetBackupCountsAsync; backupBook.DecryptBook.Completed += SetBackupCountsAsync;
await backupBook.ProcessFirstValidAsync(); await backupBook.ProcessFirstValidAsync();
} }

View File

@ -16,10 +16,11 @@ namespace LibationWinForm.BookLiberation
{ {
var backupBook = new BackupBook(); var backupBook = new BackupBook();
backupBook.Download.Begin += (_, __) => wireUpDownloadable(backupBook.Download); backupBook.DownloadBook.Begin += (_, __) => wireUpDownloadable(backupBook.DownloadBook);
backupBook.Decrypt.Begin += (_, __) => wireUpDecryptable(backupBook.Decrypt); backupBook.DecryptBook.Begin += (_, __) => wireUpDecryptable(backupBook.DecryptBook);
backupBook.DownloadPdf.Begin += (_, __) => wireUpDecryptable(backupBook.DecryptBook);
return backupBook; return backupBook;
} }
public static DecryptBook GetWiredUpDecryptBook() public static DecryptBook GetWiredUpDecryptBook()
{ {
@ -191,34 +192,43 @@ namespace LibationWinForm.BookLiberation
#endregion #endregion
#region define how model actions will affect form behavior #region define how model actions will affect form behavior
void downloadBegin(object _, string str) => automatedBackupsForm.AppendText("DownloadStep_Begin: " + str); void downloadBookBegin(object _, string str) => automatedBackupsForm.AppendText("DownloadStep_Begin: " + str);
void statusUpdate(object _, string str) => automatedBackupsForm.AppendText("- " + str); void statusUpdate(object _, string str) => automatedBackupsForm.AppendText("- " + str);
void downloadCompleted(object _, string str) => automatedBackupsForm.AppendText("DownloadStep_Completed: " + str); void downloadBookCompleted(object _, string str) => automatedBackupsForm.AppendText("DownloadStep_Completed: " + str);
void decryptBegin(object _, string str) => automatedBackupsForm.AppendText("DecryptStep_Begin: " + str); void decryptBookBegin(object _, string str) => automatedBackupsForm.AppendText("DecryptStep_Begin: " + str);
// extra line after book is completely finished // extra line after book is completely finished
void decryptCompleted(object _, string str) => automatedBackupsForm.AppendText("DecryptStep_Completed: " + str + Environment.NewLine); void decryptBookCompleted(object _, string str) => automatedBackupsForm.AppendText("DecryptStep_Completed: " + str + Environment.NewLine);
#endregion void downloadPdfBegin(object _, string str) => automatedBackupsForm.AppendText("PdfStep_Begin: " + str);
// extra line after book is completely finished
void downloadPdfCompleted(object _, string str) => automatedBackupsForm.AppendText("PdfStep_Completed: " + str + Environment.NewLine);
#endregion
#region subscribe new form to model's events #region subscribe new form to model's events
backupBook.Download.Begin += downloadBegin; backupBook.DownloadBook.Begin += downloadBookBegin;
backupBook.Download.StatusUpdate += statusUpdate; backupBook.DownloadBook.StatusUpdate += statusUpdate;
backupBook.Download.Completed += downloadCompleted; backupBook.DownloadBook.Completed += downloadBookCompleted;
backupBook.Decrypt.Begin += decryptBegin; backupBook.DecryptBook.Begin += decryptBookBegin;
backupBook.Decrypt.StatusUpdate += statusUpdate; backupBook.DecryptBook.StatusUpdate += statusUpdate;
backupBook.Decrypt.Completed += decryptCompleted; backupBook.DecryptBook.Completed += decryptBookCompleted;
#endregion backupBook.DownloadPdf.Begin += downloadPdfBegin;
backupBook.DownloadPdf.StatusUpdate += statusUpdate;
backupBook.DownloadPdf.Completed += downloadPdfCompleted;
#endregion
#region when form closes, unsubscribe from model's events #region when form closes, unsubscribe from model's events
// unsubscribe so disposed forms aren't still trying to receive notifications // unsubscribe so disposed forms aren't still trying to receive notifications
automatedBackupsForm.FormClosing += (_, __) => automatedBackupsForm.FormClosing += (_, __) =>
{ {
backupBook.Download.Begin -= downloadBegin; backupBook.DownloadBook.Begin -= downloadBookBegin;
backupBook.Download.StatusUpdate -= statusUpdate; backupBook.DownloadBook.StatusUpdate -= statusUpdate;
backupBook.Download.Completed -= downloadCompleted; backupBook.DownloadBook.Completed -= downloadBookCompleted;
backupBook.Decrypt.Begin -= decryptBegin; backupBook.DecryptBook.Begin -= decryptBookBegin;
backupBook.Decrypt.StatusUpdate -= statusUpdate; backupBook.DecryptBook.StatusUpdate -= statusUpdate;
backupBook.Decrypt.Completed -= decryptCompleted; backupBook.DecryptBook.Completed -= decryptBookCompleted;
}; backupBook.DownloadPdf.Begin -= downloadPdfBegin;
backupBook.DownloadPdf.StatusUpdate -= statusUpdate;
backupBook.DownloadPdf.Completed -= downloadPdfCompleted;
};
#endregion #endregion
await runBackupLoop(backupBook, automatedBackupsForm); await runBackupLoop(backupBook, automatedBackupsForm);

View File

@ -281,9 +281,10 @@ namespace LibationWinForm
private async void beginBookBackupsToolStripMenuItem_Click(object sender, EventArgs e) private async void beginBookBackupsToolStripMenuItem_Click(object sender, EventArgs e)
{ {
var backupBook = BookLiberation.ProcessorAutomationController.GetWiredUpBackupBook(); var backupBook = BookLiberation.ProcessorAutomationController.GetWiredUpBackupBook();
backupBook.Download.Completed += setBackupCountsAsync; backupBook.DownloadBook.Completed += setBackupCountsAsync;
backupBook.Decrypt.Completed += setBackupCountsAsync; backupBook.DecryptBook.Completed += setBackupCountsAsync;
await BookLiberation.ProcessorAutomationController.RunAutomaticBackup(backupBook); backupBook.DownloadPdf.Completed += setBackupCountsAsync;
await BookLiberation.ProcessorAutomationController.RunAutomaticBackup(backupBook);
} }
private async void beginPdfBackupsToolStripMenuItem_Click(object sender, EventArgs e) private async void beginPdfBackupsToolStripMenuItem_Click(object sender, EventArgs e)

View File

@ -1,11 +1,24 @@
-- begin BETA --------------------------------------------------------------------------------------------------------------------- -- begin BETA ---------------------------------------------------------------------------------------------------------------------
throttle needed for downloading pdf.s throttle needed for downloading pdf.s
picture storage should be more responsive if on disk?
download now incl pdf
pdf menu should be 'PDFs only'
update wording in menu
update readme
no mdf,ldf files created in C:\Users\[username] no mdf,ldf files created in C:\Users\[username]
Warn of known performance issues Warn of known performance issues
- Library import - Library import
incl image d/l. need to throttle
- Tag add/edit - Tag add/edit
- Grid is slow to respond loading when books aren't liberated - Grid is slow to respond loading when books aren't liberated
- get decrypt key -- unavoidable - get decrypt key -- unavoidable
@ -16,14 +29,16 @@ https://dotnetcoretutorials.com/2019/06/20/publishing-a-single-exe-file-in-net-c
-- begin ENHANCEMENT, IMPORT UI --------------------------------------------------------------------------------------------------------------------- -- begin ENHANCEMENT, IMPORT UI ---------------------------------------------------------------------------------------------------------------------
scan library in background? scan library in background?
can include a notice somewhere that it's in-process can include a notice somewhere that a scan is in-process
why block the UI at all? why block the UI at all?
what to do if new books? don't want to refresh grid when user isn't expecting it
-- end ENHANCEMENT, IMPORT UI --------------------------------------------------------------------------------------------------------------------- -- end ENHANCEMENT, IMPORT UI ---------------------------------------------------------------------------------------------------------------------
-- begin BUG, FILE DOWNLOAD --------------------------------------------------------------------------------------------------------------------- -- begin BUG, FILE DOWNLOAD ---------------------------------------------------------------------------------------------------------------------
reproduce: try to do the api download with a bad codec reproduce: try to do the api download with a bad codec
result: DownloadsFinal dir .aax file 1 kb result: DownloadsFinal dir .aax file 1 kb
this resulted from an exception. we should not be keeping a file after exception this resulted from an exception. we should not be keeping a file after exception
if error: show error. DownloadBook delete bad file
-- end BUG, FILE DOWNLOAD --------------------------------------------------------------------------------------------------------------------- -- end BUG, FILE DOWNLOAD ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, PERFORMANCE: IMPORT --------------------------------------------------------------------------------------------------------------------- -- begin ENHANCEMENT, PERFORMANCE: IMPORT ---------------------------------------------------------------------------------------------------------------------
@ -31,7 +46,7 @@ imports are PAINFULLY slow for just a few hundred items. wtf is taking so long?
-- end ENHANCEMENT, PERFORMANCE: IMPORT --------------------------------------------------------------------------------------------------------------------- -- end ENHANCEMENT, PERFORMANCE: IMPORT ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, PERFORMANCE: GRID --------------------------------------------------------------------------------------------------------------------- -- begin ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
when a book/pdf is NOT liberated, calculating the grid's [Liberated][NOT d/l'ed] label is very slow when a book/pdf is NOT liberated, calculating the grid's [Liberated][NOT d/l'ed] label is very slow. use something similar to PictureStorage's timer to run on a separate thread
https://stackoverflow.com/a/12046333 https://stackoverflow.com/a/12046333
https://codereview.stackexchange.com/a/135074 https://codereview.stackexchange.com/a/135074
// do NOT use lock() or Monitor with async/await // do NOT use lock() or Monitor with async/await