tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pre-beta: BackupBook now includes downloading pdf. This replaces the need for throttling pdf downloads

Browse Source
This commit is contained in:
Robert McRackan 2019-11-13 09:49:23 -05:00
parent 88d49acdad
commit e69df2abbc
9 changed files with 152 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
FileLiberator/UNTESTED/BackupBook.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using DataLayer;
using Dinah.Core.ErrorHandling;
@ -7,7 +8,7 @@ using FileManager;
namespace FileLiberator
{
/// <summary>
/// Download DRM book and decrypt audiobook files.
/// Download DRM book and decrypt audiobook files
///
/// Processes:
/// Download: download aax file: the DRM encrypted audiobook
@ -20,11 +21,12 @@ namespace FileLiberator
public event EventHandler<string> StatusUpdate;
public event EventHandler<string> Completed;
public DownloadBook Download { get; } = new DownloadBook();
public DecryptBook Decrypt { get; } = new DecryptBook();
public DownloadBook DownloadBook { get; } = new DownloadBook();
public DecryptBook DecryptBook { get; } = new DecryptBook();
public DownloadPdf DownloadPdf { get; } = new DownloadPdf();
// ValidateAsync() doesn't need UI context
public async Task<bool> ValidateAsync(LibraryBook libraryBook)
// ValidateAsync() doesn't need UI context
public async Task<bool> ValidateAsync(LibraryBook libraryBook)
=> await validateAsync_ConfigureAwaitFalse(libraryBook.Book.AudibleProductId).ConfigureAwait(false);
private async Task<bool> validateAsync_ConfigureAwaitFalse(string productId)
=> !await AudibleFileStorage.Audio.ExistsAsync(productId);
@ -33,22 +35,42 @@ namespace FileLiberator
// often does a lot with forms in the UI context
public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook)
{
var displayMessage = $"[{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}";
var productId = libraryBook.Book.AudibleProductId;
var displayMessage = $"[{productId}] {libraryBook.Book.Title}";
Begin?.Invoke(this, displayMessage);
try
{
var aaxExists = await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId);
if (!aaxExists)
await Download.ProcessAsync(libraryBook);
{
{
var statusHandler = await processAsync(libraryBook, AudibleFileStorage.AAX, DownloadBook);
if (statusHandler.Any())
return statusHandler;
}
return await Decrypt.ProcessAsync(libraryBook);
}
finally
{
var statusHandler = await processAsync(libraryBook, AudibleFileStorage.Audio, DecryptBook);
if (statusHandler.Any())
return statusHandler;
}
{
var statusHandler = await processAsync(libraryBook, AudibleFileStorage.PDF, DownloadPdf);
if (statusHandler.Any())
return statusHandler;
}
return new StatusHandler();
}
finally
{
Completed?.Invoke(this, displayMessage);
}
}
}
private static async Task<StatusHandler> processAsync(LibraryBook libraryBook, AudibleFileStorage afs, IProcessable processable)
=> !await afs.ExistsAsync(libraryBook.Book.AudibleProductId)
? await processable.ProcessAsync(libraryBook)
: new StatusHandler();
}
}

2
FileLiberator/UNTESTED/DecryptBook.cs
View File

@ -12,7 +12,7 @@ using FileManager;
namespace FileLiberator
{
/// <summary>
/// Download DRM book and decrypt audiobook files.
/// Decrypt audiobook files
///
/// Processes:
/// Download: download aax file: the DRM encrypted audiobook

4
FileLiberator/UNTESTED/DownloadBook.cs
View File

@ -1,14 +1,14 @@
using System;
using System.IO;
using System.Threading.Tasks;
using FileManager;
using DataLayer;
using Dinah.Core.ErrorHandling;
using FileManager;
namespace FileLiberator
{
/// <summary>
/// Download DRM book and decrypt audiobook files.
/// Download DRM book
///
/// Processes:
/// Download: download aax file: the DRM encrypted audiobook

99
FileLiberator/UNTESTED/DownloadPdf.cs
View File

@ -9,7 +9,7 @@ using FileManager;
namespace FileLiberator
{
public class DownloadPdf : DownloadableBase
public class DownloadPdf : DownloadableBase
{
static DownloadPdf()
{
@ -18,71 +18,68 @@ namespace FileLiberator
}
public override async Task<bool> ValidateAsync(LibraryBook libraryBook)
{
var product = libraryBook.Book;
{
var product = libraryBook.Book;
if (!product.Supplements.Any())
return false;
if (!product.Supplements.Any())
return false;
return !await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);
}
return !await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);
}
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
{
var product = libraryBook.Book;
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
{
var product = libraryBook.Book;
if (product == null)
return new StatusHandler { "Book not found" };
if (product == null)
return new StatusHandler { "Book not found" };
var urls = product.Supplements.Select(d => d.Url).ToList();
if (urls.Count == 0)
return new StatusHandler { "PDF download url not found" };
var urls = product.Supplements.Select(d => d.Url).ToList();
if (urls.Count == 0)
return new StatusHandler { "PDF download url not found" };
// sanity check
if (urls.Count > 1)
throw new Exception("Multiple PDF downloads are not currently supported. typically indicates an error");
// sanity check
if (urls.Count > 1)
throw new Exception("Multiple PDF downloads are not currently supported. Typically indicates an error");
var url = urls.Single();
var destinationDir = await getDestinationDirectoryAsync(product.AudibleProductId);
if (destinationDir == null)
return new StatusHandler { "Destination directory not found for PDF download" };
var destinationDir = await getDestinationDirectory(product.AudibleProductId);
if (destinationDir == null)
return new StatusHandler { "Destination directory not found for PDF download" };
var url = urls.Single();
var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url));
await performDownloadAsync(url, destinationFilename);
var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url));
var statusHandler = new StatusHandler();
var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);
if (!exists)
statusHandler.AddError("Downloaded PDF cannot be found");
return statusHandler;
}
using var webClient = GetWebClient(destinationFilename);
await webClient.DownloadFileTaskAsync(url, destinationFilename);
private async Task<string> getDestinationDirectoryAsync(string productId)
{
// if audio file exists, get it's dir
var audioFile = await AudibleFileStorage.Audio.GetAsync(productId);
if (audioFile != null)
return Path.GetDirectoryName(audioFile);
var statusHandler = new StatusHandler();
var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);
if (!exists)
statusHandler.AddError("Downloaded PDF cannot be found");
return statusHandler;
}
private async Task<string> getDestinationDirectory(string productId)
{
// if audio file exists, get it's dir
var audioFile = await AudibleFileStorage.Audio.GetAsync(productId);
if (audioFile != null)
return Path.GetDirectoryName(audioFile);
// else return base Book dir
return AudibleFileStorage.PDF.StorageDirectory;
}
// else return base Book dir
return AudibleFileStorage.PDF.StorageDirectory;
}
// other user agents from my chrome. from: https://www.whoishostingthis.com/tools/user-agent/
private static string[] userAgents { get; } = new[]
{
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
};
private WebClient GetWebClient(string downloadMessage)
private async Task performDownloadAsync(string url, string destinationFilename)
{
var webClient = new WebClient();
using var webClient = new WebClient();
var userAgentIndex = new Random().Next(0, userAgents.Length); // upper bound is exclusive
webClient.Headers["User-Agent"] = userAgents[userAgentIndex];
@ -93,13 +90,13 @@ namespace FileLiberator
webClient.Headers["Accept-Language"] = "en-US,en;q=0.9";
webClient.DownloadProgressChanged += (s, e) => Invoke_DownloadProgressChanged(s, new Dinah.Core.Net.Http.DownloadProgress { BytesReceived = e.BytesReceived, ProgressPercentage = e.ProgressPercentage, TotalBytesToReceive = e.TotalBytesToReceive });
webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
Invoke_DownloadBegin(downloadMessage);
Invoke_DownloadBegin(destinationFilename);
return webClient;
await webClient.DownloadFileTaskAsync(url, destinationFilename);
}
}
}

4
FileManager/UNTESTED/AudibleFileStorage.cs
View File

@ -29,8 +29,10 @@ namespace FileManager
[".aac"] = FileType.Audio,
[".mp4"] = FileType.Audio,
[".m4a"] = FileType.Audio,
[".ogg"] = FileType.Audio,
[".flac"] = FileType.Audio,
[".aax"] = FileType.AAX,
[".aax"] = FileType.AAX,
[".pdf"] = FileType.PDF,
[".zip"] = FileType.PDF,

8
LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs
View File

@ -23,8 +23,8 @@ namespace LibationWinForm.BookLiberation
return;
var backupBook = new BackupBook();
backupBook.Download.Completed += SetBackupCountsAsync;
backupBook.Decrypt.Completed += SetBackupCountsAsync;
backupBook.DownloadBook.Completed += SetBackupCountsAsync;
backupBook.DecryptBook.Completed += SetBackupCountsAsync;
await ProcessValidateLibraryBookAsync(backupBook, libraryBook);
}
@ -55,8 +55,8 @@ namespace LibationWinForm.BookLiberation
async Task BackupFirstBookAsync()
{
var backupBook = ProcessorAutomationController.GetWiredUpBackupBook();
backupBook.Download.Completed += SetBackupCountsAsync;
backupBook.Decrypt.Completed += SetBackupCountsAsync;
backupBook.DownloadBook.Completed += SetBackupCountsAsync;
backupBook.DecryptBook.Completed += SetBackupCountsAsync;
await backupBook.ProcessFirstValidAsync();
}

62
LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.cs
View File

@ -16,10 +16,11 @@ namespace LibationWinForm.BookLiberation
{
var backupBook = new BackupBook();
backupBook.Download.Begin += (_, __) => wireUpDownloadable(backupBook.Download);
backupBook.Decrypt.Begin += (_, __) => wireUpDecryptable(backupBook.Decrypt);
backupBook.DownloadBook.Begin += (_, __) => wireUpDownloadable(backupBook.DownloadBook);
backupBook.DecryptBook.Begin += (_, __) => wireUpDecryptable(backupBook.DecryptBook);
backupBook.DownloadPdf.Begin += (_, __) => wireUpDecryptable(backupBook.DecryptBook);
return backupBook;
return backupBook;
}
public static DecryptBook GetWiredUpDecryptBook()
{
@ -191,34 +192,43 @@ namespace LibationWinForm.BookLiberation
#endregion
#region define how model actions will affect form behavior
void downloadBegin(object _, string str) => automatedBackupsForm.AppendText("DownloadStep_Begin: " + str);
void downloadBookBegin(object _, string str) => automatedBackupsForm.AppendText("DownloadStep_Begin: " + str);
void statusUpdate(object _, string str) => automatedBackupsForm.AppendText("- " + str);
void downloadCompleted(object _, string str) => automatedBackupsForm.AppendText("DownloadStep_Completed: " + str);
void decryptBegin(object _, string str) => automatedBackupsForm.AppendText("DecryptStep_Begin: " + str);
void downloadBookCompleted(object _, string str) => automatedBackupsForm.AppendText("DownloadStep_Completed: " + str);
void decryptBookBegin(object _, string str) => automatedBackupsForm.AppendText("DecryptStep_Begin: " + str);
// extra line after book is completely finished
void decryptCompleted(object _, string str) => automatedBackupsForm.AppendText("DecryptStep_Completed: " + str + Environment.NewLine);
#endregion
void decryptBookCompleted(object _, string str) => automatedBackupsForm.AppendText("DecryptStep_Completed: " + str + Environment.NewLine);
void downloadPdfBegin(object _, string str) => automatedBackupsForm.AppendText("PdfStep_Begin: " + str);
// extra line after book is completely finished
void downloadPdfCompleted(object _, string str) => automatedBackupsForm.AppendText("PdfStep_Completed: " + str + Environment.NewLine);
#endregion
#region subscribe new form to model's events
backupBook.Download.Begin += downloadBegin;
backupBook.Download.StatusUpdate += statusUpdate;
backupBook.Download.Completed += downloadCompleted;
backupBook.Decrypt.Begin += decryptBegin;
backupBook.Decrypt.StatusUpdate += statusUpdate;
backupBook.Decrypt.Completed += decryptCompleted;
#endregion
#region subscribe new form to model's events
backupBook.DownloadBook.Begin += downloadBookBegin;
backupBook.DownloadBook.StatusUpdate += statusUpdate;
backupBook.DownloadBook.Completed += downloadBookCompleted;
backupBook.DecryptBook.Begin += decryptBookBegin;
backupBook.DecryptBook.StatusUpdate += statusUpdate;
backupBook.DecryptBook.Completed += decryptBookCompleted;
backupBook.DownloadPdf.Begin += downloadPdfBegin;
backupBook.DownloadPdf.StatusUpdate += statusUpdate;
backupBook.DownloadPdf.Completed += downloadPdfCompleted;
#endregion
#region when form closes, unsubscribe from model's events
// unsubscribe so disposed forms aren't still trying to receive notifications
automatedBackupsForm.FormClosing += (_, __) =>
#region when form closes, unsubscribe from model's events
// unsubscribe so disposed forms aren't still trying to receive notifications
automatedBackupsForm.FormClosing += (_, __) =>
{
backupBook.Download.Begin -= downloadBegin;
backupBook.Download.StatusUpdate -= statusUpdate;
backupBook.Download.Completed -= downloadCompleted;
backupBook.Decrypt.Begin -= decryptBegin;
backupBook.Decrypt.StatusUpdate -= statusUpdate;
backupBook.Decrypt.Completed -= decryptCompleted;
};
backupBook.DownloadBook.Begin -= downloadBookBegin;
backupBook.DownloadBook.StatusUpdate -= statusUpdate;
backupBook.DownloadBook.Completed -= downloadBookCompleted;
backupBook.DecryptBook.Begin -= decryptBookBegin;
backupBook.DecryptBook.StatusUpdate -= statusUpdate;
backupBook.DecryptBook.Completed -= decryptBookCompleted;
backupBook.DownloadPdf.Begin -= downloadPdfBegin;
backupBook.DownloadPdf.StatusUpdate -= statusUpdate;
backupBook.DownloadPdf.Completed -= downloadPdfCompleted;
};
#endregion
await runBackupLoop(backupBook, automatedBackupsForm);

7
LibationWinForm/UNTESTED/Form1.cs
View File

@ -281,9 +281,10 @@ namespace LibationWinForm
private async void beginBookBackupsToolStripMenuItem_Click(object sender, EventArgs e)
{
var backupBook = BookLiberation.ProcessorAutomationController.GetWiredUpBackupBook();
backupBook.Download.Completed += setBackupCountsAsync;
backupBook.Decrypt.Completed += setBackupCountsAsync;
await BookLiberation.ProcessorAutomationController.RunAutomaticBackup(backupBook);
backupBook.DownloadBook.Completed += setBackupCountsAsync;
backupBook.DecryptBook.Completed += setBackupCountsAsync;
backupBook.DownloadPdf.Completed += setBackupCountsAsync;
await BookLiberation.ProcessorAutomationController.RunAutomaticBackup(backupBook);
}
private async void beginPdfBackupsToolStripMenuItem_Click(object sender, EventArgs e)

21
__TODO.txt
View File

@ -1,11 +1,24 @@
-- begin BETA ---------------------------------------------------------------------------------------------------------------------
throttle needed for downloading pdf.s
picture storage should be more responsive if on disk?
download now incl pdf
pdf menu should be 'PDFs only'
update wording in menu
update readme
no mdf,ldf files created in C:\Users\[username]
Warn of known performance issues
- Library import
incl image d/l. need to throttle
- Tag add/edit
- Grid is slow to respond loading when books aren't liberated
- get decrypt key -- unavoidable
@ -16,14 +29,16 @@ https://dotnetcoretutorials.com/2019/06/20/publishing-a-single-exe-file-in-net-c
-- begin ENHANCEMENT, IMPORT UI ---------------------------------------------------------------------------------------------------------------------
scan library in background?
can include a notice somewhere that it's in-process
can include a notice somewhere that a scan is in-process
why block the UI at all?
what to do if new books? don't want to refresh grid when user isn't expecting it
-- end ENHANCEMENT, IMPORT UI ---------------------------------------------------------------------------------------------------------------------
-- begin BUG, FILE DOWNLOAD ---------------------------------------------------------------------------------------------------------------------
reproduce: try to do the api download with a bad codec
result: DownloadsFinal dir .aax file 1 kb
this resulted from an exception. we should not be keeping a file after exception
if error: show error. DownloadBook delete bad file
-- end BUG, FILE DOWNLOAD ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, PERFORMANCE: IMPORT ---------------------------------------------------------------------------------------------------------------------
@ -31,7 +46,7 @@ imports are PAINFULLY slow for just a few hundred items. wtf is taking so long?
-- end ENHANCEMENT, PERFORMANCE: IMPORT ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
when a book/pdf is NOT liberated, calculating the grid's [Liberated][NOT d/l'ed] label is very slow
when a book/pdf is NOT liberated, calculating the grid's [Liberated][NOT d/l'ed] label is very slow. use something similar to PictureStorage's timer to run on a separate thread
https://stackoverflow.com/a/12046333
https://codereview.stackexchange.com/a/135074
// do NOT use lock() or Monitor with async/await