From d53a617bc85e6dd3ab3760a0e88e1b537d82fdae Mon Sep 17 00:00:00 2001 From: Robert McRackan Date: Fri, 15 Nov 2019 12:50:00 -0500 Subject: [PATCH] Download logic in DownloadPdf should look more like DownloadBook. Extract common d/l pattern to base class --- FileLiberator/UNTESTED/BackupBook.cs | 12 +-- FileLiberator/UNTESTED/DecryptBook.cs | 13 ++- FileLiberator/UNTESTED/DownloadBook.cs | 63 ++++++-------- FileLiberator/UNTESTED/DownloadPdf.cs | 99 ++++++---------------- FileLiberator/UNTESTED/DownloadableBase.cs | 38 ++++++--- __TODO.txt | 3 +- 6 files changed, 94 insertions(+), 134 deletions(-) diff --git a/FileLiberator/UNTESTED/BackupBook.cs b/FileLiberator/UNTESTED/BackupBook.cs index 4833627f..e61dfc68 100644 --- a/FileLiberator/UNTESTED/BackupBook.cs +++ b/FileLiberator/UNTESTED/BackupBook.cs @@ -31,9 +31,9 @@ namespace FileLiberator private async Task validateAsync_ConfigureAwaitFalse(string productId) => !await AudibleFileStorage.Audio.ExistsAsync(productId); - // do NOT use ConfigureAwait(false) on ProcessUnregistered() - // often does a lot with forms in the UI context - public async Task ProcessAsync(LibraryBook libraryBook) + // do NOT use ConfigureAwait(false) on ProcessAsync() + // often calls events which prints to forms in the UI context + public async Task ProcessAsync(LibraryBook libraryBook) { var productId = libraryBook.Book.AudibleProductId; var displayMessage = $"[{productId}] {libraryBook.Book.Title}"; @@ -44,19 +44,19 @@ namespace FileLiberator { { var statusHandler = await processAsync(libraryBook, AudibleFileStorage.AAX, DownloadBook); - if (statusHandler.Any()) + if (statusHandler.HasErrors) return statusHandler; } { var statusHandler = await processAsync(libraryBook, AudibleFileStorage.Audio, DecryptBook); - if (statusHandler.Any()) + if (statusHandler.HasErrors) return statusHandler; } { var statusHandler = await processAsync(libraryBook, AudibleFileStorage.PDF, DownloadPdf); - if (statusHandler.Any()) + if (statusHandler.HasErrors) return statusHandler; } diff --git a/FileLiberator/UNTESTED/DecryptBook.cs b/FileLiberator/UNTESTED/DecryptBook.cs index b599790d..74ca38b3 100644 --- a/FileLiberator/UNTESTED/DecryptBook.cs +++ b/FileLiberator/UNTESTED/DecryptBook.cs @@ -41,9 +41,9 @@ namespace FileLiberator => await AudibleFileStorage.AAX.ExistsAsync(productId) && !await AudibleFileStorage.Audio.ExistsAsync(productId); - // do NOT use ConfigureAwait(false) on ProcessUnregistered() - // often does a lot with forms in the UI context - public async Task ProcessAsync(LibraryBook libraryBook) + // do NOT use ConfigureAwait(false) on ProcessAsync() + // often calls events which prints to forms in the UI context + public async Task ProcessAsync(LibraryBook libraryBook) { var displayMessage = $"[{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}"; @@ -60,11 +60,8 @@ namespace FileLiberator if (await AudibleFileStorage.Audio.ExistsAsync(libraryBook.Book.AudibleProductId)) return new StatusHandler { "Cannot find decrypt. Final audio file already exists" }; - string proposedOutputFile = Path.Combine(AudibleFileStorage.DecryptInProgress, $"[{libraryBook.Book.AudibleProductId}].m4b"); - - string outputAudioFilename; - //outputAudioFilename = await inAudibleDecrypt(proposedOutputFile, aaxFilename); - outputAudioFilename = await aaxToM4bConverterDecrypt(proposedOutputFile, aaxFilename); + var proposedOutputFile = Path.Combine(AudibleFileStorage.DecryptInProgress, $"[{libraryBook.Book.AudibleProductId}].m4b"); + var outputAudioFilename = await aaxToM4bConverterDecrypt(proposedOutputFile, aaxFilename); // decrypt failed if (outputAudioFilename == null) diff --git a/FileLiberator/UNTESTED/DownloadBook.cs b/FileLiberator/UNTESTED/DownloadBook.cs index 3ee189a4..52b54583 100644 --- a/FileLiberator/UNTESTED/DownloadBook.cs +++ b/FileLiberator/UNTESTED/DownloadBook.cs @@ -21,55 +21,46 @@ namespace FileLiberator => !await AudibleFileStorage.Audio.ExistsAsync(libraryBook.Book.AudibleProductId) && !await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId); - public override async Task ProcessItemAsync(LibraryBook libraryBook) + public override async Task ProcessItemAsync(LibraryBook libraryBook) { - var tempAaxFilename = FileUtility.GetValidFilename( + var tempAaxFilename = getDownloadPath(libraryBook); + var actualFilePath = await downloadBookAsync(libraryBook, tempAaxFilename); + moveBook(libraryBook, actualFilePath); + return await verifyDownloadAsync(libraryBook); + } + + private static string getDownloadPath(LibraryBook libraryBook) + => FileUtility.GetValidFilename( AudibleFileStorage.DownloadsInProgress, libraryBook.Book.Title, "aax", libraryBook.Book.AudibleProductId); - // if getting from full title: - // '?' is allowed - // colons are inconsistent but not problematic to just leave them - // - 1 colon: sometimes full title is used. sometimes only the part before the colon is used - // - multple colons: only the part before the final colon is used - // e.g. Alien: Out of the Shadows: An Audible Original Drama => Alien: Out of the Shadows - // in cases where title includes '&', just use everything before the '&' and ignore the rest - //// var adhTitle = product.Title.Split('&')[0] + private async Task downloadBookAsync(LibraryBook libraryBook, string tempAaxFilename) + { + var api = await AudibleApi.EzApiCreator.GetApiAsync(AudibleApiStorage.IdentityTokensFile); - // new/api method - tempAaxFilename = await performApiDownloadAsync(libraryBook, tempAaxFilename); + var actualFilePath = await PerformDownloadAsync( + tempAaxFilename, + (p) => api.DownloadAaxWorkaroundAsync(libraryBook.Book.AudibleProductId, tempAaxFilename, p)); - // move - var aaxFilename = FileUtility.GetValidFilename( + return actualFilePath; + } + + private void moveBook(LibraryBook libraryBook, string actualFilePath) + { + var newAaxFilename = FileUtility.GetValidFilename( AudibleFileStorage.DownloadsFinal, libraryBook.Book.Title, "aax", libraryBook.Book.AudibleProductId); - File.Move(tempAaxFilename, aaxFilename); - - var statusHandler = new StatusHandler(); - var isDownloaded = await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId); - if (isDownloaded) - Invoke_StatusUpdate($"Downloaded: {aaxFilename}"); - else - statusHandler.AddError("Downloaded AAX file cannot be found"); - return statusHandler; + File.Move(actualFilePath, newAaxFilename); + Invoke_StatusUpdate($"Successfully downloaded. Moved to: {newAaxFilename}"); } - private async Task performApiDownloadAsync(LibraryBook libraryBook, string tempAaxFilename) - { - var api = await AudibleApi.EzApiCreator.GetApiAsync(AudibleApiStorage.IdentityTokensFile); - - var progress = new Progress(); - progress.ProgressChanged += (_, e) => Invoke_DownloadProgressChanged(this, e); - - Invoke_DownloadBegin(tempAaxFilename); - var actualFilePath = await api.DownloadAaxWorkaroundAsync(libraryBook.Book.AudibleProductId, tempAaxFilename, progress); - Invoke_DownloadCompleted(this, $"Completed: {actualFilePath}"); - - return actualFilePath; - } + private static async Task verifyDownloadAsync(LibraryBook libraryBook) + => !await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId) + ? new StatusHandler { "Downloaded AAX file cannot be found" } + : new StatusHandler(); } } diff --git a/FileLiberator/UNTESTED/DownloadPdf.cs b/FileLiberator/UNTESTED/DownloadPdf.cs index c726a65a..83eeadbc 100644 --- a/FileLiberator/UNTESTED/DownloadPdf.cs +++ b/FileLiberator/UNTESTED/DownloadPdf.cs @@ -1,102 +1,57 @@ using System; using System.IO; using System.Linq; -using System.Net; +using System.Net.Http; using System.Threading.Tasks; using DataLayer; using Dinah.Core.ErrorHandling; +using Dinah.Core.Net.Http; using FileManager; namespace FileLiberator { public class DownloadPdf : DownloadableBase { - static DownloadPdf() - { - // https://stackoverflow.com/a/15483698 - ServicePointManager.ServerCertificateValidationCallback = delegate { return true; }; - } - public override async Task ValidateAsync(LibraryBook libraryBook) { - var product = libraryBook.Book; - - if (!product.Supplements.Any()) + if (string.IsNullOrWhiteSpace(getdownloadUrl(libraryBook))) return false; - return !await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId); + return !await AudibleFileStorage.PDF.ExistsAsync(libraryBook.Book.AudibleProductId); } + private static string getdownloadUrl(LibraryBook libraryBook) + => libraryBook?.Book?.Supplements?.FirstOrDefault()?.Url; + public override async Task ProcessItemAsync(LibraryBook libraryBook) { - var product = libraryBook.Book; - - if (product == null) - return new StatusHandler { "Book not found" }; - - var urls = product.Supplements.Select(d => d.Url).ToList(); - if (urls.Count == 0) - return new StatusHandler { "PDF download url not found" }; - - // sanity check - if (urls.Count > 1) - throw new Exception("Multiple PDF downloads are not currently supported. Typically indicates an error"); - - var destinationDir = await getDestinationDirectoryAsync(product.AudibleProductId); - if (destinationDir == null) - return new StatusHandler { "Destination directory not found for PDF download" }; - - var url = urls.Single(); - var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url)); - await performDownloadAsync(url, destinationFilename); - - var statusHandler = new StatusHandler(); - var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId); - if (!exists) - statusHandler.AddError("Downloaded PDF cannot be found"); - return statusHandler; + var proposedDownloadFilePath = await getProposedDownloadFilePathAsync(libraryBook); + await downloadPdfAsync(libraryBook, proposedDownloadFilePath); + return await verifyDownloadAsync(libraryBook); } - private async Task getDestinationDirectoryAsync(string productId) + private static async Task getProposedDownloadFilePathAsync(LibraryBook libraryBook) { - // if audio file exists, get it's dir - var audioFile = await AudibleFileStorage.Audio.GetAsync(productId); - if (audioFile != null) - return Path.GetDirectoryName(audioFile); + // if audio file exists, get it's dir. else return base Book dir + var destinationDir = + // this is safe b/c GetDirectoryName(null) == null + Path.GetDirectoryName(await AudibleFileStorage.Audio.GetAsync(libraryBook.Book.AudibleProductId)) + ?? AudibleFileStorage.PDF.StorageDirectory; - // else return base Book dir - return AudibleFileStorage.PDF.StorageDirectory; + return Path.Combine(destinationDir, Path.GetFileName(getdownloadUrl(libraryBook))); } - // other user agents from my chrome. from: https://www.whoishostingthis.com/tools/user-agent/ - private static string[] userAgents { get; } = new[] + private async Task downloadPdfAsync(LibraryBook libraryBook, string proposedDownloadFilePath) { - "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36", - }; - - private async Task performDownloadAsync(string url, string destinationFilename) - { - using var webClient = new WebClient(); - - var userAgentIndex = new Random().Next(0, userAgents.Length); // upper bound is exclusive - webClient.Headers["User-Agent"] = userAgents[userAgentIndex]; - webClient.Headers["Referer"] = "https://google.com"; - webClient.Headers["Upgrade-Insecure-Requests"] = "1"; - webClient.Headers["DNT"] = "1"; - webClient.Headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"; - webClient.Headers["Accept-Language"] = "en-US,en;q=0.9"; - - webClient.DownloadProgressChanged += (s, e) => Invoke_DownloadProgressChanged(s, new Dinah.Core.Net.Http.DownloadProgress { BytesReceived = e.BytesReceived, ProgressPercentage = e.ProgressPercentage, TotalBytesToReceive = e.TotalBytesToReceive }); - webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}"); - webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}"); - webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}"); - - Invoke_DownloadBegin(destinationFilename); - - await webClient.DownloadFileTaskAsync(url, destinationFilename); + var client = new HttpClient(); + var actualDownloadedFilePath = await PerformDownloadAsync( + proposedDownloadFilePath, + (p) => client.DownloadFileAsync(getdownloadUrl(libraryBook), proposedDownloadFilePath, p)); } + + private static async Task verifyDownloadAsync(LibraryBook libraryBook) + => !await AudibleFileStorage.PDF.ExistsAsync(libraryBook.Book.AudibleProductId) + ? new StatusHandler { "Downloaded PDF cannot be found" } + : new StatusHandler(); } } diff --git a/FileLiberator/UNTESTED/DownloadableBase.cs b/FileLiberator/UNTESTED/DownloadableBase.cs index 02fd75f4..d0055537 100644 --- a/FileLiberator/UNTESTED/DownloadableBase.cs +++ b/FileLiberator/UNTESTED/DownloadableBase.cs @@ -2,6 +2,7 @@ using System.Threading.Tasks; using DataLayer; using Dinah.Core.ErrorHandling; +using Dinah.Core.Net.Http; namespace FileLiberator { @@ -10,23 +11,20 @@ namespace FileLiberator public event EventHandler Begin; public event EventHandler Completed; - public event EventHandler StatusUpdate; public event EventHandler DownloadBegin; - public event EventHandler DownloadProgressChanged; + public event EventHandler DownloadProgressChanged; public event EventHandler DownloadCompleted; - protected void Invoke_StatusUpdate(string message) => StatusUpdate?.Invoke(this, message); - protected void Invoke_DownloadBegin(string downloadMessage) => DownloadBegin?.Invoke(this, downloadMessage); - protected void Invoke_DownloadProgressChanged(object sender, Dinah.Core.Net.Http.DownloadProgress progress) => DownloadProgressChanged?.Invoke(sender, progress); - protected void Invoke_DownloadCompleted(object sender, string str) => DownloadCompleted?.Invoke(sender, str); + public event EventHandler StatusUpdate; + protected void Invoke_StatusUpdate(string message) => StatusUpdate?.Invoke(this, message); public abstract Task ValidateAsync(LibraryBook libraryBook); public abstract Task ProcessItemAsync(LibraryBook libraryBook); - // do NOT use ConfigureAwait(false) on ProcessUnregistered() - // often does a lot with forms in the UI context - public async Task ProcessAsync(LibraryBook libraryBook) + // do NOT use ConfigureAwait(false) on ProcessAsync() + // often calls events which prints to forms in the UI context + public async Task ProcessAsync(LibraryBook libraryBook) { var displayMessage = $"[{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}"; @@ -41,5 +39,25 @@ namespace FileLiberator Completed?.Invoke(this, displayMessage); } } - } + + protected async Task PerformDownloadAsync(string proposedDownloadFilePath, Func, Task> func) + { + var progress = new Progress(); + progress.ProgressChanged += (_, e) => DownloadProgressChanged?.Invoke(this, e); + + DownloadBegin?.Invoke(this, proposedDownloadFilePath); + + try + { + var result = await func(progress); + StatusUpdate?.Invoke(this, result); + + return result; + } + finally + { + DownloadCompleted?.Invoke(this, proposedDownloadFilePath); + } + } + } } diff --git a/__TODO.txt b/__TODO.txt index 5a6a56c8..da696e40 100644 --- a/__TODO.txt +++ b/__TODO.txt @@ -1,6 +1,5 @@ -- begin BETA --------------------------------------------------------------------------------------------------------------------- -OLD WEB DOWNLOADER -download logic in DownloadPdf should look more like DownloadBook +Replace StatusHandler with CSharpFunctionalExtensions TESTING BUG dbl clk. long pause. exception: