tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Download logic in DownloadPdf should look more like DownloadBook. Extract common d/l pattern to base class

Browse Source
This commit is contained in:
Robert McRackan 2019-11-15 12:50:00 -05:00
parent 9076fae6f6
commit d53a617bc8
6 changed files with 94 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
FileLiberator/UNTESTED/BackupBook.cs
View File

@ -31,9 +31,9 @@ namespace FileLiberator
private async Task<bool> validateAsync_ConfigureAwaitFalse(string productId)
=> !await AudibleFileStorage.Audio.ExistsAsync(productId);
// do NOT use ConfigureAwait(false) on ProcessUnregistered()
// often does a lot with forms in the UI context
public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook)
// do NOT use ConfigureAwait(false) on ProcessAsync()
// often calls events which prints to forms in the UI context
public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook)
{
var productId = libraryBook.Book.AudibleProductId;
var displayMessage = $"[{productId}] {libraryBook.Book.Title}";
@ -44,19 +44,19 @@ namespace FileLiberator
{
{
var statusHandler = await processAsync(libraryBook, AudibleFileStorage.AAX, DownloadBook);
if (statusHandler.Any())
if (statusHandler.HasErrors)
return statusHandler;
}
{
var statusHandler = await processAsync(libraryBook, AudibleFileStorage.Audio, DecryptBook);
if (statusHandler.Any())
if (statusHandler.HasErrors)
return statusHandler;
}
{
var statusHandler = await processAsync(libraryBook, AudibleFileStorage.PDF, DownloadPdf);
if (statusHandler.Any())
if (statusHandler.HasErrors)
return statusHandler;
}

13
FileLiberator/UNTESTED/DecryptBook.cs
View File

@ -41,9 +41,9 @@ namespace FileLiberator
=> await AudibleFileStorage.AAX.ExistsAsync(productId)
&& !await AudibleFileStorage.Audio.ExistsAsync(productId);
// do NOT use ConfigureAwait(false) on ProcessUnregistered()
// often does a lot with forms in the UI context
public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook)
// do NOT use ConfigureAwait(false) on ProcessAsync()
// often calls events which prints to forms in the UI context
public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook)
{
var displayMessage = $"[{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}";
@ -60,11 +60,8 @@ namespace FileLiberator
if (await AudibleFileStorage.Audio.ExistsAsync(libraryBook.Book.AudibleProductId))
return new StatusHandler { "Cannot find decrypt. Final audio file already exists" };
string proposedOutputFile = Path.Combine(AudibleFileStorage.DecryptInProgress, $"[{libraryBook.Book.AudibleProductId}].m4b");
string outputAudioFilename;
//outputAudioFilename = await inAudibleDecrypt(proposedOutputFile, aaxFilename);
outputAudioFilename = await aaxToM4bConverterDecrypt(proposedOutputFile, aaxFilename);
var proposedOutputFile = Path.Combine(AudibleFileStorage.DecryptInProgress, $"[{libraryBook.Book.AudibleProductId}].m4b");
var outputAudioFilename = await aaxToM4bConverterDecrypt(proposedOutputFile, aaxFilename);
// decrypt failed
if (outputAudioFilename == null)

63
FileLiberator/UNTESTED/DownloadBook.cs
View File

@ -21,55 +21,46 @@ namespace FileLiberator
=> !await AudibleFileStorage.Audio.ExistsAsync(libraryBook.Book.AudibleProductId)
&& !await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId);
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
{
var tempAaxFilename = FileUtility.GetValidFilename(
var tempAaxFilename = getDownloadPath(libraryBook);
var actualFilePath = await downloadBookAsync(libraryBook, tempAaxFilename);
moveBook(libraryBook, actualFilePath);
return await verifyDownloadAsync(libraryBook);
}
private static string getDownloadPath(LibraryBook libraryBook)
=> FileUtility.GetValidFilename(
AudibleFileStorage.DownloadsInProgress,
libraryBook.Book.Title,
"aax",
libraryBook.Book.AudibleProductId);
// if getting from full title:
// '?' is allowed
// colons are inconsistent but not problematic to just leave them
// - 1 colon: sometimes full title is used. sometimes only the part before the colon is used
// - multple colons: only the part before the final colon is used
// e.g. Alien: Out of the Shadows: An Audible Original Drama => Alien: Out of the Shadows
// in cases where title includes '&', just use everything before the '&' and ignore the rest
//// var adhTitle = product.Title.Split('&')[0]
private async Task<string> downloadBookAsync(LibraryBook libraryBook, string tempAaxFilename)
{
var api = await AudibleApi.EzApiCreator.GetApiAsync(AudibleApiStorage.IdentityTokensFile);
// new/api method
tempAaxFilename = await performApiDownloadAsync(libraryBook, tempAaxFilename);
var actualFilePath = await PerformDownloadAsync(
tempAaxFilename,
(p) => api.DownloadAaxWorkaroundAsync(libraryBook.Book.AudibleProductId, tempAaxFilename, p));
// move
var aaxFilename = FileUtility.GetValidFilename(
return actualFilePath;
}
private void moveBook(LibraryBook libraryBook, string actualFilePath)
{
var newAaxFilename = FileUtility.GetValidFilename(
AudibleFileStorage.DownloadsFinal,
libraryBook.Book.Title,
"aax",
libraryBook.Book.AudibleProductId);
File.Move(tempAaxFilename, aaxFilename);
var statusHandler = new StatusHandler();
var isDownloaded = await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId);
if (isDownloaded)
Invoke_StatusUpdate($"Downloaded: {aaxFilename}");
else
statusHandler.AddError("Downloaded AAX file cannot be found");
return statusHandler;
File.Move(actualFilePath, newAaxFilename);
Invoke_StatusUpdate($"Successfully downloaded. Moved to: {newAaxFilename}");
}
private async Task<string> performApiDownloadAsync(LibraryBook libraryBook, string tempAaxFilename)
{
var api = await AudibleApi.EzApiCreator.GetApiAsync(AudibleApiStorage.IdentityTokensFile);
var progress = new Progress<Dinah.Core.Net.Http.DownloadProgress>();
progress.ProgressChanged += (_, e) => Invoke_DownloadProgressChanged(this, e);
Invoke_DownloadBegin(tempAaxFilename);
var actualFilePath = await api.DownloadAaxWorkaroundAsync(libraryBook.Book.AudibleProductId, tempAaxFilename, progress);
Invoke_DownloadCompleted(this, $"Completed: {actualFilePath}");
return actualFilePath;
}
private static async Task<StatusHandler> verifyDownloadAsync(LibraryBook libraryBook)
=> !await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId)
? new StatusHandler { "Downloaded AAX file cannot be found" }
: new StatusHandler();
}
}

99
FileLiberator/UNTESTED/DownloadPdf.cs
View File

@ -1,102 +1,57 @@
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading.Tasks;
using DataLayer;
using Dinah.Core.ErrorHandling;
using Dinah.Core.Net.Http;
using FileManager;
namespace FileLiberator
{
public class DownloadPdf : DownloadableBase
{
static DownloadPdf()
{
// https://stackoverflow.com/a/15483698
ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
}
public override async Task<bool> ValidateAsync(LibraryBook libraryBook)
{
var product = libraryBook.Book;
if (!product.Supplements.Any())
if (string.IsNullOrWhiteSpace(getdownloadUrl(libraryBook)))
return false;
return !await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);
return !await AudibleFileStorage.PDF.ExistsAsync(libraryBook.Book.AudibleProductId);
}
private static string getdownloadUrl(LibraryBook libraryBook)
=> libraryBook?.Book?.Supplements?.FirstOrDefault()?.Url;
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
{
var product = libraryBook.Book;
if (product == null)
return new StatusHandler { "Book not found" };
var urls = product.Supplements.Select(d => d.Url).ToList();
if (urls.Count == 0)
return new StatusHandler { "PDF download url not found" };
// sanity check
if (urls.Count > 1)
throw new Exception("Multiple PDF downloads are not currently supported. Typically indicates an error");
var destinationDir = await getDestinationDirectoryAsync(product.AudibleProductId);
if (destinationDir == null)
return new StatusHandler { "Destination directory not found for PDF download" };
var url = urls.Single();
var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url));
await performDownloadAsync(url, destinationFilename);
var statusHandler = new StatusHandler();
var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);
if (!exists)
statusHandler.AddError("Downloaded PDF cannot be found");
return statusHandler;
var proposedDownloadFilePath = await getProposedDownloadFilePathAsync(libraryBook);
await downloadPdfAsync(libraryBook, proposedDownloadFilePath);
return await verifyDownloadAsync(libraryBook);
}
private async Task<string> getDestinationDirectoryAsync(string productId)
private static async Task<string> getProposedDownloadFilePathAsync(LibraryBook libraryBook)
{
// if audio file exists, get it's dir
var audioFile = await AudibleFileStorage.Audio.GetAsync(productId);
if (audioFile != null)
return Path.GetDirectoryName(audioFile);
// if audio file exists, get it's dir. else return base Book dir
var destinationDir =
// this is safe b/c GetDirectoryName(null) == null
Path.GetDirectoryName(await AudibleFileStorage.Audio.GetAsync(libraryBook.Book.AudibleProductId))
?? AudibleFileStorage.PDF.StorageDirectory;
// else return base Book dir
return AudibleFileStorage.PDF.StorageDirectory;
return Path.Combine(destinationDir, Path.GetFileName(getdownloadUrl(libraryBook)));
}
// other user agents from my chrome. from: https://www.whoishostingthis.com/tools/user-agent/
private static string[] userAgents { get; } = new[]
private async Task downloadPdfAsync(LibraryBook libraryBook, string proposedDownloadFilePath)
{
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
};
private async Task performDownloadAsync(string url, string destinationFilename)
{
using var webClient = new WebClient();
var userAgentIndex = new Random().Next(0, userAgents.Length); // upper bound is exclusive
webClient.Headers["User-Agent"] = userAgents[userAgentIndex];
webClient.Headers["Referer"] = "https://google.com";
webClient.Headers["Upgrade-Insecure-Requests"] = "1";
webClient.Headers["DNT"] = "1";
webClient.Headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
webClient.Headers["Accept-Language"] = "en-US,en;q=0.9";
webClient.DownloadProgressChanged += (s, e) => Invoke_DownloadProgressChanged(s, new Dinah.Core.Net.Http.DownloadProgress { BytesReceived = e.BytesReceived, ProgressPercentage = e.ProgressPercentage, TotalBytesToReceive = e.TotalBytesToReceive });
webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {destinationFilename}");
Invoke_DownloadBegin(destinationFilename);
await webClient.DownloadFileTaskAsync(url, destinationFilename);
var client = new HttpClient();
var actualDownloadedFilePath = await PerformDownloadAsync(
proposedDownloadFilePath,
(p) => client.DownloadFileAsync(getdownloadUrl(libraryBook), proposedDownloadFilePath, p));
}
private static async Task<StatusHandler> verifyDownloadAsync(LibraryBook libraryBook)
=> !await AudibleFileStorage.PDF.ExistsAsync(libraryBook.Book.AudibleProductId)
? new StatusHandler { "Downloaded PDF cannot be found" }
: new StatusHandler();
}
}

38
FileLiberator/UNTESTED/DownloadableBase.cs
View File

@ -2,6 +2,7 @@
using System.Threading.Tasks;
using DataLayer;
using Dinah.Core.ErrorHandling;
using Dinah.Core.Net.Http;
namespace FileLiberator
{
@ -10,23 +11,20 @@ namespace FileLiberator
public event EventHandler<string> Begin;
public event EventHandler<string> Completed;
public event EventHandler<string> StatusUpdate;
public event EventHandler<string> DownloadBegin;
public event EventHandler<Dinah.Core.Net.Http.DownloadProgress> DownloadProgressChanged;
public event EventHandler<DownloadProgress> DownloadProgressChanged;
public event EventHandler<string> DownloadCompleted;
protected void Invoke_StatusUpdate(string message) => StatusUpdate?.Invoke(this, message);
protected void Invoke_DownloadBegin(string downloadMessage) => DownloadBegin?.Invoke(this, downloadMessage);
protected void Invoke_DownloadProgressChanged(object sender, Dinah.Core.Net.Http.DownloadProgress progress) => DownloadProgressChanged?.Invoke(sender, progress);
protected void Invoke_DownloadCompleted(object sender, string str) => DownloadCompleted?.Invoke(sender, str);
public event EventHandler<string> StatusUpdate;
protected void Invoke_StatusUpdate(string message) => StatusUpdate?.Invoke(this, message);
public abstract Task<bool> ValidateAsync(LibraryBook libraryBook);
public abstract Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook);
// do NOT use ConfigureAwait(false) on ProcessUnregistered()
// often does a lot with forms in the UI context
public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook)
// do NOT use ConfigureAwait(false) on ProcessAsync()
// often calls events which prints to forms in the UI context
public async Task<StatusHandler> ProcessAsync(LibraryBook libraryBook)
{
var displayMessage = $"[{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}";
@ -41,5 +39,25 @@ namespace FileLiberator
Completed?.Invoke(this, displayMessage);
}
}
}
protected async Task<string> PerformDownloadAsync(string proposedDownloadFilePath, Func<Progress<DownloadProgress>, Task<string>> func)
{
var progress = new Progress<DownloadProgress>();
progress.ProgressChanged += (_, e) => DownloadProgressChanged?.Invoke(this, e);
DownloadBegin?.Invoke(this, proposedDownloadFilePath);
try
{
var result = await func(progress);
StatusUpdate?.Invoke(this, result);
return result;
}
finally
{
DownloadCompleted?.Invoke(this, proposedDownloadFilePath);
}
}
}
}

3
__TODO.txt
View File

@ -1,6 +1,5 @@
-- begin BETA ---------------------------------------------------------------------------------------------------------------------
OLD WEB DOWNLOADER
download logic in DownloadPdf should look more like DownloadBook
Replace StatusHandler with CSharpFunctionalExtensions
TESTING BUG
dbl clk. long pause. exception: