Improved performance calculating "liberated" status

This commit is contained in:
Robert McRackan 2019-11-21 23:07:06 -05:00
parent d5d72a13f6
commit 1375da2065
12 changed files with 92 additions and 194 deletions

View File

@ -25,11 +25,8 @@ namespace FileLiberator
public DecryptBook DecryptBook { get; } = new DecryptBook();
public DownloadPdf DownloadPdf { get; } = new DownloadPdf();
// ValidateAsync() doesn't need UI context
public async Task<bool> ValidateAsync(LibraryBook libraryBook)
=> await validateAsync_ConfigureAwaitFalse(libraryBook.Book.AudibleProductId).ConfigureAwait(false);
private async Task<bool> validateAsync_ConfigureAwaitFalse(string productId)
=> !await AudibleFileStorage.Audio.ExistsAsync(productId);
public bool Validate(LibraryBook libraryBook)
=> !AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId);
// do NOT use ConfigureAwait(false) on ProcessAsync()
// often calls events which prints to forms in the UI context

View File

@ -34,12 +34,9 @@ namespace FileLiberator
public event EventHandler<string> DecryptCompleted;
public event EventHandler<string> Completed;
// ValidateAsync() doesn't need UI context
public async Task<bool> ValidateAsync(LibraryBook libraryBook)
=> await validateAsync_ConfigureAwaitFalse(libraryBook.Book.AudibleProductId).ConfigureAwait(false);
private async Task<bool> validateAsync_ConfigureAwaitFalse(string productId)
=> await AudibleFileStorage.AAX.ExistsAsync(productId)
&& !await AudibleFileStorage.Audio.ExistsAsync(productId);
public bool Validate(LibraryBook libraryBook)
=> AudibleFileStorage.AAX.Exists(libraryBook.Book.AudibleProductId)
&& !AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId);
// do NOT use ConfigureAwait(false) on ProcessAsync()
// often calls events which prints to forms in the UI context
@ -51,13 +48,13 @@ namespace FileLiberator
try
{
var aaxFilename = await AudibleFileStorage.AAX.GetAsync(libraryBook.Book.AudibleProductId);
var aaxFilename = AudibleFileStorage.AAX.GetPath(libraryBook.Book.AudibleProductId);
if (aaxFilename == null)
return new StatusHandler { "aaxFilename parameter is null" };
if (!FileUtility.FileExists(aaxFilename))
return new StatusHandler { $"Cannot find AAX file: {aaxFilename}" };
if (await AudibleFileStorage.Audio.ExistsAsync(libraryBook.Book.AudibleProductId))
if (AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId))
return new StatusHandler { "Cannot find decrypt. Final audio file already exists" };
var proposedOutputFile = Path.Combine(AudibleFileStorage.DecryptInProgress, $"[{libraryBook.Book.AudibleProductId}].m4b");
@ -72,7 +69,7 @@ namespace FileLiberator
Dinah.Core.IO.FileExt.SafeDelete(aaxFilename);
var statusHandler = new StatusHandler();
var finalAudioExists = await AudibleFileStorage.Audio.ExistsAsync(libraryBook.Book.AudibleProductId);
var finalAudioExists = AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId);
if (!finalAudioExists)
statusHandler.AddError("Cannot find final audio file after decryption");
return statusHandler;

View File

@ -17,16 +17,16 @@ namespace FileLiberator
/// </summary>
public class DownloadBook : DownloadableBase
{
public override async Task<bool> ValidateAsync(LibraryBook libraryBook)
=> !await AudibleFileStorage.Audio.ExistsAsync(libraryBook.Book.AudibleProductId)
&& !await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId);
public override bool Validate(LibraryBook libraryBook)
=> !AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId)
&& !AudibleFileStorage.AAX.Exists(libraryBook.Book.AudibleProductId);
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
{
var tempAaxFilename = getDownloadPath(libraryBook);
var actualFilePath = await downloadBookAsync(libraryBook, tempAaxFilename);
moveBook(libraryBook, actualFilePath);
return await verifyDownloadAsync(libraryBook);
return verifyDownload(libraryBook);
}
private static string getDownloadPath(LibraryBook libraryBook)
@ -58,8 +58,8 @@ namespace FileLiberator
Invoke_StatusUpdate($"Successfully downloaded. Moved to: {newAaxFilename}");
}
private static async Task<StatusHandler> verifyDownloadAsync(LibraryBook libraryBook)
=> !await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId)
private static StatusHandler verifyDownload(LibraryBook libraryBook)
=> !AudibleFileStorage.AAX.Exists(libraryBook.Book.AudibleProductId)
? new StatusHandler { "Downloaded AAX file cannot be found" }
: new StatusHandler();
}

View File

@ -12,26 +12,26 @@ namespace FileLiberator
{
public class DownloadPdf : DownloadableBase
{
public override async Task<bool> ValidateAsync(LibraryBook libraryBook)
public override bool Validate(LibraryBook libraryBook)
=> !string.IsNullOrWhiteSpace(getdownloadUrl(libraryBook))
&& !await AudibleFileStorage.PDF.ExistsAsync(libraryBook.Book.AudibleProductId);
&& !AudibleFileStorage.PDF.Exists(libraryBook.Book.AudibleProductId);
private static string getdownloadUrl(LibraryBook libraryBook)
=> libraryBook?.Book?.Supplements?.FirstOrDefault()?.Url;
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
{
var proposedDownloadFilePath = await getProposedDownloadFilePathAsync(libraryBook);
var proposedDownloadFilePath = getProposedDownloadFilePath(libraryBook);
await downloadPdfAsync(libraryBook, proposedDownloadFilePath);
return await verifyDownloadAsync(libraryBook);
return verifyDownload(libraryBook);
}
private static async Task<string> getProposedDownloadFilePathAsync(LibraryBook libraryBook)
private static string getProposedDownloadFilePath(LibraryBook libraryBook)
{
// if audio file exists, get it's dir. else return base Book dir
var destinationDir =
// this is safe b/c GetDirectoryName(null) == null
Path.GetDirectoryName(await AudibleFileStorage.Audio.GetAsync(libraryBook.Book.AudibleProductId))
Path.GetDirectoryName(AudibleFileStorage.Audio.GetPath(libraryBook.Book.AudibleProductId))
?? AudibleFileStorage.PDF.StorageDirectory;
return Path.Combine(destinationDir, Path.GetFileName(getdownloadUrl(libraryBook)));
@ -45,8 +45,8 @@ namespace FileLiberator
(p) => client.DownloadFileAsync(getdownloadUrl(libraryBook), proposedDownloadFilePath, p));
}
private static async Task<StatusHandler> verifyDownloadAsync(LibraryBook libraryBook)
=> !await AudibleFileStorage.PDF.ExistsAsync(libraryBook.Book.AudibleProductId)
private static StatusHandler verifyDownload(LibraryBook libraryBook)
=> !AudibleFileStorage.PDF.Exists(libraryBook.Book.AudibleProductId)
? new StatusHandler { "Downloaded PDF cannot be found" }
: new StatusHandler();
}

View File

@ -18,7 +18,7 @@ namespace FileLiberator
public event EventHandler<string> StatusUpdate;
protected void Invoke_StatusUpdate(string message) => StatusUpdate?.Invoke(this, message);
public abstract Task<bool> ValidateAsync(LibraryBook libraryBook);
public abstract bool Validate(LibraryBook libraryBook);
public abstract Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook);

View File

@ -14,8 +14,8 @@ namespace FileLiberator
event EventHandler<string> Completed;
/// <returns>True == Valid</returns>
Task<bool> ValidateAsync(LibraryBook libraryBook);
/// <returns>True == Valid</returns>
bool Validate(LibraryBook libraryBook);
/// <returns>True == success</returns>
Task<StatusHandler> ProcessAsync(LibraryBook libraryBook);

View File

@ -9,8 +9,7 @@ namespace FileLiberator
{
//
// DO NOT USE ConfigureAwait(false) WITH ProcessAsync() unless ensuring ProcessAsync() implementation is cross-thread compatible
// - ValidateAsync() doesn't need UI context. however, each class already uses ConfigureAwait(false)
// - ProcessAsync() often does a lot with forms in the UI context
// ProcessAsync() often does a lot with forms in the UI context
//
@ -18,7 +17,7 @@ namespace FileLiberator
/// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
public static async Task<StatusHandler> ProcessFirstValidAsync(this IProcessable processable)
{
var libraryBook = await processable.GetNextValidAsync();
var libraryBook = processable.GetNextValid();
if (libraryBook == null)
return null;
@ -30,19 +29,19 @@ namespace FileLiberator
return status;
}
public static async Task<LibraryBook> GetNextValidAsync(this IProcessable processable)
public static LibraryBook GetNextValid(this IProcessable processable)
{
var libraryBooks = LibraryQueries.GetLibrary_Flat_NoTracking();
foreach (var libraryBook in libraryBooks)
if (await processable.ValidateAsync(libraryBook))
if (processable.Validate(libraryBook))
return libraryBook;
return null;
}
public static async Task<StatusHandler> TryProcessAsync(this IProcessable processable, LibraryBook libraryBook)
=> await processable.ValidateAsync(libraryBook)
=> processable.Validate(libraryBook)
? await processable.ProcessAsync(libraryBook)
: new StatusHandler();
}

View File

@ -2,7 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Text.RegularExpressions;
using Dinah.Core;
using Dinah.Core.Collections.Generic;
@ -21,23 +21,6 @@ namespace FileManager
public sealed class AudibleFileStorage : Enumeration<AudibleFileStorage>
{
#region static
// centralize filetype mappings to ensure uniqueness
private static Dictionary<string, FileType> extensionMap => new Dictionary<string, FileType>
{
[".m4b"] = FileType.Audio,
[".mp3"] = FileType.Audio,
[".aac"] = FileType.Audio,
[".mp4"] = FileType.Audio,
[".m4a"] = FileType.Audio,
[".ogg"] = FileType.Audio,
[".flac"] = FileType.Audio,
[".aax"] = FileType.AAX,
[".pdf"] = FileType.PDF,
[".zip"] = FileType.PDF,
};
public static AudibleFileStorage Audio { get; }
public static AudibleFileStorage AAX { get; }
public static AudibleFileStorage PDF { get; }
@ -81,9 +64,9 @@ namespace FileManager
// must do this in static ctor, not w/inline properties
// static properties init before static ctor so these dir.s would still be null
Audio = new AudibleFileStorage(FileType.Audio, BooksDirectory);
AAX = new AudibleFileStorage(FileType.AAX, DownloadsFinal);
PDF = new AudibleFileStorage(FileType.PDF, BooksDirectory);
Audio = new AudibleFileStorage(FileType.Audio, BooksDirectory, "m4b", "mp3", "aac", "mp4", "m4a", "ogg", "flac");
AAX = new AudibleFileStorage(FileType.AAX, DownloadsFinal, "aax");
PDF = new AudibleFileStorage(FileType.PDF, BooksDirectory, "pdf", "zip");
}
#endregion
@ -92,9 +75,14 @@ namespace FileManager
public string StorageDirectory => DisplayName;
public IEnumerable<string> Extensions => extensionMap.Where(kvp => kvp.Value == FileType).Select(kvp => kvp.Key);
private IEnumerable<string> extensions_noDots { get; }
private string extAggr { get; }
private AudibleFileStorage(FileType fileType, string storageDirectory) : base((int)fileType, storageDirectory) { }
private AudibleFileStorage(FileType fileType, string storageDirectory, params string[] extensions) : base((int)fileType, storageDirectory)
{
extensions_noDots = extensions.Select(ext => ext.Trim('.')).ToList();
extAggr = extensions_noDots.Aggregate((a, b) => $"{a}|{b}");
}
/// <summary>
/// Example for full books:
@ -102,78 +90,30 @@ namespace FileManager
/// - a directory name has the product id and an audio file is immediately inside
/// - any audio filename contains the product id
/// </summary>
public async Task<bool> ExistsAsync(string productId)
=> (await GetAsync(productId).ConfigureAwait(false)) != null;
public bool Exists(string productId)
=> GetPath(productId) != null;
public async Task<string> GetAsync(string productId)
=> await getAsync(productId).ConfigureAwait(false);
private async Task<string> getAsync(string productId)
public string GetPath(string productId)
{
{
{
var cachedFile = FilePathCache.GetPath(productId, FileType);
if (cachedFile != null)
return cachedFile;
}
// this is how files are saved by default. check this method first
{
var diskFile_byDirName = (await Task.Run(() => getFile_checkDirName(productId)).ConfigureAwait(false));
if (diskFile_byDirName != null)
{
FilePathCache.Upsert(productId, FileType, diskFile_byDirName);
return diskFile_byDirName;
}
}
var firstOrNull =
Directory
.EnumerateFiles(StorageDirectory, "*.*", SearchOption.AllDirectories)
.FirstOrDefault(s => Regex.IsMatch(s, $@"{productId}.*?\.({extAggr})$", RegexOptions.IgnoreCase));
{
var diskFile_byFileName = (await Task.Run(() => getFile_checkFileName(productId, StorageDirectory, SearchOption.AllDirectories)).ConfigureAwait(false));
if (diskFile_byFileName != null)
{
FilePathCache.Upsert(productId, FileType, diskFile_byFileName);
return diskFile_byFileName;
}
}
return null;
if (firstOrNull is null)
return null;
FilePathCache.Upsert(productId, FileType, firstOrNull);
return firstOrNull;
}
// returns audio file if there is a directory where both are true
// - the directory name contains the productId
// - the directory contains an audio file in it's top dir (not recursively)
private string getFile_checkDirName(string productId)
{
foreach (var d in Directory.EnumerateDirectories(StorageDirectory, "*.*", SearchOption.AllDirectories))
{
if (!fileHasId(d, productId))
continue;
var firstAudio = Directory
.EnumerateFiles(d, "*.*", SearchOption.TopDirectoryOnly)
.FirstOrDefault(f => IsFileTypeMatch(f));
if (firstAudio != null)
return firstAudio;
}
return null;
}
// returns audio file if there is an file where both are true
// - the file name contains the productId
// - the file is an audio type
private string getFile_checkFileName(string productId, string dir, SearchOption searchOption)
=> Directory
.EnumerateFiles(dir, "*.*", searchOption)
.FirstOrDefault(f => fileHasId(f, productId) && IsFileTypeMatch(f));
public bool IsFileTypeMatch(string filename)
=> Extensions.ContainsInsensative(Path.GetExtension(filename));
public bool IsFileTypeMatch(FileInfo fileInfo)
=> Extensions.ContainsInsensative(fileInfo.Extension);
// use GetFileName, NOT GetFileNameWithoutExtension. This tests files AND directories. if the dir has a dot in the final part of the path, it will be treated like the file extension
private static bool fileHasId(string file, string productId)
=> Path.GetFileName(file).ContainsInsensitive(productId);
=> extensions_noDots.ContainsInsensative(fileInfo.Extension.Trim('.'));
#endregion
}
}

View File

@ -15,7 +15,7 @@ namespace FileManager
public string Path { get; set; }
}
static List<CacheEntry> inMemoryCache = new List<CacheEntry>();
static List<CacheEntry> inMemoryCache { get; } = new List<CacheEntry>();
public static string JsonFile => Path.Combine(Configuration.Instance.LibationFiles, "FilePaths.json");

View File

@ -30,7 +30,7 @@ namespace LibationWinForm.BookLiberation
static async Task<StatusHandler> ProcessValidateLibraryBookAsync(IProcessable processable, LibraryBook libraryBook)
{
if (!await processable.ValidateAsync(libraryBook))
if (!processable.Validate(libraryBook))
return new StatusHandler { "Validation failed" };
return await processable.ProcessAsync(libraryBook);
}

View File

@ -37,7 +37,7 @@ namespace LibationWinForm
beginPdfBackupsToolStripMenuItem_format = beginPdfBackupsToolStripMenuItem.Text;
}
private async void Form1_Load(object sender, EventArgs e)
private void Form1_Load(object sender, EventArgs e)
{
// call static ctor. There are bad race conditions if static ctor is first executed when we're running in parallel in setBackupCountsAsync()
var foo = FilePathCache.JsonFile;
@ -59,7 +59,7 @@ namespace LibationWinForm
backupsCountsLbl.Text = "[Calculating backed up book quantities]";
pdfsCountsLbl.Text = "[Calculating backed up PDFs]";
await setBackupCountsAsync();
setBackupCounts();
}
}
@ -103,52 +103,34 @@ namespace LibationWinForm
#endregion
#region bottom: backup counts
private async Task setBackupCountsAsync()
private void setBackupCounts()
{
var books = LibraryQueries.GetLibrary_Flat_NoTracking()
.Select(sp => sp.Book)
.ToList();
await setBookBackupCountsAsync(books).ConfigureAwait(false);
await setPdfBackupCountsAsync(books).ConfigureAwait(false);
setBookBackupCounts(books);
setPdfBackupCounts(books);
}
enum AudioFileState { full, aax, none }
private async Task setBookBackupCountsAsync(IEnumerable<Book> books)
private void setBookBackupCounts(IEnumerable<Book> books)
{
var libraryProductIds = books
.Select(b => b.AudibleProductId)
.ToList();
var noProgress = 0;
var downloadedOnly = 0;
var fullyBackedUp = 0;
//// serial
//foreach (var productId in libraryProductIds)
//{
// if (await AudibleFileStorage.Audio.ExistsAsync(productId))
// fullyBackedUp++;
// else if (await AudibleFileStorage.AAX.ExistsAsync(productId))
// downloadedOnly++;
// else
// noProgress++;
//}
// parallel
async Task<AudioFileState> getAudioFileStateAsync(string productId)
AudioFileState getAudioFileState(string productId)
{
if (await AudibleFileStorage.Audio.ExistsAsync(productId))
if (AudibleFileStorage.Audio.Exists(productId))
return AudioFileState.full;
if (await AudibleFileStorage.AAX.ExistsAsync(productId))
if (AudibleFileStorage.AAX.Exists(productId))
return AudioFileState.aax;
return AudioFileState.none;
}
var tasks = libraryProductIds.Select(productId => getAudioFileStateAsync(productId));
var results = await Task.WhenAll(tasks).ConfigureAwait(false);
fullyBackedUp = results.Count(r => r == AudioFileState.full);
downloadedOnly = results.Count(r => r == AudioFileState.aax);
noProgress = results.Count(r => r == AudioFileState.none);
}
var results = books
.AsParallel()
.Select(b => getAudioFileState(b.AudibleProductId))
.ToList();
var fullyBackedUp = results.Count(r => r == AudioFileState.full);
var downloadedOnly = results.Count(r => r == AudioFileState.aax);
var noProgress = results.Count(r => r == AudioFileState.none);
// update bottom numbers
var pending = noProgress + downloadedOnly;
@ -166,32 +148,15 @@ namespace LibationWinForm
menuStrip1.UIThread(() => beginBookBackupsToolStripMenuItem.Enabled = pending > 0);
menuStrip1.UIThread(() => beginBookBackupsToolStripMenuItem.Text = string.Format(beginBookBackupsToolStripMenuItem_format, menuItemText));
}
private async Task setPdfBackupCountsAsync(IEnumerable<Book> books)
private void setPdfBackupCounts(IEnumerable<Book> books)
{
var libraryProductIds = books
.Where(b => b.Supplements.Any())
.Select(b => b.AudibleProductId)
.ToList();
int notDownloaded;
int downloaded;
//// serial
//notDownloaded = 0;
//downloaded = 0;
//foreach (var productId in libraryProductIds)
//{
// if (await AudibleFileStorage.PDF.ExistsAsync(productId))
// downloaded++;
// else
// notDownloaded++;
//}
// parallel
var tasks = libraryProductIds.Select(productId => AudibleFileStorage.PDF.ExistsAsync(productId));
var boolResults = await Task.WhenAll(tasks).ConfigureAwait(false);
downloaded = boolResults.Count(r => r);
notDownloaded = boolResults.Count(r => !r);
var boolResults = books
.AsParallel()
.Where(b => b.Supplements.Any())
.Select(b => AudibleFileStorage.PDF.Exists(b.AudibleProductId))
.ToList();
var downloaded = boolResults.Count(r => r);
var notDownloaded = boolResults.Count(r => !r);
// update bottom numbers
var text
@ -258,7 +223,7 @@ namespace LibationWinForm
#endregion
#region index menu
private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e)
private void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e)
{
using var dialog = new IndexLibraryDialog();
dialog.ShowDialog();
@ -270,7 +235,7 @@ namespace LibationWinForm
// update backup counts if we have new library items
if (newAdded > 0)
await setBackupCountsAsync();
setBackupCounts();
if (totalProcessed > 0)
reloadGrid();
@ -278,21 +243,21 @@ namespace LibationWinForm
#endregion
#region liberate menu
private async void setBackupCountsAsync(object _, string __) => await setBackupCountsAsync();
private void setBackupCounts(object _, string __) => setBackupCounts();
private async void beginBookBackupsToolStripMenuItem_Click(object sender, EventArgs e)
{
var backupBook = BookLiberation.ProcessorAutomationController.GetWiredUpBackupBook();
backupBook.DownloadBook.Completed += setBackupCountsAsync;
backupBook.DecryptBook.Completed += setBackupCountsAsync;
backupBook.DownloadPdf.Completed += setBackupCountsAsync;
backupBook.DownloadBook.Completed += setBackupCounts;
backupBook.DecryptBook.Completed += setBackupCounts;
backupBook.DownloadPdf.Completed += setBackupCounts;
await BookLiberation.ProcessorAutomationController.RunAutomaticBackup(backupBook);
}
private async void beginPdfBackupsToolStripMenuItem_Click(object sender, EventArgs e)
{
var downloadPdf = BookLiberation.ProcessorAutomationController.GetWiredUpDownloadPdf();
downloadPdf.Completed += setBackupCountsAsync;
downloadPdf.Completed += setBackupCounts;
await BookLiberation.ProcessorAutomationController.RunAutomaticDownload(downloadPdf);
}
#endregion

View File

@ -168,8 +168,8 @@ namespace LibationWinForm
get
{
var print
= FileManager.AudibleFileStorage.Audio.ExistsAsync(book.AudibleProductId).GetAwaiter().GetResult() ? "Liberated"
: FileManager.AudibleFileStorage.AAX.ExistsAsync(book.AudibleProductId).GetAwaiter().GetResult() ? "DRM"
= FileManager.AudibleFileStorage.Audio.Exists(book.AudibleProductId) ? "Liberated"
: FileManager.AudibleFileStorage.AAX.Exists(book.AudibleProductId) ? "DRM"
: "NOT d/l'ed";
if (!book.Supplements.Any())
@ -178,7 +178,7 @@ namespace LibationWinForm
print += "\r\n";
var downloadStatuses = book.Supplements
.Select(d => FileManager.AudibleFileStorage.PDF.ExistsAsync(book.AudibleProductId).GetAwaiter().GetResult())
.Select(d => FileManager.AudibleFileStorage.PDF.Exists(book.AudibleProductId))
// break delayed execution right now!
.ToList();
var count = downloadStatuses.Count;