From 2f082a96568d3764aed40f11ce4de2ef7ed5166e Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Tue, 22 Jul 2025 19:44:56 -0600 Subject: [PATCH] Refactor and optimize audiobook download and decrypt process - Add more null safety - Fix possible FilePathCache race condition - Add MoveFilesToBooksDir progress reporting - All metadata is now downloaded in parallel with other post-success tasks. - Improve download resuming and file cleanup reliability - The downloader creates temp files with a UUID filename and does not insert them into the FilePathCache. Created files only receive their final file names when they are moved into the Books folder. This is to prepare for a future plan re naming templates --- .../AaxDecrypter/AaxcDownloadConvertBase.cs | 42 +- .../AaxcDownloadMultiConverter.cs | 38 +- .../AaxcDownloadSingleConverter.cs | 31 +- Source/AaxDecrypter/AudiobookDownloadBase.cs | 157 ++--- Source/AaxDecrypter/IDownloadOptions.cs | 6 - Source/AaxDecrypter/NetworkFileStream.cs | 6 + Source/AaxDecrypter/TempFile.cs | 17 + .../UnencryptedAudiobookDownloader.cs | 13 +- Source/FileLiberator/AudioFileStorageExt.cs | 24 +- Source/FileLiberator/DownloadDecryptBook.cs | 652 +++++++++++------- Source/FileLiberator/DownloadOptions.cs | 41 -- Source/LibationFileManager/FilePathCache.cs | 16 +- 12 files changed, 562 insertions(+), 481 deletions(-) create mode 100644 Source/AaxDecrypter/TempFile.cs diff --git a/Source/AaxDecrypter/AaxcDownloadConvertBase.cs b/Source/AaxDecrypter/AaxcDownloadConvertBase.cs index fb33474f..f4ae4134 100644 --- a/Source/AaxDecrypter/AaxcDownloadConvertBase.cs +++ b/Source/AaxDecrypter/AaxcDownloadConvertBase.cs @@ -1,19 +1,21 @@ using AAXClean; using System; +using System.IO; using System.Linq; using System.Threading.Tasks; +#nullable enable namespace AaxDecrypter { public abstract class AaxcDownloadConvertBase : AudiobookDownloadBase { - public event EventHandler RetrievedMetadata; + public event EventHandler? RetrievedMetadata; - public Mp4File AaxFile { get; private set; } - protected Mp4Operation AaxConversion { get; set; } + public Mp4File? AaxFile { get; private set; } + protected Mp4Operation? AaxConversion { get; set; } - protected AaxcDownloadConvertBase(string outFileName, string cacheDirectory, IDownloadOptions dlOptions) - : base(outFileName, cacheDirectory, dlOptions) { } + protected AaxcDownloadConvertBase(string outDirectory, string cacheDirectory, IDownloadOptions dlOptions) + : base(outDirectory, cacheDirectory, dlOptions) { } /// Setting cover art by this method will insert the art into the audiobook metadata public override void SetCoverArt(byte[] coverArt) @@ -31,11 +33,13 @@ namespace AaxDecrypter private Mp4File Open() { - if (DownloadOptions.InputType is FileType.Dash) + if (DownloadOptions.DecryptionKeys is not KeyData[] keys || keys.Length == 0) + throw new InvalidOperationException($"{nameof(DownloadOptions.DecryptionKeys)} cannot be null or empty for a '{DownloadOptions.InputType}' file."); + else if (DownloadOptions.InputType is FileType.Dash) { //We may have multiple keys , so use the key whose key ID matches //the dash files default Key ID. - var keyIds = DownloadOptions.DecryptionKeys.Select(k => new Guid(k.KeyPart1, bigEndian: true)).ToArray(); + var keyIds = keys.Select(k => new Guid(k.KeyPart1, bigEndian: true)).ToArray(); var dash = new DashFile(InputFileStream); var kidIndex = Array.IndexOf(keyIds, dash.Tenc.DefaultKID); @@ -43,26 +47,38 @@ namespace AaxDecrypter if (kidIndex == -1) throw new InvalidOperationException($"None of the {keyIds.Length} key IDs match the dash file's default KeyID of {dash.Tenc.DefaultKID}"); - DownloadOptions.DecryptionKeys[0] = DownloadOptions.DecryptionKeys[kidIndex]; - var keyId = DownloadOptions.DecryptionKeys[kidIndex].KeyPart1; - var key = DownloadOptions.DecryptionKeys[kidIndex].KeyPart2; - + keys[0] = keys[kidIndex]; + var keyId = keys[kidIndex].KeyPart1; + var key = keys[kidIndex].KeyPart2 ?? throw new InvalidOperationException($"{nameof(DownloadOptions.DecryptionKeys)} for '{DownloadOptions.InputType}' must have a non-null decryption key (KeyPart2)."); dash.SetDecryptionKey(keyId, key); + WriteKeyFile($"KeyId={Convert.ToHexString(keyId)}{Environment.NewLine}Key={Convert.ToHexString(key)}"); return dash; } else if (DownloadOptions.InputType is FileType.Aax) { var aax = new AaxFile(InputFileStream); - aax.SetDecryptionKey(DownloadOptions.DecryptionKeys[0].KeyPart1); + var key = keys[0].KeyPart1; + aax.SetDecryptionKey(keys[0].KeyPart1); + WriteKeyFile($"ActivationBytes={Convert.ToHexString(key)}"); return aax; } else if (DownloadOptions.InputType is FileType.Aaxc) { var aax = new AaxFile(InputFileStream); - aax.SetDecryptionKey(DownloadOptions.DecryptionKeys[0].KeyPart1, DownloadOptions.DecryptionKeys[0].KeyPart2); + var key = keys[0].KeyPart1; + var iv = keys[0].KeyPart2 ?? throw new InvalidOperationException($"{nameof(DownloadOptions.DecryptionKeys)} for '{DownloadOptions.InputType}' must have a non-null initialization vector (KeyPart2)."); + aax.SetDecryptionKey(keys[0].KeyPart1, iv); + WriteKeyFile($"Key={Convert.ToHexString(key)}{Environment.NewLine}IV={Convert.ToHexString(iv)}"); return aax; } else throw new InvalidOperationException($"{nameof(DownloadOptions.InputType)} of '{DownloadOptions.InputType}' is unknown."); + + void WriteKeyFile(string contents) + { + var keyFile = Path.Combine(Path.ChangeExtension(InputFileStream.SaveFilePath, ".key")); + File.WriteAllText(keyFile, contents + Environment.NewLine); + OnTempFileCreated(new(keyFile)); + } } protected bool Step_GetMetadata() diff --git a/Source/AaxDecrypter/AaxcDownloadMultiConverter.cs b/Source/AaxDecrypter/AaxcDownloadMultiConverter.cs index 1948593e..bc21b0ea 100644 --- a/Source/AaxDecrypter/AaxcDownloadMultiConverter.cs +++ b/Source/AaxDecrypter/AaxcDownloadMultiConverter.cs @@ -5,20 +5,20 @@ using System; using System.IO; using System.Threading.Tasks; +#nullable enable namespace AaxDecrypter { public class AaxcDownloadMultiConverter : AaxcDownloadConvertBase { private static readonly TimeSpan minChapterLength = TimeSpan.FromSeconds(3); - private FileStream workingFileStream; + private FileStream? workingFileStream; - public AaxcDownloadMultiConverter(string outFileName, string cacheDirectory, IDownloadOptions dlOptions) - : base(outFileName, cacheDirectory, dlOptions) + public AaxcDownloadMultiConverter(string outDirectory, string cacheDirectory, IDownloadOptions dlOptions) + : base(outDirectory, cacheDirectory, dlOptions) { AsyncSteps.Name = $"Download, Convert Aaxc To {DownloadOptions.OutputFormat}, and Split"; AsyncSteps["Step 1: Get Aaxc Metadata"] = () => Task.Run(Step_GetMetadata); AsyncSteps["Step 2: Download Decrypted Audiobook"] = Step_DownloadAndDecryptAudiobookAsync; - AsyncSteps["Step 3: Download Clips and Bookmarks"] = Step_DownloadClipsBookmarksAsync; } protected override void OnInitialized() @@ -59,6 +59,7 @@ That naming may not be desirable for everyone, but it's an easy change to instea */ protected async override Task Step_DownloadAndDecryptAudiobookAsync() { + if (AaxFile is null) return false; var chapters = DownloadOptions.ChapterInfo.Chapters; // Ensure split files are at least minChapterLength in duration. @@ -83,10 +84,10 @@ That naming may not be desirable for everyone, but it's an easy change to instea try { - await (AaxConversion = decryptMultiAsync(splitChapters)); + await (AaxConversion = decryptMultiAsync(AaxFile, splitChapters)); if (AaxConversion.IsCompletedSuccessfully) - await moveMoovToBeginning(workingFileStream?.Name); + await moveMoovToBeginning(AaxFile, workingFileStream?.Name); return AaxConversion.IsCompletedSuccessfully; } @@ -97,17 +98,17 @@ That naming may not be desirable for everyone, but it's an easy change to instea } } - private Mp4Operation decryptMultiAsync(ChapterInfo splitChapters) + private Mp4Operation decryptMultiAsync(Mp4File aaxFile, ChapterInfo splitChapters) { var chapterCount = 0; return DownloadOptions.OutputFormat == OutputFormat.M4b - ? AaxFile.ConvertToMultiMp4aAsync + ? aaxFile.ConvertToMultiMp4aAsync ( splitChapters, newSplitCallback => newSplit(++chapterCount, splitChapters, newSplitCallback) ) - : AaxFile.ConvertToMultiMp3Async + : aaxFile.ConvertToMultiMp3Async ( splitChapters, newSplitCallback => newSplit(++chapterCount, splitChapters, newSplitCallback), @@ -116,33 +117,32 @@ That naming may not be desirable for everyone, but it's an easy change to instea void newSplit(int currentChapter, ChapterInfo splitChapters, INewSplitCallback newSplitCallback) { + moveMoovToBeginning(aaxFile, workingFileStream?.Name).GetAwaiter().GetResult(); + var newTempFile = GetNewTempFilePath(DownloadOptions.OutputFormat.ToString()); MultiConvertFileProperties props = new() { - OutputFileName = OutputFileName, + OutputFileName = newTempFile.FilePath, PartsPosition = currentChapter, PartsTotal = splitChapters.Count, - Title = newSplitCallback?.Chapter?.Title, + Title = newSplitCallback.Chapter?.Title, }; - moveMoovToBeginning(workingFileStream?.Name).GetAwaiter().GetResult(); - newSplitCallback.OutputFile = workingFileStream = createOutputFileStream(props); newSplitCallback.TrackTitle = DownloadOptions.GetMultipartTitle(props); newSplitCallback.TrackNumber = currentChapter; newSplitCallback.TrackCount = splitChapters.Count; - OnFileCreated(workingFileStream.Name); + OnTempFileCreated(newTempFile with { PartProperties = props }); } FileStream createOutputFileStream(MultiConvertFileProperties multiConvertFileProperties) { - var fileName = DownloadOptions.GetMultipartFileName(multiConvertFileProperties); - FileUtility.SaferDelete(fileName); - return File.Open(fileName, FileMode.OpenOrCreate, FileAccess.ReadWrite); + FileUtility.SaferDelete(multiConvertFileProperties.OutputFileName); + return File.Open(multiConvertFileProperties.OutputFileName, FileMode.OpenOrCreate, FileAccess.ReadWrite); } } - private Mp4Operation moveMoovToBeginning(string filename) + private Mp4Operation moveMoovToBeginning(Mp4File aaxFile, string? filename) { if (DownloadOptions.OutputFormat is OutputFormat.M4b && DownloadOptions.MoveMoovToBeginning @@ -151,7 +151,7 @@ That naming may not be desirable for everyone, but it's an easy change to instea { return Mp4File.RelocateMoovAsync(filename); } - else return Mp4Operation.FromCompleted(AaxFile); + else return Mp4Operation.FromCompleted(aaxFile); } } } diff --git a/Source/AaxDecrypter/AaxcDownloadSingleConverter.cs b/Source/AaxDecrypter/AaxcDownloadSingleConverter.cs index 153a6c8d..aa957746 100644 --- a/Source/AaxDecrypter/AaxcDownloadSingleConverter.cs +++ b/Source/AaxDecrypter/AaxcDownloadSingleConverter.cs @@ -6,13 +6,16 @@ using System; using System.IO; using System.Threading.Tasks; +#nullable enable namespace AaxDecrypter { public class AaxcDownloadSingleConverter : AaxcDownloadConvertBase { private readonly AverageSpeed averageSpeed = new(); - public AaxcDownloadSingleConverter(string outFileName, string cacheDirectory, IDownloadOptions dlOptions) - : base(outFileName, cacheDirectory, dlOptions) + private TempFile? outputTempFile; + + public AaxcDownloadSingleConverter(string outDirectory, string cacheDirectory, IDownloadOptions dlOptions) + : base(outDirectory, cacheDirectory, dlOptions) { var step = 1; @@ -21,7 +24,6 @@ namespace AaxDecrypter AsyncSteps[$"Step {step++}: Download Decrypted Audiobook"] = Step_DownloadAndDecryptAudiobookAsync; if (DownloadOptions.MoveMoovToBeginning && DownloadOptions.OutputFormat is OutputFormat.M4b) AsyncSteps[$"Step {step++}: Move moov atom to beginning"] = Step_MoveMoov; - AsyncSteps[$"Step {step++}: Download Clips and Bookmarks"] = Step_DownloadClipsBookmarksAsync; AsyncSteps[$"Step {step++}: Create Cue"] = Step_CreateCueAsync; } @@ -39,14 +41,16 @@ namespace AaxDecrypter protected async override Task Step_DownloadAndDecryptAudiobookAsync() { - FileUtility.SaferDelete(OutputFileName); + if (AaxFile is null) return false; + outputTempFile = GetNewTempFilePath(DownloadOptions.OutputFormat.ToString()); + FileUtility.SaferDelete(outputTempFile.FilePath); - using var outputFile = File.Open(OutputFileName, FileMode.OpenOrCreate, FileAccess.ReadWrite); - OnFileCreated(OutputFileName); + using var outputFile = File.Open(outputTempFile.FilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite); + OnTempFileCreated(outputTempFile); try { - await (AaxConversion = decryptAsync(outputFile)); + await (AaxConversion = decryptAsync(AaxFile, outputFile)); return AaxConversion.IsCompletedSuccessfully; } @@ -58,14 +62,15 @@ namespace AaxDecrypter private async Task Step_MoveMoov() { - AaxConversion = Mp4File.RelocateMoovAsync(OutputFileName); + if (outputTempFile is null) return false; + AaxConversion = Mp4File.RelocateMoovAsync(outputTempFile.FilePath); AaxConversion.ConversionProgressUpdate += AaxConversion_MoovProgressUpdate; await AaxConversion; AaxConversion.ConversionProgressUpdate -= AaxConversion_MoovProgressUpdate; return AaxConversion.IsCompletedSuccessfully; } - private void AaxConversion_MoovProgressUpdate(object sender, ConversionProgressEventArgs e) + private void AaxConversion_MoovProgressUpdate(object? sender, ConversionProgressEventArgs e) { averageSpeed.AddPosition(e.ProcessPosition.TotalSeconds); @@ -84,20 +89,20 @@ namespace AaxDecrypter }); } - private Mp4Operation decryptAsync(Stream outputFile) + private Mp4Operation decryptAsync(Mp4File aaxFile, Stream outputFile) => DownloadOptions.OutputFormat == OutputFormat.Mp3 - ? AaxFile.ConvertToMp3Async + ? aaxFile.ConvertToMp3Async ( outputFile, DownloadOptions.LameConfig, DownloadOptions.ChapterInfo ) : DownloadOptions.FixupFile - ? AaxFile.ConvertToMp4aAsync + ? aaxFile.ConvertToMp4aAsync ( outputFile, DownloadOptions.ChapterInfo ) - : AaxFile.ConvertToMp4aAsync(outputFile); + : aaxFile.ConvertToMp4aAsync(outputFile); } } diff --git a/Source/AaxDecrypter/AudiobookDownloadBase.cs b/Source/AaxDecrypter/AudiobookDownloadBase.cs index c5389d29..f439955e 100644 --- a/Source/AaxDecrypter/AudiobookDownloadBase.cs +++ b/Source/AaxDecrypter/AudiobookDownloadBase.cs @@ -6,55 +6,50 @@ using System; using System.IO; using System.Threading.Tasks; +#nullable enable namespace AaxDecrypter { public enum OutputFormat { M4b, Mp3 } public abstract class AudiobookDownloadBase { - public event EventHandler RetrievedTitle; - public event EventHandler RetrievedAuthors; - public event EventHandler RetrievedNarrators; - public event EventHandler RetrievedCoverArt; - public event EventHandler DecryptProgressUpdate; - public event EventHandler DecryptTimeRemaining; - public event EventHandler FileCreated; + public event EventHandler? RetrievedTitle; + public event EventHandler? RetrievedAuthors; + public event EventHandler? RetrievedNarrators; + public event EventHandler? RetrievedCoverArt; + public event EventHandler? DecryptProgressUpdate; + public event EventHandler? DecryptTimeRemaining; + public event EventHandler? TempFileCreated; public bool IsCanceled { get; protected set; } protected AsyncStepSequence AsyncSteps { get; } = new(); - protected string OutputFileName { get; } + protected string OutputDirectory { get; } public IDownloadOptions DownloadOptions { get; } - protected NetworkFileStream InputFileStream => nfsPersister.NetworkFileStream; + protected NetworkFileStream InputFileStream => NfsPersister.NetworkFileStream; protected virtual long InputFilePosition => InputFileStream.Position; private bool downloadFinished; - private readonly NetworkFileStreamPersister nfsPersister; + private NetworkFileStreamPersister? m_nfsPersister; + private NetworkFileStreamPersister NfsPersister => m_nfsPersister ??= OpenNetworkFileStream(); private readonly DownloadProgress zeroProgress; private readonly string jsonDownloadState; private readonly string tempFilePath; - protected AudiobookDownloadBase(string outFileName, string cacheDirectory, IDownloadOptions dlOptions) - { - OutputFileName = ArgumentValidator.EnsureNotNullOrWhiteSpace(outFileName, nameof(outFileName)); + protected AudiobookDownloadBase(string outDirectory, string cacheDirectory, IDownloadOptions dlOptions) + { + OutputDirectory = ArgumentValidator.EnsureNotNullOrWhiteSpace(outDirectory, nameof(outDirectory)); + DownloadOptions = ArgumentValidator.EnsureNotNull(dlOptions, nameof(dlOptions)); + DownloadOptions.DownloadSpeedChanged += (_, speed) => InputFileStream.SpeedLimit = speed; - var outDir = Path.GetDirectoryName(OutputFileName); - if (!Directory.Exists(outDir)) - Directory.CreateDirectory(outDir); + if (!Directory.Exists(OutputDirectory)) + Directory.CreateDirectory(OutputDirectory); if (!Directory.Exists(cacheDirectory)) Directory.CreateDirectory(cacheDirectory); - jsonDownloadState = Path.Combine(cacheDirectory, Path.GetFileName(Path.ChangeExtension(OutputFileName, ".json"))); + jsonDownloadState = Path.Combine(cacheDirectory, $"{DownloadOptions.AudibleProductId}.json"); tempFilePath = Path.ChangeExtension(jsonDownloadState, ".aaxc"); - DownloadOptions = ArgumentValidator.EnsureNotNull(dlOptions, nameof(dlOptions)); - DownloadOptions.DownloadSpeedChanged += (_, speed) => InputFileStream.SpeedLimit = speed; - - // delete file after validation is complete - FileUtility.SaferDelete(OutputFileName); - - nfsPersister = OpenNetworkFileStream(); - zeroProgress = new DownloadProgress { BytesReceived = 0, @@ -65,24 +60,30 @@ namespace AaxDecrypter OnDecryptProgressUpdate(zeroProgress); } + protected TempFile GetNewTempFilePath(string extension) + { + extension = FileUtility.GetStandardizedExtension(extension); + var path = Path.Combine(OutputDirectory, Guid.NewGuid().ToString("N") + extension); + return new(path, extension); + } + public async Task RunAsync() { await InputFileStream.BeginDownloadingAsync(); var progressTask = Task.Run(reportProgress); - AsyncSteps[$"Cleanup"] = CleanupAsync; (bool success, var elapsed) = await AsyncSteps.RunAsync(); //Stop the downloader so it doesn't keep running in the background. if (!success) - nfsPersister.Dispose(); + NfsPersister.Dispose(); await progressTask; var speedup = DownloadOptions.RuntimeLength / elapsed; Serilog.Log.Information($"Speedup is {speedup:F0}x realtime."); - nfsPersister.Dispose(); + NfsPersister.Dispose(); return success; async Task reportProgress() @@ -129,50 +130,43 @@ namespace AaxDecrypter protected abstract Task Step_DownloadAndDecryptAudiobookAsync(); public virtual void SetCoverArt(byte[] coverArt) { } - - protected void OnRetrievedTitle(string title) + protected void OnRetrievedTitle(string? title) => RetrievedTitle?.Invoke(this, title); - protected void OnRetrievedAuthors(string authors) + protected void OnRetrievedAuthors(string? authors) => RetrievedAuthors?.Invoke(this, authors); - protected void OnRetrievedNarrators(string narrators) + protected void OnRetrievedNarrators(string? narrators) => RetrievedNarrators?.Invoke(this, narrators); - protected void OnRetrievedCoverArt(byte[] coverArt) + protected void OnRetrievedCoverArt(byte[]? coverArt) => RetrievedCoverArt?.Invoke(this, coverArt); protected void OnDecryptProgressUpdate(DownloadProgress downloadProgress) => DecryptProgressUpdate?.Invoke(this, downloadProgress); protected void OnDecryptTimeRemaining(TimeSpan timeRemaining) => DecryptTimeRemaining?.Invoke(this, timeRemaining); - protected void OnFileCreated(string path) - => FileCreated?.Invoke(this, path); + public void OnTempFileCreated(TempFile path) + => TempFileCreated?.Invoke(this, path); protected virtual void FinalizeDownload() { - nfsPersister?.Dispose(); + NfsPersister.Dispose(); downloadFinished = true; } - protected async Task Step_DownloadClipsBookmarksAsync() - { - if (!IsCanceled && DownloadOptions.DownloadClipsBookmarks) - { - var recordsFile = await DownloadOptions.SaveClipsAndBookmarksAsync(OutputFileName); - - if (File.Exists(recordsFile)) - OnFileCreated(recordsFile); - } - return !IsCanceled; - } - protected async Task Step_CreateCueAsync() { if (!DownloadOptions.CreateCueSheet) return !IsCanceled; + if (DownloadOptions.ChapterInfo.Count <= 1) + { + Serilog.Log.Logger.Information($"Skipped creating .cue because book has no chapters."); + return !IsCanceled; + } + // not a critical step. its failure should not prevent future steps from running try { - var path = Path.ChangeExtension(OutputFileName, ".cue"); - await File.WriteAllTextAsync(path, Cue.CreateContents(Path.GetFileName(OutputFileName), DownloadOptions.ChapterInfo)); - OnFileCreated(path); + var tempFile = GetNewTempFilePath(".cue"); + await File.WriteAllTextAsync(tempFile.FilePath, Cue.CreateContents(Path.GetFileName(tempFile.FilePath), DownloadOptions.ChapterInfo)); + OnTempFileCreated(tempFile); } catch (Exception ex) { @@ -181,58 +175,9 @@ namespace AaxDecrypter return !IsCanceled; } - private async Task CleanupAsync() - { - if (IsCanceled) return false; - - FileUtility.SaferDelete(jsonDownloadState); - - if (DownloadOptions.DecryptionKeys != null && - DownloadOptions.RetainEncryptedFile && - DownloadOptions.InputType is AAXClean.FileType fileType) - { - //Write aax decryption key - string keyPath = Path.ChangeExtension(tempFilePath, ".key"); - FileUtility.SaferDelete(keyPath); - string aaxPath; - - if (fileType is AAXClean.FileType.Aax) - { - await File.WriteAllTextAsync(keyPath, $"ActivationBytes={Convert.ToHexString(DownloadOptions.DecryptionKeys[0].KeyPart1)}"); - aaxPath = Path.ChangeExtension(tempFilePath, ".aax"); - } - else if (fileType is AAXClean.FileType.Aaxc) - { - await File.WriteAllTextAsync(keyPath, - $"Key={Convert.ToHexString(DownloadOptions.DecryptionKeys[0].KeyPart1)}{Environment.NewLine}" + - $"IV={Convert.ToHexString(DownloadOptions.DecryptionKeys[0].KeyPart2)}"); - aaxPath = Path.ChangeExtension(tempFilePath, ".aaxc"); - } - else if (fileType is AAXClean.FileType.Dash) - { - await File.WriteAllTextAsync(keyPath, - $"KeyId={Convert.ToHexString(DownloadOptions.DecryptionKeys[0].KeyPart1)}{Environment.NewLine}" + - $"Key={Convert.ToHexString(DownloadOptions.DecryptionKeys[0].KeyPart2)}"); - aaxPath = Path.ChangeExtension(tempFilePath, ".dash"); - } - else - throw new InvalidOperationException($"Unknown file type: {fileType}"); - - if (tempFilePath != aaxPath) - FileUtility.SaferMove(tempFilePath, aaxPath); - - OnFileCreated(aaxPath); - OnFileCreated(keyPath); - } - else - FileUtility.SaferDelete(tempFilePath); - - return !IsCanceled; - } - private NetworkFileStreamPersister OpenNetworkFileStream() { - NetworkFileStreamPersister nfsp = default; + NetworkFileStreamPersister? nfsp = default; try { if (!File.Exists(jsonDownloadState)) @@ -253,8 +198,14 @@ namespace AaxDecrypter } finally { - nfsp.NetworkFileStream.RequestHeaders["User-Agent"] = DownloadOptions.UserAgent; - nfsp.NetworkFileStream.SpeedLimit = DownloadOptions.DownloadSpeedBps; + //nfsp will only be null when an unhandled exception occurs. Let the caller handle it. + if (nfsp is not null) + { + nfsp.NetworkFileStream.RequestHeaders["User-Agent"] = DownloadOptions.UserAgent; + nfsp.NetworkFileStream.SpeedLimit = DownloadOptions.DownloadSpeedBps; + OnTempFileCreated(new(tempFilePath, DownloadOptions.InputType.ToString())); + OnTempFileCreated(new(jsonDownloadState)); + } } NetworkFileStreamPersister newNetworkFilePersister() diff --git a/Source/AaxDecrypter/IDownloadOptions.cs b/Source/AaxDecrypter/IDownloadOptions.cs index fb22350e..d80a5b95 100644 --- a/Source/AaxDecrypter/IDownloadOptions.cs +++ b/Source/AaxDecrypter/IDownloadOptions.cs @@ -1,6 +1,5 @@ using AAXClean; using System; -using System.Threading.Tasks; #nullable enable namespace AaxDecrypter @@ -33,11 +32,8 @@ namespace AaxDecrypter KeyData[]? DecryptionKeys { get; } TimeSpan RuntimeLength { get; } OutputFormat OutputFormat { get; } - bool TrimOutputToChapterLength { get; } - bool RetainEncryptedFile { get; } bool StripUnabridged { get; } bool CreateCueSheet { get; } - bool DownloadClipsBookmarks { get; } long DownloadSpeedBps { get; } ChapterInfo ChapterInfo { get; } bool FixupFile { get; } @@ -52,9 +48,7 @@ namespace AaxDecrypter bool Downsample { get; } bool MatchSourceBitrate { get; } bool MoveMoovToBeginning { get; } - string GetMultipartFileName(MultiConvertFileProperties props); string GetMultipartTitle(MultiConvertFileProperties props); - Task SaveClipsAndBookmarksAsync(string fileName); public FileType? InputType { get; } } } diff --git a/Source/AaxDecrypter/NetworkFileStream.cs b/Source/AaxDecrypter/NetworkFileStream.cs index 1be60545..769a76fc 100644 --- a/Source/AaxDecrypter/NetworkFileStream.cs +++ b/Source/AaxDecrypter/NetworkFileStream.cs @@ -100,6 +100,12 @@ namespace AaxDecrypter Position = WritePosition }; + if (_writeFile.Length < WritePosition) + { + _writeFile.Dispose(); + throw new InvalidDataException($"{SaveFilePath} file length is shorter than {WritePosition}"); + } + _readFile = new FileStream(SaveFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); SetUriForSameFile(uri); diff --git a/Source/AaxDecrypter/TempFile.cs b/Source/AaxDecrypter/TempFile.cs new file mode 100644 index 00000000..f6b37ee4 --- /dev/null +++ b/Source/AaxDecrypter/TempFile.cs @@ -0,0 +1,17 @@ +using FileManager; + +#nullable enable +namespace AaxDecrypter; + +public record TempFile +{ + public LongPath FilePath { get; init; } + public string Extension { get; } + public MultiConvertFileProperties? PartProperties { get; init; } + public TempFile(LongPath filePath, string? extension = null) + { + FilePath = filePath; + extension ??= System.IO.Path.GetExtension(filePath); + Extension = FileUtility.GetStandardizedExtension(extension).ToLowerInvariant(); + } +} diff --git a/Source/AaxDecrypter/UnencryptedAudiobookDownloader.cs b/Source/AaxDecrypter/UnencryptedAudiobookDownloader.cs index 18c57054..1c6f09e8 100644 --- a/Source/AaxDecrypter/UnencryptedAudiobookDownloader.cs +++ b/Source/AaxDecrypter/UnencryptedAudiobookDownloader.cs @@ -1,5 +1,4 @@ using FileManager; -using System; using System.Threading.Tasks; namespace AaxDecrypter @@ -8,13 +7,12 @@ namespace AaxDecrypter { protected override long InputFilePosition => InputFileStream.WritePosition; - public UnencryptedAudiobookDownloader(string outFileName, string cacheDirectory, IDownloadOptions dlLic) - : base(outFileName, cacheDirectory, dlLic) + public UnencryptedAudiobookDownloader(string outDirectory, string cacheDirectory, IDownloadOptions dlLic) + : base(outDirectory, cacheDirectory, dlLic) { AsyncSteps.Name = "Download Unencrypted Audiobook"; AsyncSteps["Step 1: Download Audiobook"] = Step_DownloadAndDecryptAudiobookAsync; - AsyncSteps["Step 2: Download Clips and Bookmarks"] = Step_DownloadClipsBookmarksAsync; - AsyncSteps["Step 3: Create Cue"] = Step_CreateCueAsync; + AsyncSteps["Step 2: Create Cue"] = Step_CreateCueAsync; } protected override async Task Step_DownloadAndDecryptAudiobookAsync() @@ -26,8 +24,9 @@ namespace AaxDecrypter else { FinalizeDownload(); - FileUtility.SaferMove(InputFileStream.SaveFilePath, OutputFileName); - OnFileCreated(OutputFileName); + var tempFile = GetNewTempFilePath(DownloadOptions.OutputFormat.ToString()); + FileUtility.SaferMove(InputFileStream.SaveFilePath, tempFile.FilePath); + OnTempFileCreated(tempFile); return true; } } diff --git a/Source/FileLiberator/AudioFileStorageExt.cs b/Source/FileLiberator/AudioFileStorageExt.cs index c0ce9bd0..90ab45b1 100644 --- a/Source/FileLiberator/AudioFileStorageExt.cs +++ b/Source/FileLiberator/AudioFileStorageExt.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using AaxDecrypter; using DataLayer; using LibationFileManager; using LibationFileManager.Templates; @@ -34,30 +35,17 @@ namespace FileLiberator return Templates.Folder.GetFilename(libraryBook.ToDto(), AudibleFileStorage.BooksDirectory, ""); } - /// - /// DownloadDecryptBook: - /// Path: in progress directory. - /// File name: final file name. - /// - public static string GetInProgressFilename(this AudioFileStorage _, LibraryBookDto libraryBook, string extension) - => Templates.File.GetFilename(libraryBook, AudibleFileStorage.DecryptInProgressDirectory, extension, returnFirstExisting: true); - /// /// PDF: audio file does not exist /// - public static string GetBooksDirectoryFilename(this AudioFileStorage _, LibraryBook libraryBook, string extension) - => Templates.File.GetFilename(libraryBook.ToDto(), AudibleFileStorage.BooksDirectory, extension); - - /// - /// PDF: audio file does not exist - /// - public static string GetBooksDirectoryFilename(this AudioFileStorage _, LibraryBookDto dto, string extension) - => Templates.File.GetFilename(dto, AudibleFileStorage.BooksDirectory, extension); + public static string GetBooksDirectoryFilename(this AudioFileStorage _, LibraryBook libraryBook, string extension, bool returnFirstExisting = false) + => Templates.File.GetFilename(libraryBook.ToDto(), AudibleFileStorage.BooksDirectory, extension, returnFirstExisting: returnFirstExisting); /// /// PDF: audio file already exists /// - public static string GetCustomDirFilename(this AudioFileStorage _, LibraryBook libraryBook, string dirFullPath, string extension) - => Templates.File.GetFilename(libraryBook.ToDto(), dirFullPath, extension); + public static string GetCustomDirFilename(this AudioFileStorage _, LibraryBook libraryBook, string dirFullPath, string extension, MultiConvertFileProperties partProperties = null, bool returnFirstExisting = false) + => partProperties is null ? Templates.File.GetFilename(libraryBook.ToDto(), dirFullPath, extension, returnFirstExisting: returnFirstExisting) + : Templates.ChapterFile.GetFilename(libraryBook.ToDto(), partProperties, dirFullPath, extension, returnFirstExisting: returnFirstExisting); } } diff --git a/Source/FileLiberator/DownloadDecryptBook.cs b/Source/FileLiberator/DownloadDecryptBook.cs index 26ffb9b9..8bdae9d0 100644 --- a/Source/FileLiberator/DownloadDecryptBook.cs +++ b/Source/FileLiberator/DownloadDecryptBook.cs @@ -1,116 +1,99 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using AaxDecrypter; +using AaxDecrypter; using ApplicationServices; using AudibleApi.Common; using DataLayer; using Dinah.Core; using Dinah.Core.ErrorHandling; +using Dinah.Core.Net.Http; using FileManager; using LibationFileManager; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +#nullable enable namespace FileLiberator { - public class DownloadDecryptBook : AudioDecodable - { - public override string Name => "Download & Decrypt"; - private AudiobookDownloadBase abDownloader; - private readonly CancellationTokenSource cancellationTokenSource = new(); + public class DownloadDecryptBook : AudioDecodable + { + public override string Name => "Download & Decrypt"; + private CancellationTokenSource? cancellationTokenSource; + private AudiobookDownloadBase? abDownloader; public override bool Validate(LibraryBook libraryBook) => !libraryBook.Book.Audio_Exists(); - public override async Task CancelAsync() - { - cancellationTokenSource.Cancel(); - if (abDownloader is not null) - await abDownloader.CancelAsync(); + public override async Task CancelAsync() + { + if (abDownloader is not null) await abDownloader.CancelAsync(); + if (cancellationTokenSource is not null) await cancellationTokenSource.CancelAsync(); } - public override async Task ProcessAsync(LibraryBook libraryBook) - { - var entries = new List(); - // these only work so minimally b/c CacheEntry is a record. - // in case of parallel decrypts, only capture the ones for this book id. - // if user somehow starts multiple decrypts of the same book in parallel: on their own head be it - void FilePathCache_Inserted(object sender, FilePathCache.CacheEntry e) - { - if (e.Id.EqualsInsensitive(libraryBook.Book.AudibleProductId)) - entries.Add(e); - } - void FilePathCache_Removed(object sender, FilePathCache.CacheEntry e) - { - if (e.Id.EqualsInsensitive(libraryBook.Book.AudibleProductId)) - entries.Remove(e); - } - - OnBegin(libraryBook); - var cancellationToken = cancellationTokenSource.Token; + public override async Task ProcessAsync(LibraryBook libraryBook) + { + OnBegin(libraryBook); + cancellationTokenSource = new CancellationTokenSource(); + var cancellationToken = cancellationTokenSource.Token; try - { - if (libraryBook.Book.Audio_Exists()) - return new StatusHandler { "Cannot find decrypt. Final audio file already exists" }; + { + if (libraryBook.Book.Audio_Exists()) + return new StatusHandler { "Cannot find decrypt. Final audio file already exists" }; + + DownloadValidation(libraryBook); - downloadValidation(libraryBook); var api = await libraryBook.GetApiAsync(); - var config = Configuration.Instance; - using var downloadOptions = await DownloadOptions.InitiateDownloadAsync(api, config, libraryBook, cancellationToken); - - bool success = false; - try - { - FilePathCache.Inserted += FilePathCache_Inserted; - FilePathCache.Removed += FilePathCache_Removed; - - success = await downloadAudiobookAsync(api, config, downloadOptions); - } - finally - { - FilePathCache.Inserted -= FilePathCache_Inserted; - FilePathCache.Removed -= FilePathCache_Removed; - } - - // decrypt failed - if (!success || getFirstAudioFile(entries) == default) - { - await Task.WhenAll( - entries - .Where(f => f.FileType != FileType.AAXC) - .Select(f => Task.Run(() => FileUtility.SaferDelete(f.Path)))); + using var downloadOptions = await DownloadOptions.InitiateDownloadAsync(api, Configuration.Instance, libraryBook, cancellationToken); + var result = await DownloadAudiobookAsync(api, downloadOptions, cancellationToken); + if (!result.Success || getFirstAudioFile(result.ResultFiles) == default) + { + // decrypt failed. Delete all output entries but leave the cache files. + result.ResultFiles.ForEach(f => FileUtility.SaferDelete(f.FilePath)); cancellationToken.ThrowIfCancellationRequested(); return new StatusHandler { "Decrypt failed" }; - } + } - var finalStorageDir = getDestinationDirectory(libraryBook); + if (Configuration.Instance.RetainAaxFile) + { + //Add the cached aaxc and key files to the entries list to be moved to the Books directory. + result.ResultFiles.AddRange(getAaxcFiles(result.CacheFiles)); + } - var moveFilesTask = Task.Run(() => moveFilesToBooksDir(libraryBook, entries, cancellationToken)); - Task[] finalTasks = - [ - Task.Run(() => downloadCoverArt(downloadOptions, cancellationToken)), - moveFilesTask, - Task.Run(() => WindowsDirectory.SetCoverAsFolderIcon(libraryBook.Book.PictureId, finalStorageDir, cancellationToken)) - ]; + var finalStorageDir = getDestinationDirectory(libraryBook); + + //post-download tasks done in parallel. + var moveFilesTask = Task.Run(() => MoveFilesToBooksDir(libraryBook, finalStorageDir, result.ResultFiles, cancellationToken)); + Task[] finalTasks = + [ + moveFilesTask, + Task.Run(() => DownloadCoverArt(finalStorageDir, downloadOptions, cancellationToken)), + Task.Run(() => DownloadRecordsAsync(api, finalStorageDir, downloadOptions, cancellationToken)), + Task.Run(() => DownloadMetadataAsync(api, finalStorageDir, downloadOptions, cancellationToken)), + Task.Run(() => WindowsDirectory.SetCoverAsFolderIcon(libraryBook.Book.PictureId, finalStorageDir, cancellationToken)) + ]; try - { + { await Task.WhenAll(finalTasks); } - catch when (!moveFilesTask.IsFaulted) + catch when (!moveFilesTask.IsFaulted) { - //Swallow downloadCoverArt and SetCoverAsFolderIcon exceptions. + //Swallow DownloadCoverArt, SetCoverAsFolderIcon, and SaveMetadataAsync exceptions. //Only fail if the downloaded audio files failed to move to Books directory } finally - { - if (moveFilesTask.IsCompletedSuccessfully && !cancellationToken.IsCancellationRequested) - { - await Task.Run(() => libraryBook.UpdateBookStatus(LiberatedStatus.Liberated, Configuration.LibationVersion)); - + { + if (moveFilesTask.IsCompletedSuccessfully && !cancellationToken.IsCancellationRequested) + { + libraryBook.UpdateBookStatus(LiberatedStatus.Liberated, Configuration.LibationVersion!); SetDirectoryTime(libraryBook, finalStorageDir); + foreach (var cacheFile in result.CacheFiles.Where(f => File.Exists(f.FilePath))) + { + //Delete cache files only after the download/decrypt operation completes successfully. + FileUtility.SaferDelete(cacheFile.FilePath); + } } } @@ -122,59 +105,86 @@ namespace FileLiberator return new StatusHandler { "Cancelled" }; } finally - { - OnCompleted(libraryBook); - } - } - - private async Task downloadAudiobookAsync(AudibleApi.Api api, Configuration config, DownloadOptions dlOptions) - { - var outFileName = AudibleFileStorage.Audio.GetInProgressFilename(dlOptions.LibraryBookDto, dlOptions.OutputFormat.ToString().ToLower()); - var cacheDir = AudibleFileStorage.DownloadsInProgressDirectory; - - if (dlOptions.DrmType is not DrmType.Adrm and not DrmType.Widevine) - abDownloader = new UnencryptedAudiobookDownloader(outFileName, cacheDir, dlOptions); - else - { - AaxcDownloadConvertBase converter - = config.SplitFilesByChapter ? - new AaxcDownloadMultiConverter(outFileName, cacheDir, dlOptions) : - new AaxcDownloadSingleConverter(outFileName, cacheDir, dlOptions); - - if (config.AllowLibationFixup) - converter.RetrievedMetadata += Converter_RetrievedMetadata; - - abDownloader = converter; - } - - abDownloader.DecryptProgressUpdate += OnStreamingProgressChanged; - abDownloader.DecryptTimeRemaining += OnStreamingTimeRemaining; - abDownloader.RetrievedTitle += OnTitleDiscovered; - abDownloader.RetrievedAuthors += OnAuthorsDiscovered; - abDownloader.RetrievedNarrators += OnNarratorsDiscovered; - abDownloader.RetrievedCoverArt += AaxcDownloader_RetrievedCoverArt; - abDownloader.FileCreated += (_, path) => OnFileCreated(dlOptions.LibraryBook, path); - - // REAL WORK DONE HERE - var success = await abDownloader.RunAsync(); - - if (success && config.SaveMetadataToFile) - { - var metadataFile = LibationFileManager.Templates.Templates.File.GetFilename(dlOptions.LibraryBookDto, Path.GetDirectoryName(outFileName), ".metadata.json"); - - var item = await api.GetCatalogProductAsync(dlOptions.LibraryBook.Book.AudibleProductId, AudibleApi.CatalogOptions.ResponseGroupOptions.ALL_OPTIONS); - item.SourceJson.Add(nameof(ContentMetadata.ChapterInfo), Newtonsoft.Json.Linq.JObject.FromObject(dlOptions.ContentMetadata.ChapterInfo)); - item.SourceJson.Add(nameof(ContentMetadata.ContentReference), Newtonsoft.Json.Linq.JObject.FromObject(dlOptions.ContentMetadata.ContentReference)); - - File.WriteAllText(metadataFile, item.SourceJson.ToString()); - OnFileCreated(dlOptions.LibraryBook, metadataFile); - } - return success; + { + OnCompleted(libraryBook); + cancellationTokenSource.Dispose(); + cancellationTokenSource = null; + } } - private void Converter_RetrievedMetadata(object sender, AAXClean.AppleTags tags) + private record AudiobookDecryptResult(bool Success, List ResultFiles, List CacheFiles); + + private async Task DownloadAudiobookAsync(AudibleApi.Api api, DownloadOptions dlOptions, CancellationToken cancellationToken) { - if (sender is not AaxcDownloadConvertBase converter || converter.DownloadOptions is not DownloadOptions options) + var outpoutDir = AudibleFileStorage.DecryptInProgressDirectory; + var cacheDir = AudibleFileStorage.DownloadsInProgressDirectory; + var result = new AudiobookDecryptResult(false, [], []); + + try + { + if (dlOptions.DrmType is not DrmType.Adrm and not DrmType.Widevine) + abDownloader = new UnencryptedAudiobookDownloader(outpoutDir, cacheDir, dlOptions); + else + { + AaxcDownloadConvertBase converter + = dlOptions.Config.SplitFilesByChapter ? + new AaxcDownloadMultiConverter(outpoutDir, cacheDir, dlOptions) : + new AaxcDownloadSingleConverter(outpoutDir, cacheDir, dlOptions); + + if (dlOptions.Config.AllowLibationFixup) + converter.RetrievedMetadata += Converter_RetrievedMetadata; + + abDownloader = converter; + } + + abDownloader.DecryptProgressUpdate += OnStreamingProgressChanged; + abDownloader.DecryptTimeRemaining += OnStreamingTimeRemaining; + abDownloader.RetrievedTitle += OnTitleDiscovered; + abDownloader.RetrievedAuthors += OnAuthorsDiscovered; + abDownloader.RetrievedNarrators += OnNarratorsDiscovered; + abDownloader.RetrievedCoverArt += AaxcDownloader_RetrievedCoverArt; + abDownloader.TempFileCreated += AbDownloader_TempFileCreated; + + // REAL WORK DONE HERE + bool success = await abDownloader.RunAsync(); + return result with { Success = success }; + } + catch (Exception ex) + { + if (!cancellationToken.IsCancellationRequested) + Serilog.Log.Logger.Error(ex, "Error downloading audiobook {@Book}", dlOptions.LibraryBook.LogFriendly()); + //don't throw any exceptions so the caller can delete any temp files. + return result; + } + finally + { + OnStreamingProgressChanged(new() { ProgressPercentage = 100 }); + } + + void AbDownloader_TempFileCreated(object? sender, TempFile e) + { + if (Path.GetDirectoryName(e.FilePath) == outpoutDir) + { + result.ResultFiles.Add(e); + } + else if (Path.GetDirectoryName(e.FilePath) == cacheDir) + { + result.CacheFiles.Add(e); + // Notify that the aaxc file has been created so that + // the UI can know about partially-downloaded files + if (getFileType(e) is FileType.AAXC) + OnFileCreated(dlOptions.LibraryBook, e.FilePath); + } + } + } + + #region Decryptor event handlers + private void Converter_RetrievedMetadata(object? sender, AAXClean.AppleTags tags) + { + if (sender is not AaxcDownloadConvertBase converter || + converter.AaxFile is not AAXClean.Mp4File aaxFile || + converter.DownloadOptions is not DownloadOptions options || + options.ChapterInfo.Chapters is not List chapters) return; #region Prevent erroneous truncation due to incorrect chapter info @@ -185,159 +195,287 @@ namespace FileLiberator //the chapter. This is never desirable, so pad the last chapter to match //the original audio length. - var fileDuration = converter.AaxFile.Duration; - if (options.Config.StripAudibleBrandAudio) - fileDuration -= TimeSpan.FromMilliseconds(options.ContentMetadata.ChapterInfo.BrandOutroDurationMs); + var fileDuration = aaxFile.Duration; + if (options.Config.StripAudibleBrandAudio) + fileDuration -= TimeSpan.FromMilliseconds(options.ContentMetadata.ChapterInfo.BrandOutroDurationMs); - var durationDelta = fileDuration - options.ChapterInfo.EndOffset; + var durationDelta = fileDuration - options.ChapterInfo.EndOffset; //Remove the last chapter and re-add it with the durationDelta that will - //make the chapter's end coincide with the end of the audio file. - var chapters = options.ChapterInfo.Chapters as List; + //make the chapter's end coincide with the end of the audio file. var lastChapter = chapters[^1]; chapters.Remove(lastChapter); options.ChapterInfo.Add(lastChapter.Title, lastChapter.Duration + durationDelta); - - #endregion + + #endregion tags.Title ??= options.LibraryBookDto.TitleWithSubtitle; - tags.Album ??= tags.Title; - tags.Artist ??= string.Join("; ", options.LibraryBook.Book.Authors.Select(a => a.Name)); - tags.AlbumArtists ??= tags.Artist; + tags.Album ??= tags.Title; + tags.Artist ??= string.Join("; ", options.LibraryBook.Book.Authors.Select(a => a.Name)); + tags.AlbumArtists ??= tags.Artist; tags.Generes = string.Join(", ", options.LibraryBook.Book.LowestCategoryNames()); - tags.ProductID ??= options.ContentMetadata.ContentReference.Sku; - tags.Comment ??= options.LibraryBook.Book.Description; - tags.LongDescription ??= tags.Comment; - tags.Publisher ??= options.LibraryBook.Book.Publisher; + tags.ProductID ??= options.ContentMetadata.ContentReference.Sku; + tags.Comment ??= options.LibraryBook.Book.Description; + tags.LongDescription ??= tags.Comment; + tags.Publisher ??= options.LibraryBook.Book.Publisher; tags.Narrator ??= string.Join("; ", options.LibraryBook.Book.Narrators.Select(n => n.Name)); tags.Asin = options.LibraryBook.Book.AudibleProductId; tags.Acr = options.ContentMetadata.ContentReference.Acr; tags.Version = options.ContentMetadata.ContentReference.Version; if (options.LibraryBook.Book.DatePublished is DateTime pubDate) - { - tags.Year ??= pubDate.Year.ToString(); - tags.ReleaseDate ??= pubDate.ToString("dd-MMM-yyyy"); - } - } - - private static void downloadValidation(LibraryBook libraryBook) - { - string errorString(string field) - => $"{errorTitle()}\r\nCannot download book. {field} is not known. Try re-importing the account which owns this book."; - - string errorTitle() - { - var title - = (libraryBook.Book.TitleWithSubtitle.Length > 53) - ? $"{libraryBook.Book.TitleWithSubtitle.Truncate(50)}..." - : libraryBook.Book.TitleWithSubtitle; - var errorBookTitle = $"{title} [{libraryBook.Book.AudibleProductId}]"; - return errorBookTitle; - }; - - if (string.IsNullOrWhiteSpace(libraryBook.Account)) - throw new Exception(errorString("Account")); - - if (string.IsNullOrWhiteSpace(libraryBook.Book.Locale)) - throw new Exception(errorString("Locale")); - } - - private void AaxcDownloader_RetrievedCoverArt(object _, byte[] e) - { - if (Configuration.Instance.AllowLibationFixup) - { - try - { - e = OnRequestCoverArt(); - abDownloader.SetCoverArt(e); - } - catch (Exception ex) - { - Serilog.Log.Logger.Error(ex, "Failed to retrieve cover art from server."); - } - } - - if (e is not null) - OnCoverImageDiscovered(e); + { + tags.Year ??= pubDate.Year.ToString(); + tags.ReleaseDate ??= pubDate.ToString("dd-MMM-yyyy"); + } } - /// Move new files to 'Books' directory - /// Return directory if audiobook file(s) were successfully created and can be located on disk. Else null. - private static void moveFilesToBooksDir(LibraryBook libraryBook, List entries, CancellationToken cancellationToken) - { - // create final directory. move each file into it - var destinationDir = getDestinationDirectory(libraryBook); - cancellationToken.ThrowIfCancellationRequested(); - - for (var i = 0; i < entries.Count; i++) - { - var entry = entries[i]; - - var realDest - = FileUtility.SaferMoveToValidPath( - entry.Path, - Path.Combine(destinationDir, Path.GetFileName(entry.Path)), - Configuration.Instance.ReplacementCharacters, - overwrite: Configuration.Instance.OverwriteExisting); - - SetFileTime(libraryBook, realDest); - FilePathCache.Insert(libraryBook.Book.AudibleProductId, realDest); - - // propagate corrected path. Must update cache with corrected path. Also want updated path for cue file (after this for-loop) - entries[i] = entry with { Path = realDest }; - cancellationToken.ThrowIfCancellationRequested(); + private void AaxcDownloader_RetrievedCoverArt(object? sender, byte[]? e) + { + if (Configuration.Instance.AllowLibationFixup && sender is AaxcDownloadConvertBase downloader) + { + try + { + e = OnRequestCoverArt(); + downloader.SetCoverArt(e); + } + catch (Exception ex) + { + Serilog.Log.Logger.Error(ex, "Failed to retrieve cover art from server."); + } } - var cue = entries.FirstOrDefault(f => f.FileType == FileType.Cue); - if (cue != default) - { - Cue.UpdateFileName(cue.Path, getFirstAudioFile(entries).Path); - SetFileTime(libraryBook, cue.Path); + if (e is not null) + OnCoverImageDiscovered(e); + } + #endregion + + #region Validation + + private static void DownloadValidation(LibraryBook libraryBook) + { + string errorString(string field) + => $"{errorTitle()}\r\nCannot download book. {field} is not known. Try re-importing the account which owns this book."; + + string errorTitle() + { + var title + = (libraryBook.Book.TitleWithSubtitle.Length > 53) + ? $"{libraryBook.Book.TitleWithSubtitle.Truncate(50)}..." + : libraryBook.Book.TitleWithSubtitle; + var errorBookTitle = $"{title} [{libraryBook.Book.AudibleProductId}]"; + return errorBookTitle; + }; + + if (string.IsNullOrWhiteSpace(libraryBook.Account)) + throw new InvalidOperationException(errorString("Account")); + + if (string.IsNullOrWhiteSpace(libraryBook.Book.Locale)) + throw new InvalidOperationException(errorString("Locale")); + } + #endregion + + #region Post-success routines + /// Move new files to 'Books' directory + /// Return directory if audiobook file(s) were successfully created and can be located on disk. Else null. + private void MoveFilesToBooksDir(LibraryBook libraryBook, LongPath destinationDir, List entries, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + AverageSpeed averageSpeed = new(); + + var totalSizeToMove = entries.Sum(f => new FileInfo(f.FilePath).Length); + long totalBytesMoved = 0; + + for (var i = 0; i < entries.Count; i++) + { + var entry = entries[i]; + + var destFileName + = AudibleFileStorage.Audio.GetCustomDirFilename( + libraryBook, + destinationDir, + entry.Extension, + entry.PartProperties, + Configuration.Instance.OverwriteExisting); + + var realDest + = FileUtility.SaferMoveToValidPath( + entry.FilePath, + destFileName, + Configuration.Instance.ReplacementCharacters, + entry.Extension, + Configuration.Instance.OverwriteExisting); + + #region File Move Progress + totalBytesMoved += new FileInfo(realDest).Length; + averageSpeed.AddPosition(totalBytesMoved); + var estSecsRemaining = (totalSizeToMove - totalBytesMoved) / averageSpeed.Average; + + if (double.IsNormal(estSecsRemaining)) + OnStreamingTimeRemaining(TimeSpan.FromSeconds(estSecsRemaining)); + + OnStreamingProgressChanged(new DownloadProgress + { + ProgressPercentage = 100d * totalBytesMoved / totalSizeToMove, + BytesReceived = totalBytesMoved, + TotalBytesToReceive = totalSizeToMove + }); + #endregion + + // propagate corrected path for cue file (after this for-loop) + entries[i] = entry with { FilePath = realDest }; + + SetFileTime(libraryBook, realDest); + OnFileCreated(libraryBook, realDest); + cancellationToken.ThrowIfCancellationRequested(); + } + + if (entries.FirstOrDefault(f => getFileType(f) is FileType.Cue) is TempFile cue + && getFirstAudioFile(entries)?.FilePath is LongPath audioFilePath) + { + Cue.UpdateFileName(cue.FilePath, audioFilePath); + SetFileTime(libraryBook, cue.FilePath); } cancellationToken.ThrowIfCancellationRequested(); AudibleFileStorage.Audio.Refresh(); - } - - private static string getDestinationDirectory(LibraryBook libraryBook) - { - var destinationDir = AudibleFileStorage.Audio.GetDestinationDirectory(libraryBook); - if (!Directory.Exists(destinationDir)) - Directory.CreateDirectory(destinationDir); - return destinationDir; } - private static FilePathCache.CacheEntry getFirstAudioFile(IEnumerable entries) - => entries.FirstOrDefault(f => f.FileType == FileType.Audio); + private void DownloadCoverArt(LongPath destinationDir, DownloadOptions options, CancellationToken cancellationToken) + { + if (!options.Config.DownloadCoverArt) return; - private static void downloadCoverArt(DownloadOptions options, CancellationToken cancellationToken) - { - if (!Configuration.Instance.DownloadCoverArt) return; + var coverPath = "[null]"; - var coverPath = "[null]"; + try + { + coverPath + = AudibleFileStorage.Audio.GetCustomDirFilename( + options.LibraryBook, + destinationDir, + extension: ".jpg", + returnFirstExisting: Configuration.Instance.OverwriteExisting); - try - { - var destinationDir = getDestinationDirectory(options.LibraryBook); - coverPath = AudibleFileStorage.Audio.GetBooksDirectoryFilename(options.LibraryBookDto, ".jpg"); - coverPath = Path.Combine(destinationDir, Path.GetFileName(coverPath)); + if (File.Exists(coverPath)) + FileUtility.SaferDelete(coverPath); - if (File.Exists(coverPath)) - FileUtility.SaferDelete(coverPath); + var picBytes = PictureStorage.GetPictureSynchronously(new(options.LibraryBook.Book.PictureLarge ?? options.LibraryBook.Book.PictureId, PictureSize.Native), cancellationToken); + if (picBytes.Length > 0) + { + File.WriteAllBytes(coverPath, picBytes); + SetFileTime(options.LibraryBook, coverPath); + OnFileCreated(options.LibraryBook, coverPath); + } + } + catch (Exception ex) + { + //Failure to download cover art should not be considered a failure to download the book + if (!cancellationToken.IsCancellationRequested) + Serilog.Log.Logger.Error(ex, "Error downloading cover art for {@Book} to {@metadataFile}.", options.LibraryBook, coverPath); + throw; + } + } - var picBytes = PictureStorage.GetPictureSynchronously(new(options.LibraryBook.Book.PictureLarge ?? options.LibraryBook.Book.PictureId, PictureSize.Native), cancellationToken); - if (picBytes.Length > 0) - { - File.WriteAllBytes(coverPath, picBytes); - SetFileTime(options.LibraryBook, coverPath); - } - } - catch (Exception ex) - { - //Failure to download cover art should not be considered a failure to download the book - Serilog.Log.Logger.Error(ex, $"Error downloading cover art of {options.LibraryBook.Book.AudibleProductId} to {coverPath} catalog product."); - throw; - } - } - } + public async Task DownloadRecordsAsync(AudibleApi.Api api, LongPath destinationDir, DownloadOptions options, CancellationToken cancellationToken) + { + if (!options.Config.DownloadClipsBookmarks) return; + + var recordsPath = "[null]"; + var format = options.Config.ClipsBookmarksFileFormat; + var formatExtension = FileUtility.GetStandardizedExtension(format.ToString().ToLowerInvariant()); + + try + { + recordsPath + = AudibleFileStorage.Audio.GetCustomDirFilename( + options.LibraryBook, + destinationDir, + extension: formatExtension, + returnFirstExisting: Configuration.Instance.OverwriteExisting); + + if (File.Exists(recordsPath)) + FileUtility.SaferDelete(recordsPath); + + var records = await api.GetRecordsAsync(options.AudibleProductId); + + switch (format) + { + case Configuration.ClipBookmarkFormat.CSV: + RecordExporter.ToCsv(recordsPath, records); + break; + case Configuration.ClipBookmarkFormat.Xlsx: + RecordExporter.ToXlsx(recordsPath, records); + break; + case Configuration.ClipBookmarkFormat.Json: + RecordExporter.ToJson(recordsPath, options.LibraryBook, records); + break; + default: + throw new NotSupportedException($"Unsupported record export format: {format}"); + } + + SetFileTime(options.LibraryBook, recordsPath); + OnFileCreated(options.LibraryBook, recordsPath); + } + catch (Exception ex) + { + //Failure to download records should not be considered a failure to download the book + if (!cancellationToken.IsCancellationRequested) + Serilog.Log.Logger.Error(ex, "Error downloading clips and bookmarks for {@Book} to {@recordsPath}.", options.LibraryBook, recordsPath); + throw; + } + } + + private async Task DownloadMetadataAsync(AudibleApi.Api api, LongPath destinationDir, DownloadOptions options, CancellationToken cancellationToken) + { + if (!options.Config.SaveMetadataToFile) return; + + string metadataPath = "[null]"; + + try + { + metadataPath + = AudibleFileStorage.Audio.GetCustomDirFilename( + options.LibraryBook, + destinationDir, + extension: ".metadata.json", + returnFirstExisting: Configuration.Instance.OverwriteExisting); + + if (File.Exists(metadataPath)) + FileUtility.SaferDelete(metadataPath); + + var item = await api.GetCatalogProductAsync(options.LibraryBook.Book.AudibleProductId, AudibleApi.CatalogOptions.ResponseGroupOptions.ALL_OPTIONS); + item.SourceJson.Add(nameof(ContentMetadata.ChapterInfo), Newtonsoft.Json.Linq.JObject.FromObject(options.ContentMetadata.ChapterInfo)); + item.SourceJson.Add(nameof(ContentMetadata.ContentReference), Newtonsoft.Json.Linq.JObject.FromObject(options.ContentMetadata.ContentReference)); + + cancellationToken.ThrowIfCancellationRequested(); + File.WriteAllText(metadataPath, item.SourceJson.ToString()); + SetFileTime(options.LibraryBook, metadataPath); + OnFileCreated(options.LibraryBook, metadataPath); + } + catch (Exception ex) + { + //Failure to download metadata should not be considered a failure to download the book + if (!cancellationToken.IsCancellationRequested) + Serilog.Log.Logger.Error(ex, "Error downloading metdatat of {@Book} to {@metadataFile}.", options.LibraryBook, metadataPath); + throw; + } + } + #endregion + + #region Macros + private static string getDestinationDirectory(LibraryBook libraryBook) + { + var destinationDir = AudibleFileStorage.Audio.GetDestinationDirectory(libraryBook); + if (!Directory.Exists(destinationDir)) + Directory.CreateDirectory(destinationDir); + return destinationDir; + } + + private static FileType getFileType(TempFile file) + => FileTypes.GetFileTypeFromPath(file.FilePath); + private static TempFile? getFirstAudioFile(IEnumerable entries) + => entries.FirstOrDefault(f => getFileType(f) is FileType.Audio); + private static IEnumerable getAaxcFiles(IEnumerable entries) + => entries.Where(f => getFileType(f) is FileType.AAXC || f.Extension.Equals(".key", StringComparison.OrdinalIgnoreCase)); + #endregion + } } diff --git a/Source/FileLiberator/DownloadOptions.cs b/Source/FileLiberator/DownloadOptions.cs index 86777021..514ee2bd 100644 --- a/Source/FileLiberator/DownloadOptions.cs +++ b/Source/FileLiberator/DownloadOptions.cs @@ -3,10 +3,8 @@ using AAXClean; using Dinah.Core; using DataLayer; using LibationFileManager; -using System.Threading.Tasks; using System; using System.IO; -using ApplicationServices; using LibationFileManager.Templates; #nullable enable @@ -31,12 +29,9 @@ namespace FileLiberator public float? SeriesNumber => LibraryBookDto.FirstSeries?.Number; public NAudio.Lame.LameConfig? LameConfig { get; } public string UserAgent => AudibleApi.Resources.Download_User_Agent; - public bool TrimOutputToChapterLength => Config.AllowLibationFixup && Config.StripAudibleBrandAudio; public bool StripUnabridged => Config.AllowLibationFixup && Config.StripUnabridged; public bool CreateCueSheet => Config.CreateCueSheet; - public bool DownloadClipsBookmarks => Config.DownloadClipsBookmarks; public long DownloadSpeedBps => Config.DownloadSpeedLimit; - public bool RetainEncryptedFile => Config.RetainAaxFile; public bool FixupFile => Config.AllowLibationFixup; public bool Downsample => Config.AllowLibationFixup && Config.LameDownsampleMono; public bool MatchSourceBitrate => Config.AllowLibationFixup && Config.LameMatchSourceBR && Config.LameTargetBitrate; @@ -45,45 +40,9 @@ namespace FileLiberator public AudibleApi.Common.DrmType DrmType { get; } public AudibleApi.Common.ContentMetadata ContentMetadata { get; } - public string GetMultipartFileName(MultiConvertFileProperties props) - { - var baseDir = Path.GetDirectoryName(props.OutputFileName); - var extension = Path.GetExtension(props.OutputFileName); - return Templates.ChapterFile.GetFilename(LibraryBookDto, props, baseDir!, extension); - } - public string GetMultipartTitle(MultiConvertFileProperties props) => Templates.ChapterTitle.GetName(LibraryBookDto, props); - public async Task SaveClipsAndBookmarksAsync(string fileName) - { - if (DownloadClipsBookmarks) - { - var format = Config.ClipsBookmarksFileFormat; - - var formatExtension = format.ToString().ToLowerInvariant(); - var filePath = Path.ChangeExtension(fileName, formatExtension); - - var api = await LibraryBook.GetApiAsync(); - var records = await api.GetRecordsAsync(LibraryBook.Book.AudibleProductId); - - switch(format) - { - case Configuration.ClipBookmarkFormat.CSV: - RecordExporter.ToCsv(filePath, records); - break; - case Configuration.ClipBookmarkFormat.Xlsx: - RecordExporter.ToXlsx(filePath, records); - break; - case Configuration.ClipBookmarkFormat.Json: - RecordExporter.ToJson(filePath, LibraryBook, records); - break; - } - return filePath; - } - return string.Empty; - } - public Configuration Config { get; } private readonly IDisposable cancellation; public void Dispose() diff --git a/Source/LibationFileManager/FilePathCache.cs b/Source/LibationFileManager/FilePathCache.cs index 469de1fd..a6c99ef6 100644 --- a/Source/LibationFileManager/FilePathCache.cs +++ b/Source/LibationFileManager/FilePathCache.cs @@ -46,7 +46,9 @@ namespace LibationFileManager public static List<(FileType fileType, LongPath path)> GetFiles(string id) { - var matchingFiles = Cache.GetIdEntries(id); + List matchingFiles; + lock(locker) + matchingFiles = Cache.GetIdEntries(id); bool cacheChanged = false; @@ -68,7 +70,9 @@ namespace LibationFileManager public static LongPath? GetFirstPath(string id, FileType type) { - var matchingFiles = Cache.GetIdEntries(id).Where(e => e.FileType == type).ToList(); + List matchingFiles; + lock (locker) + matchingFiles = Cache.GetIdEntries(id).Where(e => e.FileType == type).ToList(); bool cacheChanged = false; try @@ -96,7 +100,10 @@ namespace LibationFileManager private static bool Remove(CacheEntry entry) { - if (Cache.Remove(entry.Id, entry)) + bool removed; + lock (locker) + removed = Cache.Remove(entry.Id, entry); + if (removed) { Removed?.Invoke(null, entry); return true; @@ -112,7 +119,8 @@ namespace LibationFileManager public static void Insert(CacheEntry entry) { - Cache.Add(entry.Id, entry); + lock(locker) + Cache.Add(entry.Id, entry); Inserted?.Invoke(null, entry); save(); }