Merge pull request #46 from Mbucari/master

Added resumable download support to FFMpegAaxcProcessor.

AaxDecrypter/AaxcDownloadConverter.cs

@@ -32,6 +32,7 @@ namespace AaxDecrypter
         bool Step4_RestoreMetadata();
         bool Step5_CreateCue();
         bool Step6_CreateNfo();
+        bool Step7_Cleanup();
     }
     public class AaxcDownloadConverter : IAdvancedAaxcToM4bConverter
     {
@@ -41,6 +42,7 @@ namespace AaxDecrypter
         public event EventHandler<TimeSpan> DecryptTimeRemaining;
         public string AppName { get; set; } = nameof(AaxcDownloadConverter);
         public string outDir { get; private set; }
+        public string cacheDir { get; private set; }
         public string outputFileName { get; private set; }
         public DownloadLicense downloadLicense { get; private set; }
         public AaxcTagLibFile aaxcTagLib { get; private set; }
@@ -49,26 +51,32 @@ namespace AaxDecrypter
         private StepSequence steps { get; }
         private FFMpegAaxcProcesser aaxcProcesser;
         private bool isCanceled { get; set; }
+        private string jsonDownloadState => Path.Combine(cacheDir, Path.GetFileNameWithoutExtension(outputFileName) + ".json");
+        private string tempFile => PathLib.ReplaceExtension(jsonDownloadState, ".aaxc");
 
-        public static AaxcDownloadConverter Create(string outDirectory, DownloadLicense dlLic)
+        public static AaxcDownloadConverter Create(string cacheDirectory, string outDirectory, DownloadLicense dlLic)
         {
-            var converter = new AaxcDownloadConverter(outDirectory, dlLic);
+            var converter = new AaxcDownloadConverter(cacheDirectory, outDirectory, dlLic);
             converter.SetOutputFilename(Path.GetTempFileName());
             return converter;
         }
 
-        private AaxcDownloadConverter(string outDirectory, DownloadLicense dlLic)
+        private AaxcDownloadConverter(string cacheDirectory, string outDirectory, DownloadLicense dlLic)
         {
             ArgumentValidator.EnsureNotNullOrWhiteSpace(outDirectory, nameof(outDirectory));
             ArgumentValidator.EnsureNotNull(dlLic, nameof(dlLic));
 
+            if (!Directory.Exists(outDirectory))
+                throw new ArgumentNullException(nameof(cacheDirectory), "Directory does not exist");
             if (!Directory.Exists(outDirectory))
                 throw new ArgumentNullException(nameof(outDirectory), "Directory does not exist");
+
+            cacheDir = cacheDirectory;
             outDir = outDirectory;
 
             steps = new StepSequence
             {
-                Name = "Convert Aax To M4b",
+                Name = "Download and Convert Aaxc To M4b",
 
                 ["Step 1: Create Dir"] = Step1_CreateDir,
                 ["Step 2: Get Aaxc Metadata"] = Step2_GetMetadata,
@@ -76,6 +84,7 @@ namespace AaxDecrypter
                 ["Step 4: Restore Aaxc Metadata"] = Step4_RestoreMetadata,
                 ["Step 5: Create Cue"] = Step5_CreateCue,
                 ["Step 6: Create Nfo"] = Step6_CreateNfo,
+                ["Step 7: Cleanup"] = Step7_Cleanup,
             };
 
             aaxcProcesser = new FFMpegAaxcProcesser(dlLic);
@@ -130,10 +139,6 @@ namespace AaxDecrypter
             //Get metadata from the file over http
 
             NetworkFileStreamPersister nfsPersister;
-
-            string jsonDownloadState = PathLib.ReplaceExtension(outputFileName, ".json");
-            string tempFile = PathLib.ReplaceExtension(outputFileName, ".aaxc");
-
             if (File.Exists(jsonDownloadState))
             {
                 nfsPersister = new NetworkFileStreamPersister(jsonDownloadState);
@@ -145,14 +150,12 @@ namespace AaxDecrypter
 
                 NetworkFileStream networkFileStream = new NetworkFileStream(tempFile, new Uri(downloadLicense.DownloadUrl), 0, headers);
                 nfsPersister = new NetworkFileStreamPersister(networkFileStream, jsonDownloadState);
-                nfsPersister.Target.BeginDownloading().GetAwaiter().GetResult();
             }
 
             var networkFile = new NetworkFileAbstraction(nfsPersister.NetworkFileStream);
-
+            aaxcTagLib = new AaxcTagLibFile(networkFile);
             nfsPersister.Dispose();
 
-            aaxcTagLib = new AaxcTagLibFile(networkFile);
 
             if (coverArt is null && aaxcTagLib.AppleTags.Pictures.Length > 0)
             {
@@ -169,28 +172,49 @@ namespace AaxDecrypter
         {
             DecryptProgressUpdate?.Invoke(this, int.MaxValue);
 
-            bool userSuppliedChapters = downloadLicense.ChapterInfo != null;
+            NetworkFileStreamPersister nfsPersister;
-
+            if (File.Exists(jsonDownloadState))
-            string metadataPath = null;
-
-            if (userSuppliedChapters)
             {
-                //Only write chaopters to the metadata file. All other aaxc metadata will be
+                nfsPersister = new NetworkFileStreamPersister(jsonDownloadState);
-                //wiped out but is restored in Step 3.
+                //If More thaan ~1 hour has elapsed since getting the download url, it will expire.
-                metadataPath = Path.Combine(outDir, Path.GetFileName(outputFileName) + ".ffmeta");
+                //The new url will be to the same file.
-                File.WriteAllText(metadataPath, downloadLicense.ChapterInfo.ToFFMeta(true));
+                nfsPersister.NetworkFileStream.SetUriForSameFile(new Uri(downloadLicense.DownloadUrl));
+            }
+            else
+            {
+                var headers = new System.Net.WebHeaderCollection();
+                headers.Add("User-Agent", downloadLicense.UserAgent);
+
+                NetworkFileStream networkFileStream = new NetworkFileStream(tempFile, new Uri(downloadLicense.DownloadUrl), 0, headers);
+                nfsPersister = new NetworkFileStreamPersister(networkFileStream, jsonDownloadState);
             }
 
+            string metadataPath = Path.Combine(outDir, Path.GetFileName(outputFileName) + ".ffmeta");
+
+            if (downloadLicense.ChapterInfo is null)
+            {
+                //If we want to keep the original chapters, we need to get them from the url.
+                //Ffprobe needs to seek to find metadata and it can't seek a pipe. Also, there's
+                //no guarantee that enough of the file will have been downloaded at this point
+                //to be able to use the cache file.
+                downloadLicense.ChapterInfo = new ChapterInfo(downloadLicense.DownloadUrl);
+            }
+
+            //Only write chapters to the metadata file. All other aaxc metadata will be
+            //wiped out but is restored in Step 3.                
+            File.WriteAllText(metadataPath, downloadLicense.ChapterInfo.ToFFMeta(true));
+
+
             aaxcProcesser.ProcessBook(
+                nfsPersister.NetworkFileStream,
                 outputFileName,
                 metadataPath)
                 .GetAwaiter()
                 .GetResult();
 
-            if (!userSuppliedChapters && aaxcProcesser.Succeeded)
+            nfsPersister.NetworkFileStream.Close();
-                downloadLicense.ChapterInfo = new ChapterInfo(outputFileName);
+            nfsPersister.Dispose();
 
-            if (userSuppliedChapters)
             FileExt.SafeDelete(metadataPath);
 
             DecryptProgressUpdate?.Invoke(this, 0);
@@ -255,6 +279,13 @@ namespace AaxDecrypter
             return !isCanceled;
         }
 
+        public bool Step7_Cleanup()
+        {
+            FileExt.SafeDelete(jsonDownloadState);
+            FileExt.SafeDelete(tempFile);
+            return !isCanceled;
+        }
+
         public void Cancel()
         {
             isCanceled = true;

AaxDecrypter/Chapters.cs

@@ -44,7 +44,7 @@ namespace AaxDecrypter
             var ffmetaChapters = new StringBuilder();
 
             if (includeFFMetaHeader)
-                ffmetaChapters.AppendLine(";FFMETADATA1\n");
+                ffmetaChapters.AppendLine(";FFMETADATA1");
 
             foreach (var c in Chapters)
             {

AaxDecrypter/FFMpegAaxcProcesser.cs

@@ -3,6 +3,7 @@ using System.Diagnostics;
 using System.IO;
 using System.Text;
 using System.Text.RegularExpressions;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace AaxDecrypter
@@ -31,14 +32,15 @@ namespace AaxDecrypter
         public bool IsRunning { get; private set; }
         public bool Succeeded { get; private set; }
         public string FFMpegRemuxerStandardError => remuxerError.ToString();
-        public string FFMpegDownloaderStandardError => downloaderError.ToString();
+        public string FFMpegDecrypterStandardError => decrypterError.ToString();
 
 
         private StringBuilder remuxerError { get; } = new StringBuilder();
-        private StringBuilder downloaderError { get; } = new StringBuilder();
+        private StringBuilder decrypterError { get; } = new StringBuilder();
         private static Regex processedTimeRegex { get; } = new Regex("time=(\\d{2}):(\\d{2}):(\\d{2}).\\d{2}.*speed=\\s{0,1}([0-9]*[.]?[0-9]+)(?:e\\+([0-9]+)){0,1}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
-        private Process downloader;
+        private Process decrypter;
         private Process remuxer;
+        private Stream inputFile;
         private bool isCanceled = false;
 
         public FFMpegAaxcProcesser(DownloadLicense downloadLicense)
@@ -47,11 +49,13 @@ namespace AaxDecrypter
             DownloadLicense = downloadLicense;
         }
 
-        public async Task ProcessBook(string outputFile, string ffmetaChaptersPath = null)
+        public async Task ProcessBook(Stream inputFile, string outputFile, string ffmetaChaptersPath)
         {
+            this.inputFile = inputFile;
+
             //This process gets the aaxc from the url and streams the decrypted
             //aac stream to standard output
-            downloader = new Process
+            decrypter = new Process
             {
                 StartInfo = getDownloaderStartInfo()
             };
@@ -65,64 +69,81 @@ namespace AaxDecrypter
 
             IsRunning = true;
 
-            downloader.ErrorDataReceived += Downloader_ErrorDataReceived;
+            decrypter.ErrorDataReceived += Downloader_ErrorDataReceived;
-            downloader.Start();
+            decrypter.Start();
-            downloader.BeginErrorReadLine();
+            decrypter.BeginErrorReadLine();
 
             remuxer.ErrorDataReceived += Remuxer_ErrorDataReceived;
             remuxer.Start();
             remuxer.BeginErrorReadLine();
 
-            //Thic check needs to be placed after remuxer has started
+            //Thic check needs to be placed after remuxer has started.
             if (isCanceled) return;
 
-            var pipedOutput = downloader.StandardOutput.BaseStream;
+            var decrypterInput = decrypter.StandardInput.BaseStream;
-            var pipedInput = remuxer.StandardInput.BaseStream;
+            var decrypterOutput = decrypter.StandardOutput.BaseStream;
+            var remuxerInput = remuxer.StandardInput.BaseStream;
 
+            //Read inputFile into decrypter stdin in the background
+            var t = new Thread(() => CopyStream(inputFile, decrypterInput, decrypter));           
+            t.Start();
 
             //All the work done here. Copy download standard output into
             //remuxer standard input
-            await Task.Run(() =>
+            await Task.Run(() => CopyStream(decrypterOutput, remuxerInput, remuxer));
-            {
-                int lastRead = 0;
-                byte[] buffer = new byte[32 * 1024];
-
-                do
-                {
-                    lastRead = pipedOutput.Read(buffer, 0, buffer.Length);
-                    pipedInput.Write(buffer, 0, lastRead);
-                } while (lastRead > 0 && !remuxer.HasExited);
-            });
-
-            //Closing input stream terminates remuxer
-            pipedInput.Close();
 
             //If the remuxer exited due to failure, downloader will still have
             //data in the pipe. Force kill downloader to continue.
-            if (remuxer.HasExited && !downloader.HasExited)
+            if (remuxer.HasExited && !decrypter.HasExited)
-                downloader.Kill();
+                decrypter.Kill();
 
             remuxer.WaitForExit();
-            downloader.WaitForExit();
+            decrypter.WaitForExit();
 
             IsRunning = false;
-            Succeeded = downloader.ExitCode == 0 && remuxer.ExitCode == 0;
+            Succeeded = decrypter.ExitCode == 0 && remuxer.ExitCode == 0;
         }
+
+        private void CopyStream(Stream inputStream, Stream outputStream, Process returnOnProcExit)
+        {
+            try
+            {
+                byte[] buffer = new byte[32 * 1024];
+                int lastRead;
+                do
+                {
+                    lastRead = inputStream.Read(buffer, 0, buffer.Length);
+                    outputStream.Write(buffer, 0, lastRead);
+                } while (lastRead > 0 && !returnOnProcExit.HasExited);
+            }
+            catch (IOException ex)
+            { 
+                //There is no way to tell if the process closed the input stream
+                //before trying to write to it. If it did close, throws IOException.
+                isCanceled = true; 
+            }
+            finally
+            {
+                outputStream.Close();
+            }
+        }
+
         public void Cancel()
         {
             isCanceled = true;
 
             if (IsRunning && !remuxer.HasExited)
                 remuxer.Kill();
-            if (IsRunning && !downloader.HasExited)
+            if (IsRunning && !decrypter.HasExited)
-                downloader.Kill();
+                decrypter.Kill();
+            inputFile?.Close();
         }
         private void Downloader_ErrorDataReceived(object sender, DataReceivedEventArgs e)
         {
             if (string.IsNullOrEmpty(e.Data))
                 return;
 
-            downloaderError.AppendLine(e.Data);            
+            decrypterError.AppendLine(e.Data);            
         }
 
         private void Remuxer_ErrorDataReceived(object sender, DataReceivedEventArgs e)
@@ -165,21 +186,21 @@ namespace AaxDecrypter
          {
              FileName = FFMpegPath,
              RedirectStandardError = true,
+             RedirectStandardInput = true,
              RedirectStandardOutput = true,
              CreateNoWindow = true,
              WindowStyle = ProcessWindowStyle.Hidden,
              UseShellExecute = false,
              WorkingDirectory = Path.GetDirectoryName(FFMpegPath),
              ArgumentList ={
-                    "-nostdin",
                     "-audible_key",
                     DownloadLicense.AudibleKey,
                     "-audible_iv",
                     DownloadLicense.AudibleIV,
-                    "-user_agent",
+                    "-f",
-                    DownloadLicense.UserAgent, //user-agent is requied for CDN to serve the file
+                    "mp4",
                     "-i",
-                    DownloadLicense.DownloadUrl, 
+                    "pipe:",
                     "-c:a", //audio codec
                     "copy", //copy stream
                     "-f", //force output format: adts
@@ -208,26 +229,15 @@ namespace AaxDecrypter
             startInfo.ArgumentList.Add("-i"); //read input from stdin
             startInfo.ArgumentList.Add("pipe:");
 
-            if (ffmetaChaptersPath is null)
-            {
-                //copy metadata from aaxc file.
-                startInfo.ArgumentList.Add("-user_agent");
-                startInfo.ArgumentList.Add(DownloadLicense.UserAgent);
-                startInfo.ArgumentList.Add("-i");
-                startInfo.ArgumentList.Add(DownloadLicense.DownloadUrl);
-            }
-            else
-            {
             //copy metadata from supplied metadata file
             startInfo.ArgumentList.Add("-f");
             startInfo.ArgumentList.Add("ffmetadata");
             startInfo.ArgumentList.Add("-i");
             startInfo.ArgumentList.Add(ffmetaChaptersPath);
-            }
 
             startInfo.ArgumentList.Add("-map"); //map file 0 (aac audio stream)
             startInfo.ArgumentList.Add("0");
-            startInfo.ArgumentList.Add("-map_chapters"); //copy chapter data from file 1 (either metadata file or aaxc file)
+            startInfo.ArgumentList.Add("-map_chapters"); //copy chapter data from file metadata file
             startInfo.ArgumentList.Add("1");
             startInfo.ArgumentList.Add("-c"); //copy all mapped streams
             startInfo.ArgumentList.Add("copy");

AaxDecrypter/NetworkFileStream.cs

@@ -15,13 +15,20 @@ namespace AaxDecrypter
     /// </summary>
     public class SingleUriCookieContainer : CookieContainer
     {
-        public SingleUriCookieContainer(Uri uri)
+        private Uri baseAddress;
+        public Uri Uri
         { 
-            Uri = uri;
+            get => baseAddress; 
+            set 
+            { 
+                baseAddress = new UriBuilder(value.Scheme, value.Host).Uri; 
+            } 
         }
-        public Uri Uri { get; }
 
-        public CookieCollection GetCookies() => base.GetCookies(Uri);
+        public CookieCollection GetCookies()
+        {
+            return base.GetCookies(Uri);
+        }
     }
 
     /// <summary>
@@ -43,7 +50,7 @@ namespace AaxDecrypter
         /// Http(s) address of the file to download.
         /// </summary>
         [JsonProperty(Required = Required.Always)]
-        public Uri Uri { get; }
+        public Uri Uri { get; private set; }
 
         /// <summary>
         /// All cookies set by caller or by the remote server.
@@ -73,7 +80,7 @@ namespace AaxDecrypter
 
         #region Private Properties
 
-        private HttpWebRequest HttpRequest { get; }
+        private HttpWebRequest HttpRequest { get; set; }
         private FileStream _writeFile { get; }
         private FileStream _readFile { get; }
         private Stream _networkStream { get; set; }
@@ -118,15 +125,7 @@ namespace AaxDecrypter
             Uri = uri;
             WritePosition = writePosition;
             RequestHeaders = requestHeaders ?? new WebHeaderCollection();
-            CookieContainer = cookies ?? new SingleUriCookieContainer(uri);
+            CookieContainer = cookies ?? new SingleUriCookieContainer { Uri = uri };
-
-            HttpRequest = WebRequest.CreateHttp(uri);
-
-            HttpRequest.CookieContainer = CookieContainer;
-            HttpRequest.Headers = RequestHeaders;
-            //If NetworkFileStream is resuming, Header will already contain a range.
-            HttpRequest.Headers.Remove("Range");
-            HttpRequest.AddRange(WritePosition);
             
             _writeFile = new FileStream(SaveFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite)
             {
@@ -134,6 +133,8 @@ namespace AaxDecrypter
             };
 
             _readFile = new FileStream(SaveFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+
+            SetUriForSameFile(uri);
         }
 
         #endregion
@@ -149,10 +150,33 @@ namespace AaxDecrypter
             Updated?.Invoke(this, new EventArgs());
         }
 
+        /// <summary>
+        /// Set a different <see cref="System.Uri"/> to the same file targeted by this instance of <see cref="NetworkFileStream"/>
+        /// </summary>
+        /// <param name="uriToSameFile">New <see cref="System.Uri"/> host must match existing host.</param>
+        public void SetUriForSameFile(Uri uriToSameFile)
+        {
+            ArgumentValidator.EnsureNotNullOrWhiteSpace(uriToSameFile?.AbsoluteUri, nameof(uriToSameFile));
+
+            if (uriToSameFile.Host != Uri.Host)
+                throw new ArgumentException($"New uri to the same file must have the same host.\r\n Old Host :{Uri.Host}\r\nNew Host: {uriToSameFile.Host}");
+            if (hasBegunDownloading && !finishedDownloading)
+                throw new Exception("Cannot change Uri during a download operation.");
+
+            Uri = uriToSameFile;
+            HttpRequest = WebRequest.CreateHttp(Uri);
+
+            HttpRequest.CookieContainer = CookieContainer;
+            HttpRequest.Headers = RequestHeaders;
+            //If NetworkFileStream is resuming, Header will already contain a range.
+            HttpRequest.Headers.Remove("Range");
+            HttpRequest.AddRange(WritePosition);
+        }
+
         /// <summary>
         /// Begins downloading <see cref="Uri"/> to <see cref="SaveFilePath"/> in a background thread.
         /// </summary>
-        public async Task BeginDownloading()
+        private void BeginDownloading()
         {
             if (ContentLength != 0 && WritePosition == ContentLength)
             {
@@ -164,7 +188,7 @@ namespace AaxDecrypter
             if (ContentLength != 0 && WritePosition > ContentLength)
                 throw new Exception($"Specified write position (0x{WritePosition:X10}) is larger than the file size.");
 
-            var response = await HttpRequest.GetResponseAsync() as HttpWebResponse;
+            var response = HttpRequest.GetResponse() as HttpWebResponse;
 
             if (response.StatusCode != HttpStatusCode.PartialContent)
                 throw new Exception($"Server at {Uri.Host} responded with unexpected status code: {response.StatusCode}.");
@@ -249,8 +273,9 @@ namespace AaxDecrypter
             {
                 var jObj = JObject.Load(reader);
 
-                var result = new SingleUriCookieContainer(new Uri(jObj["Uri"].Value<string>()))
+                var result = new SingleUriCookieContainer()
                 {
+                    Uri = new Uri(jObj["Uri"].Value<string>()),
                     Capacity = jObj["Capacity"].Value<int>(),
                     MaxCookieSize = jObj["MaxCookieSize"].Value<int>(),
                     PerDomainCapacity = jObj["PerDomainCapacity"].Value<int>()
@@ -360,13 +385,13 @@ namespace AaxDecrypter
         public override int Read(byte[] buffer, int offset, int count)
         {
             if (!hasBegunDownloading)
-                throw new Exception($"Must call {nameof(BeginDownloading)} before attempting to read {nameof(NetworkFileStream)};");
+                BeginDownloading();
 
             long toRead = Math.Min(count, Length - Position);
             long requiredPosition = Position + toRead;
 
             //read operation will block until file contains enough data
-            //to fulfil the request.
+            //to fulfil the request, or until cancelled.
             while (requiredPosition > WritePosition && !isCancelled)
                 Thread.Sleep(0);
 

FileLiberator/DownloadDecryptBook.cs

@@ -38,7 +38,7 @@ namespace FileLiberator
                 if (AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId))
                     return new StatusHandler { "Cannot find decrypt. Final audio file already exists" };
 
-                var outputAudioFilename = await aaxToM4bConverterDecryptAsync(AudibleFileStorage.DecryptInProgress, libraryBook);
+                var outputAudioFilename = await aaxToM4bConverterDecryptAsync(AudibleFileStorage.DownloadsInProgress, AudibleFileStorage.DecryptInProgress, libraryBook);
 
                 // decrypt failed
                 if (outputAudioFilename is null)
@@ -59,7 +59,7 @@ namespace FileLiberator
             }
         }
 
-        private async Task<string> aaxToM4bConverterDecryptAsync(string destinationDir, LibraryBook libraryBook)
+        private async Task<string> aaxToM4bConverterDecryptAsync(string cacheDir, string destinationDir, LibraryBook libraryBook)
         {
             DecryptBegin?.Invoke(this, $"Begin decrypting {libraryBook}");
 
@@ -92,7 +92,7 @@ namespace FileLiberator
                                 ));
                 }
 
-                aaxcDownloader = AaxcDownloadConverter.Create(destinationDir, aaxcDecryptDlLic);
+                aaxcDownloader = AaxcDownloadConverter.Create(cacheDir, destinationDir, aaxcDecryptDlLic);
 
                 aaxcDownloader.AppName = "Libation";