tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #46 from Mbucari/master

Browse Source 
Added resumable download support to FFMpegAaxcProcessor.
This commit is contained in:
rmcrackan 2021-07-03 21:06:25 -04:00 committed by GitHub
commit 17f3187748
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 176 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
AaxDecrypter/AaxcDownloadConverter.cs
View File

@ -32,6 +32,7 @@ namespace AaxDecrypter
bool Step4_RestoreMetadata();
bool Step5_CreateCue();
bool Step6_CreateNfo();
bool Step7_Cleanup();
}
public class AaxcDownloadConverter : IAdvancedAaxcToM4bConverter
{
@ -41,6 +42,7 @@ namespace AaxDecrypter
public event EventHandler<TimeSpan> DecryptTimeRemaining;
public string AppName { get; set; } = nameof(AaxcDownloadConverter);
public string outDir { get; private set; }
public string cacheDir { get; private set; }
public string outputFileName { get; private set; }
public DownloadLicense downloadLicense { get; private set; }
public AaxcTagLibFile aaxcTagLib { get; private set; }
@ -49,26 +51,32 @@ namespace AaxDecrypter
private StepSequence steps { get; }
private FFMpegAaxcProcesser aaxcProcesser;
private bool isCanceled { get; set; }
private string jsonDownloadState => Path.Combine(cacheDir, Path.GetFileNameWithoutExtension(outputFileName) + ".json");
private string tempFile => PathLib.ReplaceExtension(jsonDownloadState, ".aaxc");
public static AaxcDownloadConverter Create(string outDirectory, DownloadLicense dlLic)
public static AaxcDownloadConverter Create(string cacheDirectory, string outDirectory, DownloadLicense dlLic)
{
var converter = new AaxcDownloadConverter(outDirectory, dlLic);
var converter = new AaxcDownloadConverter(cacheDirectory, outDirectory, dlLic);
converter.SetOutputFilename(Path.GetTempFileName());
return converter;
}
private AaxcDownloadConverter(string outDirectory, DownloadLicense dlLic)
private AaxcDownloadConverter(string cacheDirectory, string outDirectory, DownloadLicense dlLic)
{
ArgumentValidator.EnsureNotNullOrWhiteSpace(outDirectory, nameof(outDirectory));
ArgumentValidator.EnsureNotNull(dlLic, nameof(dlLic));
if (!Directory.Exists(outDirectory))
throw new ArgumentNullException(nameof(cacheDirectory), "Directory does not exist");
if (!Directory.Exists(outDirectory))
throw new ArgumentNullException(nameof(outDirectory), "Directory does not exist");
cacheDir = cacheDirectory;
outDir = outDirectory;
steps = new StepSequence
{
Name = "Convert Aax To M4b",
Name = "Download and Convert Aaxc To M4b",
["Step 1: Create Dir"] = Step1_CreateDir,
["Step 2: Get Aaxc Metadata"] = Step2_GetMetadata,
@ -76,6 +84,7 @@ namespace AaxDecrypter
["Step 4: Restore Aaxc Metadata"] = Step4_RestoreMetadata,
["Step 5: Create Cue"] = Step5_CreateCue,
["Step 6: Create Nfo"] = Step6_CreateNfo,
["Step 7: Cleanup"] = Step7_Cleanup,
};
aaxcProcesser = new FFMpegAaxcProcesser(dlLic);
@ -130,10 +139,6 @@ namespace AaxDecrypter
//Get metadata from the file over http
NetworkFileStreamPersister nfsPersister;
string jsonDownloadState = PathLib.ReplaceExtension(outputFileName, ".json");
string tempFile = PathLib.ReplaceExtension(outputFileName, ".aaxc");
if (File.Exists(jsonDownloadState))
{
nfsPersister = new NetworkFileStreamPersister(jsonDownloadState);
@ -145,15 +150,13 @@ namespace AaxDecrypter
NetworkFileStream networkFileStream = new NetworkFileStream(tempFile, new Uri(downloadLicense.DownloadUrl), 0, headers);
nfsPersister = new NetworkFileStreamPersister(networkFileStream, jsonDownloadState);
nfsPersister.Target.BeginDownloading().GetAwaiter().GetResult();
}
var networkFile = new NetworkFileAbstraction(nfsPersister.NetworkFileStream);
aaxcTagLib = new AaxcTagLibFile(networkFile);
nfsPersister.Dispose();
aaxcTagLib = new AaxcTagLibFile(networkFile);
if (coverArt is null && aaxcTagLib.AppleTags.Pictures.Length > 0)
{
coverArt = aaxcTagLib.AppleTags.Pictures[0].Data.Data;
@ -169,29 +172,50 @@ namespace AaxDecrypter
{
DecryptProgressUpdate?.Invoke(this, int.MaxValue);
bool userSuppliedChapters = downloadLicense.ChapterInfo != null;
string metadataPath = null;
if (userSuppliedChapters)
NetworkFileStreamPersister nfsPersister;
if (File.Exists(jsonDownloadState))
{
//Only write chaopters to the metadata file. All other aaxc metadata will be
//wiped out but is restored in Step 3.
metadataPath = Path.Combine(outDir, Path.GetFileName(outputFileName) + ".ffmeta");
File.WriteAllText(metadataPath, downloadLicense.ChapterInfo.ToFFMeta(true));
nfsPersister = new NetworkFileStreamPersister(jsonDownloadState);
//If More thaan ~1 hour has elapsed since getting the download url, it will expire.
//The new url will be to the same file.
nfsPersister.NetworkFileStream.SetUriForSameFile(new Uri(downloadLicense.DownloadUrl));
}
else
{
var headers = new System.Net.WebHeaderCollection();
headers.Add("User-Agent", downloadLicense.UserAgent);
NetworkFileStream networkFileStream = new NetworkFileStream(tempFile, new Uri(downloadLicense.DownloadUrl), 0, headers);
nfsPersister = new NetworkFileStreamPersister(networkFileStream, jsonDownloadState);
}
string metadataPath = Path.Combine(outDir, Path.GetFileName(outputFileName) + ".ffmeta");
if (downloadLicense.ChapterInfo is null)
{
//If we want to keep the original chapters, we need to get them from the url.
//Ffprobe needs to seek to find metadata and it can't seek a pipe. Also, there's
//no guarantee that enough of the file will have been downloaded at this point
//to be able to use the cache file.
downloadLicense.ChapterInfo = new ChapterInfo(downloadLicense.DownloadUrl);
}
//Only write chapters to the metadata file. All other aaxc metadata will be
//wiped out but is restored in Step 3.
File.WriteAllText(metadataPath, downloadLicense.ChapterInfo.ToFFMeta(true));
aaxcProcesser.ProcessBook(
nfsPersister.NetworkFileStream,
outputFileName,
metadataPath)
.GetAwaiter()
.GetResult();
if (!userSuppliedChapters && aaxcProcesser.Succeeded)
downloadLicense.ChapterInfo = new ChapterInfo(outputFileName);
nfsPersister.NetworkFileStream.Close();
nfsPersister.Dispose();
if (userSuppliedChapters)
FileExt.SafeDelete(metadataPath);
FileExt.SafeDelete(metadataPath);
DecryptProgressUpdate?.Invoke(this, 0);
@ -255,6 +279,13 @@ namespace AaxDecrypter
return !isCanceled;
}
public bool Step7_Cleanup()
{
FileExt.SafeDelete(jsonDownloadState);
FileExt.SafeDelete(tempFile);
return !isCanceled;
}
public void Cancel()
{
isCanceled = true;

10
AaxDecrypter/Chapters.cs
View File

@ -15,7 +15,7 @@ namespace AaxDecrypter
public IEnumerable<Chapter> Chapters => _chapterList.AsEnumerable();
public int Count => _chapterList.Count;
public ChapterInfo() { }
public ChapterInfo(string audiobookFile)
public ChapterInfo(string audiobookFile)
{
var info = new ProcessStartInfo
{
@ -27,7 +27,7 @@ namespace AaxDecrypter
var chapterJObject = JObject.Parse(jString);
var chapters = chapterJObject["chapters"]
.Select(c => new Chapter(
c["tags"]?["title"]?.Value<string>(),
c["tags"]?["title"]?.Value<string>(),
c["start_time"].Value<double>(),
c["end_time"].Value<double>()
));
@ -44,7 +44,7 @@ namespace AaxDecrypter
var ffmetaChapters = new StringBuilder();
if (includeFFMetaHeader)
ffmetaChapters.AppendLine(";FFMETADATA1\n");
ffmetaChapters.AppendLine(";FFMETADATA1");
foreach (var c in Chapters)
{
@ -66,10 +66,10 @@ namespace AaxDecrypter
Title = title;
StartOffset = TimeSpan.FromMilliseconds(startOffsetMs);
EndOffset = StartOffset + TimeSpan.FromMilliseconds(lengthMs);
EndOffset = StartOffset + TimeSpan.FromMilliseconds(lengthMs);
}
public Chapter(string title, double startTimeSec, double endTimeSec)
:this(title, (long)(startTimeSec * 1000), (long)((endTimeSec - startTimeSec) * 1000))
: this(title, (long)(startTimeSec * 1000), (long)((endTimeSec - startTimeSec) * 1000))
{
}

118
AaxDecrypter/FFMpegAaxcProcesser.cs
View File

@ -3,6 +3,7 @@ using System.Diagnostics;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
namespace AaxDecrypter
@ -31,27 +32,30 @@ namespace AaxDecrypter
public bool IsRunning { get; private set; }
public bool Succeeded { get; private set; }
public string FFMpegRemuxerStandardError => remuxerError.ToString();
public string FFMpegDownloaderStandardError => downloaderError.ToString();
public string FFMpegDecrypterStandardError => decrypterError.ToString();
private StringBuilder remuxerError { get; } = new StringBuilder();
private StringBuilder downloaderError { get; } = new StringBuilder();
private StringBuilder decrypterError { get; } = new StringBuilder();
private static Regex processedTimeRegex { get; } = new Regex("time=(\\d{2}):(\\d{2}):(\\d{2}).\\d{2}.*speed=\\s{0,1}([0-9]*[.]?[0-9]+)(?:e\\+([0-9]+)){0,1}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private Process downloader;
private Process decrypter;
private Process remuxer;
private Stream inputFile;
private bool isCanceled = false;
public FFMpegAaxcProcesser( DownloadLicense downloadLicense)
public FFMpegAaxcProcesser(DownloadLicense downloadLicense)
{
FFMpegPath = DecryptSupportLibraries.ffmpegPath;
DownloadLicense = downloadLicense;
}
public async Task ProcessBook(string outputFile, string ffmetaChaptersPath = null)
public async Task ProcessBook(Stream inputFile, string outputFile, string ffmetaChaptersPath)
{
this.inputFile = inputFile;
//This process gets the aaxc from the url and streams the decrypted
//aac stream to standard output
downloader = new Process
decrypter = new Process
{
StartInfo = getDownloaderStartInfo()
};
@ -65,64 +69,81 @@ namespace AaxDecrypter
IsRunning = true;
downloader.ErrorDataReceived += Downloader_ErrorDataReceived;
downloader.Start();
downloader.BeginErrorReadLine();
decrypter.ErrorDataReceived += Downloader_ErrorDataReceived;
decrypter.Start();
decrypter.BeginErrorReadLine();
remuxer.ErrorDataReceived += Remuxer_ErrorDataReceived;
remuxer.Start();
remuxer.BeginErrorReadLine();
//Thic check needs to be placed after remuxer has started
//Thic check needs to be placed after remuxer has started.
if (isCanceled) return;
var pipedOutput = downloader.StandardOutput.BaseStream;
var pipedInput = remuxer.StandardInput.BaseStream;
var decrypterInput = decrypter.StandardInput.BaseStream;
var decrypterOutput = decrypter.StandardOutput.BaseStream;
var remuxerInput = remuxer.StandardInput.BaseStream;
//Read inputFile into decrypter stdin in the background
var t = new Thread(() => CopyStream(inputFile, decrypterInput, decrypter));
t.Start();
//All the work done here. Copy download standard output into
//remuxer standard input
await Task.Run(() =>
{
int lastRead = 0;
byte[] buffer = new byte[32 * 1024];
do
{
lastRead = pipedOutput.Read(buffer, 0, buffer.Length);
pipedInput.Write(buffer, 0, lastRead);
} while (lastRead > 0 && !remuxer.HasExited);
});
//Closing input stream terminates remuxer
pipedInput.Close();
await Task.Run(() => CopyStream(decrypterOutput, remuxerInput, remuxer));
//If the remuxer exited due to failure, downloader will still have
//data in the pipe. Force kill downloader to continue.
if (remuxer.HasExited && !downloader.HasExited)
downloader.Kill();
if (remuxer.HasExited && !decrypter.HasExited)
decrypter.Kill();
remuxer.WaitForExit();
downloader.WaitForExit();
decrypter.WaitForExit();
IsRunning = false;
Succeeded = downloader.ExitCode == 0 && remuxer.ExitCode == 0;
Succeeded = decrypter.ExitCode == 0 && remuxer.ExitCode == 0;
}
private void CopyStream(Stream inputStream, Stream outputStream, Process returnOnProcExit)
{
try
{
byte[] buffer = new byte[32 * 1024];
int lastRead;
do
{
lastRead = inputStream.Read(buffer, 0, buffer.Length);
outputStream.Write(buffer, 0, lastRead);
} while (lastRead > 0 && !returnOnProcExit.HasExited);
}
catch (IOException ex)
{
//There is no way to tell if the process closed the input stream
//before trying to write to it. If it did close, throws IOException.
isCanceled = true;
}
finally
{
outputStream.Close();
}
}
public void Cancel()
{
isCanceled = true;
if (IsRunning && !remuxer.HasExited)
remuxer.Kill();
if (IsRunning && !downloader.HasExited)
downloader.Kill();
if (IsRunning && !decrypter.HasExited)
decrypter.Kill();
inputFile?.Close();
}
private void Downloader_ErrorDataReceived(object sender, DataReceivedEventArgs e)
{
if (string.IsNullOrEmpty(e.Data))
return;
downloaderError.AppendLine(e.Data);
decrypterError.AppendLine(e.Data);
}
private void Remuxer_ErrorDataReceived(object sender, DataReceivedEventArgs e)
@ -165,21 +186,21 @@ namespace AaxDecrypter
{
FileName = FFMpegPath,
RedirectStandardError = true,
RedirectStandardInput = true,
RedirectStandardOutput = true,
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = false,
WorkingDirectory = Path.GetDirectoryName(FFMpegPath),
ArgumentList ={
"-nostdin",
"-audible_key",
DownloadLicense.AudibleKey,
"-audible_iv",
DownloadLicense.AudibleIV,
"-user_agent",
DownloadLicense.UserAgent, //user-agent is requied for CDN to serve the file
"-f",
"mp4",
"-i",
DownloadLicense.DownloadUrl,
"pipe:",
"-c:a", //audio codec
"copy", //copy stream
"-f", //force output format: adts
@ -208,26 +229,15 @@ namespace AaxDecrypter
startInfo.ArgumentList.Add("-i"); //read input from stdin
startInfo.ArgumentList.Add("pipe:");
if (ffmetaChaptersPath is null)
{
//copy metadata from aaxc file.
startInfo.ArgumentList.Add("-user_agent");
startInfo.ArgumentList.Add(DownloadLicense.UserAgent);
startInfo.ArgumentList.Add("-i");
startInfo.ArgumentList.Add(DownloadLicense.DownloadUrl);
}
else
{
//copy metadata from supplied metadata file
startInfo.ArgumentList.Add("-f");
startInfo.ArgumentList.Add("ffmetadata");
startInfo.ArgumentList.Add("-i");
startInfo.ArgumentList.Add(ffmetaChaptersPath);
}
//copy metadata from supplied metadata file
startInfo.ArgumentList.Add("-f");
startInfo.ArgumentList.Add("ffmetadata");
startInfo.ArgumentList.Add("-i");
startInfo.ArgumentList.Add(ffmetaChaptersPath);
startInfo.ArgumentList.Add("-map"); //map file 0 (aac audio stream)
startInfo.ArgumentList.Add("0");
startInfo.ArgumentList.Add("-map_chapters"); //copy chapter data from file 1 (either metadata file or aaxc file)
startInfo.ArgumentList.Add("-map_chapters"); //copy chapter data from file metadata file
startInfo.ArgumentList.Add("1");
startInfo.ArgumentList.Add("-c"); //copy all mapped streams
startInfo.ArgumentList.Add("copy");

71
AaxDecrypter/NetworkFileStream.cs
View File

@ -15,13 +15,20 @@ namespace AaxDecrypter
/// </summary>
public class SingleUriCookieContainer : CookieContainer
{
public SingleUriCookieContainer(Uri uri)
{
Uri = uri;
private Uri baseAddress;
public Uri Uri
{
get => baseAddress;
set
{
baseAddress = new UriBuilder(value.Scheme, value.Host).Uri;
}
}
public Uri Uri { get; }
public CookieCollection GetCookies() => base.GetCookies(Uri);
public CookieCollection GetCookies()
{
return base.GetCookies(Uri);
}
}
/// <summary>
@ -43,7 +50,7 @@ namespace AaxDecrypter
/// Http(s) address of the file to download.
/// </summary>
[JsonProperty(Required = Required.Always)]
public Uri Uri { get; }
public Uri Uri { get; private set; }
/// <summary>
/// All cookies set by caller or by the remote server.
@ -73,7 +80,7 @@ namespace AaxDecrypter
#region Private Properties
private HttpWebRequest HttpRequest { get; }
private HttpWebRequest HttpRequest { get; set; }
private FileStream _writeFile { get; }
private FileStream _readFile { get; }
private Stream _networkStream { get; set; }
@ -118,26 +125,20 @@ namespace AaxDecrypter
Uri = uri;
WritePosition = writePosition;
RequestHeaders = requestHeaders ?? new WebHeaderCollection();
CookieContainer = cookies ?? new SingleUriCookieContainer(uri);
HttpRequest = WebRequest.CreateHttp(uri);
HttpRequest.CookieContainer = CookieContainer;
HttpRequest.Headers = RequestHeaders;
//If NetworkFileStream is resuming, Header will already contain a range.
HttpRequest.Headers.Remove("Range");
HttpRequest.AddRange(WritePosition);
CookieContainer = cookies ?? new SingleUriCookieContainer { Uri = uri };
_writeFile = new FileStream(SaveFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite)
{
Position = WritePosition
};
_readFile = new FileStream(SaveFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
SetUriForSameFile(uri);
}
#endregion
#region Downloader
/// <summary>
@ -149,10 +150,33 @@ namespace AaxDecrypter
Updated?.Invoke(this, new EventArgs());
}
/// <summary>
/// Set a different <see cref="System.Uri"/> to the same file targeted by this instance of <see cref="NetworkFileStream"/>
/// </summary>
/// <param name="uriToSameFile">New <see cref="System.Uri"/> host must match existing host.</param>
public void SetUriForSameFile(Uri uriToSameFile)
{
ArgumentValidator.EnsureNotNullOrWhiteSpace(uriToSameFile?.AbsoluteUri, nameof(uriToSameFile));
if (uriToSameFile.Host != Uri.Host)
throw new ArgumentException($"New uri to the same file must have the same host.\r\n Old Host :{Uri.Host}\r\nNew Host: {uriToSameFile.Host}");
if (hasBegunDownloading && !finishedDownloading)
throw new Exception("Cannot change Uri during a download operation.");
Uri = uriToSameFile;
HttpRequest = WebRequest.CreateHttp(Uri);
HttpRequest.CookieContainer = CookieContainer;
HttpRequest.Headers = RequestHeaders;
//If NetworkFileStream is resuming, Header will already contain a range.
HttpRequest.Headers.Remove("Range");
HttpRequest.AddRange(WritePosition);
}
/// <summary>
/// Begins downloading <see cref="Uri"/> to <see cref="SaveFilePath"/> in a background thread.
/// </summary>
public async Task BeginDownloading()
private void BeginDownloading()
{
if (ContentLength != 0 && WritePosition == ContentLength)
{
@ -164,7 +188,7 @@ namespace AaxDecrypter
if (ContentLength != 0 && WritePosition > ContentLength)
throw new Exception($"Specified write position (0x{WritePosition:X10}) is larger than the file size.");
var response = await HttpRequest.GetResponseAsync() as HttpWebResponse;
var response = HttpRequest.GetResponse() as HttpWebResponse;
if (response.StatusCode != HttpStatusCode.PartialContent)
throw new Exception($"Server at {Uri.Host} responded with unexpected status code: {response.StatusCode}.");
@ -249,8 +273,9 @@ namespace AaxDecrypter
{
var jObj = JObject.Load(reader);
var result = new SingleUriCookieContainer(new Uri(jObj["Uri"].Value<string>()))
var result = new SingleUriCookieContainer()
{
Uri = new Uri(jObj["Uri"].Value<string>()),
Capacity = jObj["Capacity"].Value<int>(),
MaxCookieSize = jObj["MaxCookieSize"].Value<int>(),
PerDomainCapacity = jObj["PerDomainCapacity"].Value<int>()
@ -360,13 +385,13 @@ namespace AaxDecrypter
public override int Read(byte[] buffer, int offset, int count)
{
if (!hasBegunDownloading)
throw new Exception($"Must call {nameof(BeginDownloading)} before attempting to read {nameof(NetworkFileStream)};");
BeginDownloading();
long toRead = Math.Min(count, Length - Position);
long requiredPosition = Position + toRead;
//read operation will block until file contains enough data
//to fulfil the request.
//to fulfil the request, or until cancelled.
while (requiredPosition > WritePosition && !isCancelled)
Thread.Sleep(0);

6
FileLiberator/DownloadDecryptBook.cs
View File

@ -38,7 +38,7 @@ namespace FileLiberator
if (AudibleFileStorage.Audio.Exists(libraryBook.Book.AudibleProductId))
return new StatusHandler { "Cannot find decrypt. Final audio file already exists" };
var outputAudioFilename = await aaxToM4bConverterDecryptAsync(AudibleFileStorage.DecryptInProgress, libraryBook);
var outputAudioFilename = await aaxToM4bConverterDecryptAsync(AudibleFileStorage.DownloadsInProgress, AudibleFileStorage.DecryptInProgress, libraryBook);
// decrypt failed
if (outputAudioFilename is null)
@ -59,7 +59,7 @@ namespace FileLiberator
}
}
private async Task<string> aaxToM4bConverterDecryptAsync(string destinationDir, LibraryBook libraryBook)
private async Task<string> aaxToM4bConverterDecryptAsync(string cacheDir, string destinationDir, LibraryBook libraryBook)
{
DecryptBegin?.Invoke(this, $"Begin decrypting {libraryBook}");
@ -92,7 +92,7 @@ namespace FileLiberator
));
}
aaxcDownloader = AaxcDownloadConverter.Create(destinationDir, aaxcDecryptDlLic);
aaxcDownloader = AaxcDownloadConverter.Create(cacheDir, destinationDir, aaxcDecryptDlLic);
aaxcDownloader.AppName = "Libation";