diff --git a/AaxDecrypter/AaxcDownloadConverter.cs b/AaxDecrypter/AaxcDownloadConverter.cs index 2944c9d7..87629279 100644 --- a/AaxDecrypter/AaxcDownloadConverter.cs +++ b/AaxDecrypter/AaxcDownloadConverter.cs @@ -128,12 +128,32 @@ namespace AaxDecrypter public bool Step2_GetMetadata() { //Get metadata from the file over http - var client = new System.Net.Http.HttpClient(); - client.DefaultRequestHeaders.Add("User-Agent", downloadLicense.UserAgent); - var networkFile = NetworkFileAbstraction.CreateAsync(client, new Uri(downloadLicense.DownloadUrl)).GetAwaiter().GetResult(); + + NetworkFileStreamPersister nfsPersister; + + string jsonDownloadState = PathLib.ReplaceExtension(outputFileName, ".json"); + string tempFile = PathLib.ReplaceExtension(outputFileName, ".aaxc"); + + if (File.Exists(jsonDownloadState)) + { + nfsPersister = new NetworkFileStreamPersister(jsonDownloadState); + } + else + { + var headers = new System.Net.WebHeaderCollection(); + headers.Add("User-Agent", downloadLicense.UserAgent); + + NetworkFileStream networkFileStream = new NetworkFileStream(tempFile, new Uri(downloadLicense.DownloadUrl), 0, headers); + nfsPersister = new NetworkFileStreamPersister(networkFileStream, jsonDownloadState); + nfsPersister.Target.BeginDownloading().GetAwaiter().GetResult(); + } + + var networkFile = new NetworkFileAbstraction(nfsPersister.Target); + + nfsPersister.Dispose(); aaxcTagLib = new AaxcTagLibFile(networkFile); - + if (coverArt is null && aaxcTagLib.AppleTags.Pictures.Length > 0) { coverArt = aaxcTagLib.AppleTags.Pictures[0].Data.Data; diff --git a/AaxDecrypter/NetworkFileAbstraction.cs b/AaxDecrypter/NetworkFileAbstraction.cs index 136236ca..60f52a9f 100644 --- a/AaxDecrypter/NetworkFileAbstraction.cs +++ b/AaxDecrypter/NetworkFileAbstraction.cs @@ -12,26 +12,10 @@ namespace AaxDecrypter { private NetworkFileStream aaxNetworkStream; - public static async Task CreateAsync(HttpClient client, Uri webFileUri) + public NetworkFileAbstraction( NetworkFileStream networkFileStream) { - var response = await client.GetAsync(webFileUri, HttpCompletionOption.ResponseHeadersRead); - - if (response.StatusCode != System.Net.HttpStatusCode.OK) - throw new Exception("Can't read file from client."); - - var contentLength = response.Content.Headers.ContentLength ?? 0; - - var networkStream = await response.Content.ReadAsStreamAsync(); - - var networkFile = new NetworkFileAbstraction(Path.GetFileName(webFileUri.LocalPath), networkStream, contentLength); - - return networkFile; - } - - private NetworkFileAbstraction(string fileName, Stream netStream, long contentLength) - { - Name = fileName; - aaxNetworkStream = new NetworkFileStream(netStream, contentLength); + Name = networkFileStream.SaveFilePath; + aaxNetworkStream = networkFileStream; } public string Name { get; private set; } @@ -43,93 +27,5 @@ namespace AaxDecrypter { aaxNetworkStream.Close(); } - - private class NetworkFileStream : Stream - { - private const int BUFF_SZ = 2 * 1024; - - private FileStream _fileBacker; - - private Stream _networkStream; - - private long networkBytesRead = 0; - - private long _contentLength; - public NetworkFileStream(Stream netStream, long contentLength) - { - _networkStream = netStream; - _contentLength = contentLength; - _fileBacker = File.Create(Path.GetTempFileName(), BUFF_SZ, FileOptions.DeleteOnClose); - } - public override bool CanRead => true; - - public override bool CanSeek => true; - - public override bool CanWrite => false; - - public override long Length => _contentLength; - - public override long Position { get => _fileBacker.Position; set => Seek(value, 0); } - - public override void Flush() - { - throw new NotImplementedException(); - } - public override void SetLength(long value) - { - throw new NotImplementedException(); - } - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotImplementedException(); - } - public override int Read(byte[] buffer, int offset, int count) - { - long requiredLength = Position + count; - - if (requiredLength > networkBytesRead) - readWebFileToPosition(requiredLength); - - return _fileBacker.Read(buffer, offset, count); - } - - public override long Seek(long offset, SeekOrigin origin) - { - long newPosition = (long)origin + offset; - - if (newPosition > networkBytesRead) - readWebFileToPosition(newPosition); - - _fileBacker.Position = newPosition; - return newPosition; - } - - public override void Close() - { - _fileBacker.Close(); - _networkStream.Close(); - } - /// - /// Read more data from into as needed. - /// - /// Length of strem required for the operation. - private void readWebFileToPosition(long requiredLength) - { - byte[] buff = new byte[BUFF_SZ]; - - long backerPosition = _fileBacker.Position; - - _fileBacker.Position = networkBytesRead; - - while (networkBytesRead < requiredLength) - { - int bytesRead = _networkStream.Read(buff, 0, BUFF_SZ); - _fileBacker.Write(buff, 0, bytesRead); - networkBytesRead += bytesRead; - } - - _fileBacker.Position = backerPosition; - } - } } } diff --git a/AaxDecrypter/NetworkFileStream.cs b/AaxDecrypter/NetworkFileStream.cs new file mode 100644 index 00000000..bb25300b --- /dev/null +++ b/AaxDecrypter/NetworkFileStream.cs @@ -0,0 +1,431 @@ +using Dinah.Core; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace AaxDecrypter +{ + /// + /// A for a single Uri. + /// + public class SingleUriCookieContainer : CookieContainer + { + public SingleUriCookieContainer(Uri uri) + { + Uri = uri; + } + public Uri Uri { get; } + + public CookieCollection GetCookies() => base.GetCookies(Uri); + } + + /// + /// A resumable, simultaneous file downloader and reader. + /// + public class NetworkFileStream : Stream, IUpdatable + { + public event EventHandler Updated; + + #region Public Properties + + /// + /// Location to save the downloaded data. + /// + [JsonProperty(Required = Required.Always)] + public string SaveFilePath { get; } + + /// + /// Http(s) address of the file to download. + /// + [JsonProperty(Required = Required.Always)] + public Uri Uri { get; } + + /// + /// All cookies set by caller or by the remote server. + /// + [JsonProperty(Required = Required.Always)] + public SingleUriCookieContainer CookieContainer { get; } + + /// + /// Http headers to be sent to the server with the request. + /// + [JsonProperty(Required = Required.Always)] + public WebHeaderCollection RequestHeaders { get; private set; } + + /// + /// The position in that has been written and flushed to disk. + /// + [JsonProperty(Required = Required.Always)] + public long WritePosition { get; private set; } + + /// + /// The total length of the file to download. + /// + [JsonProperty(Required = Required.Always)] + public long ContentLength { get; private set; } + + #endregion + + #region Private Properties + + private HttpWebRequest HttpRequest { get; } + private FileStream _writeFile { get; } + private FileStream _readFile { get; } + private Stream _networkStream { get; set; } + private bool hasBegunDownloading { get; set; } + private bool isCancelled { get; set; } + private Action cancelDownloadCallback { get; set; } + + #endregion + + #region Constants + + //Download buffer size + private const int DOWNLOAD_BUFF_SZ = 4 * 1024; + + //NetworkFileStream will flush all data in _writeFile to disk after every + //DATA_FLUSH_SZ bytes are written to the file stream. + private const int DATA_FLUSH_SZ = 1024 * 1024; + + #endregion + + #region Constructor + + /// + /// A resumable, simultaneous file downloader and reader. + /// + /// Path to a location on disk to save the downloaded data from + /// Http(s) address of the file to download. + /// The position in to begin downloading. + /// Http headers to be sent to the server with the . + /// A with cookies to send with the . It will also be populated with any cookies set by the server. + public NetworkFileStream(string saveFilePath, Uri uri, long writePosition = 0, WebHeaderCollection requestHeaders = null, SingleUriCookieContainer cookies = null) + { + ArgumentValidator.EnsureNotNullOrWhiteSpace(saveFilePath, nameof(saveFilePath)); + ArgumentValidator.EnsureNotNullOrWhiteSpace(uri?.AbsoluteUri, nameof(uri)); + ArgumentValidator.EnsureGreaterThan(writePosition, nameof(writePosition), -1); + + if (!Directory.Exists(Path.GetDirectoryName(saveFilePath))) + throw new ArgumentException($"Specified {nameof(saveFilePath)} directory \"{Path.GetDirectoryName(saveFilePath)}\" does not exist."); + + SaveFilePath = saveFilePath; + Uri = uri; + WritePosition = writePosition; + RequestHeaders = requestHeaders ?? new WebHeaderCollection(); + CookieContainer = cookies ?? new SingleUriCookieContainer(uri); + + HttpRequest = WebRequest.CreateHttp(uri); + + HttpRequest.CookieContainer = CookieContainer; + HttpRequest.Headers = RequestHeaders; + //If NetworkFileStream is resuming, Header will already contain a range. + HttpRequest.Headers.Remove("Range"); + HttpRequest.AddRange(WritePosition); + + _writeFile = new FileStream(SaveFilePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite) + { + Position = WritePosition + }; + + _readFile = new FileStream(SaveFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + + #endregion + + #region Downloader + + /// + /// Update the . + /// + private void Update() + { + RequestHeaders = HttpRequest.Headers; + Updated?.Invoke(this, new EventArgs()); + } + + /// + /// Begins downloading to in a background thread. + /// + public async Task BeginDownloading() + { + if (ContentLength != 0 && WritePosition == ContentLength) + return; + + if (ContentLength != 0 && WritePosition > ContentLength) + throw new Exception($"Specified write position (0x{WritePosition:X10}) is larger than the file size."); + + var response = await HttpRequest.GetResponseAsync() as HttpWebResponse; + + if (response.StatusCode != HttpStatusCode.PartialContent) + throw new Exception($"Server at {Uri.Host} responded with unexpected status code: {response.StatusCode}."); + + if (response.Headers.GetValues("Accept-Ranges").FirstOrDefault(r => r.EqualsInsensitive("bytes")) is null) + throw new Exception($"Server at {Uri.Host} does not support Http ranges"); + + //Content length is the length of the range request, and it is only equal + //to the complete file length if requesting Range: bytes=0- + if (WritePosition == 0) + ContentLength = response.ContentLength; + + _networkStream = response.GetResponseStream(); + + //Download the file in the background. + Thread downloadThread = new Thread(() => DownloadFile()); + downloadThread.Start(); + + while(!File.Exists(SaveFilePath) && new FileInfo(SaveFilePath).Length > 1000) + { + Thread.Sleep(100); + } + + hasBegunDownloading = true; + return; + } + + /// + /// Downlod to . + /// + private void DownloadFile() + { + long downloadPosition = WritePosition; + long nextFlush = downloadPosition + DATA_FLUSH_SZ; + + byte[] buff = new byte[DOWNLOAD_BUFF_SZ]; + do + { + int bytesRead = _networkStream.Read(buff, 0, DOWNLOAD_BUFF_SZ); + _writeFile.Write(buff, 0, bytesRead); + + downloadPosition += bytesRead; + + if (downloadPosition > nextFlush) + { + _writeFile.Flush(); + WritePosition = downloadPosition; + Update(); + + nextFlush = downloadPosition + DATA_FLUSH_SZ; + } + + } while (downloadPosition < ContentLength && !isCancelled); + + _writeFile.Close(); + WritePosition = downloadPosition; + Update(); + _networkStream.Close(); + + if (!isCancelled && WritePosition < ContentLength) + throw new Exception("File download ended before finishing."); + + if (WritePosition > ContentLength) + throw new Exception("Downloaded file is larger than expected."); + + cancelDownloadCallback?.Invoke(); + } + + #endregion + + #region Json Connverters + + public static JsonSerializerSettings GetJsonSerializerSettings() + { + var settings = new JsonSerializerSettings(); + settings.Converters.Add(new CookieContainerConverter()); + settings.Converters.Add(new WebHeaderCollectionConverter()); + return settings; + } + + internal class CookieContainerConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + => objectType == typeof(SingleUriCookieContainer); + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var jObj = JObject.Load(reader); + + var result = new SingleUriCookieContainer(new Uri(jObj["Uri"].Value())) + { + Capacity = jObj["Capacity"].Value(), + MaxCookieSize = jObj["MaxCookieSize"].Value(), + PerDomainCapacity = jObj["PerDomainCapacity"].Value() + }; + + var cookieList = jObj["Cookies"].ToList(); + + foreach (var cookie in cookieList) + { + result.Add( + new Cookie + { + Comment = cookie["Comment"].Value(), + HttpOnly = cookie["HttpOnly"].Value(), + Discard = cookie["Discard"].Value(), + Domain = cookie["Domain"].Value(), + Expired = cookie["Expired"].Value(), + Expires = cookie["Expires"].Value(), + Name = cookie["Name"].Value(), + Path = cookie["Path"].Value(), + Port = cookie["Port"].Value(), + Secure = cookie["Secure"].Value(), + Value = cookie["Value"].Value(), + Version = cookie["Version"].Value(), + }); + } + + return result; + } + + public override bool CanWrite => true; + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var cookies = value as SingleUriCookieContainer; + var obj = (JObject)JToken.FromObject(value); + var container = cookies.GetCookies(); + var propertyNames = container.Select(c => JToken.FromObject(c)); + obj.AddFirst(new JProperty("Cookies", new JArray(propertyNames))); + obj.WriteTo(writer); + } + } + + internal class WebHeaderCollectionConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + => objectType == typeof(WebHeaderCollection); + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var jObj = JObject.Load(reader); + var result = new WebHeaderCollection(); + + foreach (var kvp in jObj) + { + result.Add(kvp.Key, kvp.Value.Value()); + } + + return result; + } + + public override bool CanWrite => true; + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + JObject jObj = new JObject(); + Type type = value.GetType(); + var headers = value as WebHeaderCollection; + var jHeaders = headers.AllKeys.Select(k => new JProperty(k, headers[k])); + jObj.Add(jHeaders); + jObj.WriteTo(writer); + } + } + + #endregion + + #region Download Stream Reader + + [JsonIgnore] + public override bool CanRead => true; + + [JsonIgnore] + public override bool CanSeek => true; + + [JsonIgnore] + public override bool CanWrite => false; + + [JsonIgnore] + public override long Length => ContentLength; + + [JsonIgnore] + public override long Position { get => _readFile.Position; set => Seek(value, SeekOrigin.Begin); } + + [JsonIgnore] + public override bool CanTimeout => base.CanTimeout; + + [JsonIgnore] + public override int ReadTimeout { get => base.ReadTimeout; set => base.ReadTimeout = value; } + + [JsonIgnore] + public override int WriteTimeout { get => base.WriteTimeout; set => base.WriteTimeout = value; } + + public override void Flush() => throw new NotImplementedException(); + public override void SetLength(long value) => throw new NotImplementedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + + public override int Read(byte[] buffer, int offset, int count) + { + if (!hasBegunDownloading) + throw new Exception($"Must call {nameof(BeginDownloading)} before attempting to read {nameof(NetworkFileStream)};"); + + //read operation will block until file contains enough data + //to fulfil the request. + return _readFile.Read(buffer, offset, count); + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + if (!hasBegunDownloading) + throw new Exception($"Must call {nameof(BeginDownloading)} before attempting to read {nameof(NetworkFileStream)};"); + + //read operation will block until file contains enough data + //to fulfil the request. + return await _readFile.ReadAsync(buffer, offset, count, cancellationToken); + } + + public override long Seek(long offset, SeekOrigin origin) + { + long newPosition; + + switch (origin) + { + case SeekOrigin.Current: + newPosition = Position + offset; + break; + case SeekOrigin.End: + newPosition = ContentLength + offset; + break; + default: + newPosition = offset; + break; + } + + ReadToPosition(newPosition); + + _readFile.Position = newPosition; + return newPosition; + } + + /// + /// Ensures that the file has downloaded to at least , then returns. + /// + /// The minimum required data length in . + private void ReadToPosition(long neededPosition) + { + long totalBytesRead = _readFile.Position; + + byte[] buff = new byte[DOWNLOAD_BUFF_SZ]; + do + { + totalBytesRead += Read(buff, 0, DOWNLOAD_BUFF_SZ); + } while (totalBytesRead < neededPosition); + } + public override void Close() + { + isCancelled = true; + cancelDownloadCallback = () => + { + _readFile.Close(); + _writeFile.Close(); + _networkStream?.Close(); + Update(); + }; + } + + #endregion + } +} diff --git a/AaxDecrypter/NetworkFileStreamPersister.cs b/AaxDecrypter/NetworkFileStreamPersister.cs new file mode 100644 index 00000000..fc79a5c6 --- /dev/null +++ b/AaxDecrypter/NetworkFileStreamPersister.cs @@ -0,0 +1,23 @@ +using Dinah.Core.IO; +using Newtonsoft.Json; + +namespace AaxDecrypter +{ + internal class NetworkFileStreamPersister : JsonFilePersister + { + + /// Alias for Target + public NetworkFileStream Identity => Target; + + /// uses path. create file if doesn't yet exist + public NetworkFileStreamPersister(NetworkFileStream networkFileStream, string path, string jsonPath = null) + : base(networkFileStream, path, jsonPath) { } + + /// load from existing file + public NetworkFileStreamPersister(string path, string jsonPath = null) + : base(path, jsonPath) { } + + protected override JsonSerializerSettings GetSerializerSettings() => NetworkFileStream.GetJsonSerializerSettings(); + + } +}