using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading.Channels;
using System.Threading.Tasks;
using System.Diagnostics;
using AudibleApi;
using AudibleApi.Common;
using Dinah.Core;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Polly;
using Polly.Retry;
namespace AudibleUtilities
{
/// USE THIS from within Libation. It wraps the call with correct JSONPath
public class ApiExtended
{
public Api Api { get; private set; }
private const int MaxConcurrency = 10;
private const int BatchSize = 50;
private ApiExtended(Api api) => Api = api;
/// Get api from existing tokens else login with 'eager' choice. External browser url is provided. Response can be external browser login or continuing with native api callbacks.
public static async Task CreateAsync(Account account, ILoginChoiceEager loginChoiceEager)
{
Serilog.Log.Logger.Information("{@DebugInfo}", new
{
LoginType = nameof(ILoginChoiceEager),
Account = account?.MaskedLogEntry ?? "[null]",
LocaleName = account?.Locale?.Name
});
var api = await EzApiCreator.GetApiAsync(
loginChoiceEager,
account.Locale,
AudibleApiStorage.AccountsSettingsFile,
account.GetIdentityTokensJsonPath());
return new ApiExtended(api);
}
/// Get api from existing tokens. Assumes you have valid login tokens. Else exception
public static async Task CreateAsync(Account account)
{
ArgumentValidator.EnsureNotNull(account, nameof(account));
ArgumentValidator.EnsureNotNull(account.Locale, nameof(account.Locale));
Serilog.Log.Logger.Information("{@DebugInfo}", new
{
AccountMaskedLogEntry = account.MaskedLogEntry
});
return await CreateAsync(account.AccountId, account.Locale.Name);
}
/// Get api from existing tokens. Assumes you have valid login tokens. Else exception
public static async Task CreateAsync(string username, string localeName)
{
Serilog.Log.Logger.Information("{@DebugInfo}", new
{
Username = username.ToMask(),
LocaleName = localeName,
});
var api = await EzApiCreator.GetApiAsync(
Localization.Get(localeName),
AudibleApiStorage.AccountsSettingsFile,
AudibleApiStorage.GetIdentityTokensJsonPath(username, localeName));
return new ApiExtended(api);
}
private static AsyncRetryPolicy policy { get; }
= Policy.Handle()
// 2 retries == 3 total
.RetryAsync(2);
public Task> GetLibraryValidatedAsync(LibraryOptions libraryOptions, bool importEpisodes = true)
{
// bug on audible's side. the 1st time after a long absence, a query to get library will return without titles or authors. a subsequent identical query will be successful. this is true whether or not tokens are refreshed
// worse, this 1st dummy call doesn't seem to help:
// var page = await api.GetLibraryAsync(new AudibleApi.LibraryOptions { NumberOfResultPerPage = 1, PageNumber = 1, PurchasedAfter = DateTime.Now.AddYears(-20), ResponseGroups = AudibleApi.LibraryOptions.ResponseGroupOptions.ALL_OPTIONS });
// i don't want to incur the cost of making a full dummy call every time because it fails sometimes
return policy.ExecuteAsync(() => getItemsAsync(libraryOptions, importEpisodes));
}
private async Task> getItemsAsync(LibraryOptions libraryOptions, bool importEpisodes)
{
Serilog.Log.Logger.Debug("Beginning library scan.");
int count = 0;
List- items = new();
List
- seriesItems = new();
var sw = Stopwatch.StartNew();
//Scan the library for all added books, and add ay episode-type items to seriesItems to be scanned for episodes/parents
await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions, BatchSize, MaxConcurrency))
{
if ((item.IsEpisodes || item.IsSeriesParent) && importEpisodes)
seriesItems.Add(item);
else if (!item.IsEpisodes && !item.IsSeriesParent)
items.Add(item);
count++;
}
Serilog.Log.Logger.Debug("Library scan complete. Found {count} books and series. Waiting on series episode scans to complete.", count);
Serilog.Log.Logger.Debug("Beginning episode scan.");
count = 0;
//'get' Tasks are activated when they are written to the channel. To avoid more concurrency than is desired, the
//channel is bounded with a capacity of 1. Channel write operations are blocked until the current item is read
var episodeChannel = Channel.CreateBounded>>(new BoundedChannelOptions(1) { SingleReader = true });
//Start scanning for all episodes. Episode batch 'get' Tasks are written to the channel.
var scanAllSeriesTask = scanAllSeries(seriesItems, episodeChannel.Writer);
//Read all episodes from the channel and add them to the import items.
//This method blocks until episodeChannel.Writer is closed by scanAllSeries()
await foreach (var ep in getAllEpisodesAsync(episodeChannel.Reader))
{
items.Add(ep);
count++;
}
//Be sure to await the scanAllSeries Task so that any exceptions are thrown
await scanAllSeriesTask;
sw.Stop();
Serilog.Log.Logger.Debug("Episode scan complete. Found {count} episodes and series.", count);
Serilog.Log.Logger.Debug($"Completed library scan in {sw.Elapsed.TotalMilliseconds:F0} ms.");
#if DEBUG
//// this will not work for multi accounts
//var library_json = "library.json";
//library_json = System.IO.Path.GetFullPath(library_json);
//if (System.IO.File.Exists(library_json))
// items = AudibleApi.Common.Converter.FromJson
>(System.IO.File.ReadAllText(library_json));
//System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
#endif
var validators = new List();
validators.AddRange(getValidators());
foreach (var v in validators)
{
var exceptions = v.Validate(items);
if (exceptions is not null && exceptions.Any())
throw new AggregateException(exceptions);
}
return items;
}
private static List getValidators()
{
var type = typeof(IValidator);
var types = AppDomain.CurrentDomain.GetAssemblies()
.SelectMany(s => s.GetTypes())
.Where(p => type.IsAssignableFrom(p) && !p.IsInterface);
return types.Select(t => Activator.CreateInstance(t) as IValidator).ToList();
}
#region episodes and podcasts
///
/// Read get tasks from the and await results. This method maintains
/// a list of up to get tasks. When any of the get tasks completes,
/// the Items are yielded, that task is removed from the list, and a new get task is read from
/// the channel.
///
private async IAsyncEnumerable- getAllEpisodesAsync(ChannelReader>> channel)
{
List>> concurentGets = new();
for (int i = 0; i < MaxConcurrency && await channel.WaitToReadAsync(); i++)
concurentGets.Add(await channel.ReadAsync());
while (concurentGets.Count > 0)
{
var completed = await Task.WhenAny(concurentGets);
concurentGets.Remove(completed);
if (await channel.WaitToReadAsync())
concurentGets.Add(await channel.ReadAsync());
foreach (var item in completed.Result)
yield return item;
}
}
///
/// Gets all child episodes and episode parents belonging to in batches and
/// writes the get tasks to .
///
private async Task scanAllSeries(IEnumerable
- seriesItems, ChannelWriter>> channel)
{
try
{
List episodeScanTasks = new();
foreach (var item in seriesItems)
{
if (item.IsEpisodes)
await channel.WriteAsync(getEpisodeParentAsync(item));
else if (item.IsSeriesParent)
episodeScanTasks.Add(getParentEpisodesAsync(item, channel));
}
//episodeScanTasks complete only after all episode batch 'gets' have been written to the channel
await Task.WhenAll(episodeScanTasks);
}
finally { channel.Complete(); }
}
private async Task
> getEpisodeParentAsync(Item episode)
{
//Item is a single episode that was added to the library.
//Get the episode's parent and add it to the database.
Serilog.Log.Logger.Debug("Supplied Parent is an episode. Beginning parent scan for {parent}", episode);
List- children = new() { episode };
var parentAsins = episode.Relationships
.Where(r => r.RelationshipToProduct == RelationshipToProduct.Parent)
.Select(p => p.Asin);
var seriesParents = await Api.GetCatalogProductsAsync(parentAsins, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS);
int numSeriesParents = seriesParents.Count(p => p.IsSeriesParent);
if (numSeriesParents != 1)
{
//There should only ever be 1 top-level parent per episode. If not, log
//so we can figure out what to do about those special cases, and don't
//import the episode.
JsonSerializerSettings Settings = new()
{
MetadataPropertyHandling = MetadataPropertyHandling.Ignore,
DateParseHandling = DateParseHandling.None,
Converters = {
new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal }
}
};
Serilog.Log.Logger.Error($"Found {numSeriesParents} parents for {episode.Asin}\r\nEpisode Product:\r\n{JsonConvert.SerializeObject(episode, Formatting.None, Settings)}");
return new();
}
var parent = seriesParents.Single(p => p.IsSeriesParent);
parent.PurchaseDate = episode.PurchaseDate;
setSeries(parent, children);
children.Add(parent);
Serilog.Log.Logger.Debug("Completed parent scan for {episode}", episode);
return children;
}
///
/// Gets all episodes belonging to in batches of and writes the batch get tasks to
/// This method only completes after all episode batch 'gets' have been written to the channel
///
private async Task getParentEpisodesAsync(Item parent, ChannelWriter>> channel)
{
Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent);
var episodeIds = parent.Relationships
.Where(r => r.RelationshipToProduct == RelationshipToProduct.Child && r.RelationshipType == RelationshipType.Episode)
.Select(r => r.Asin);
for (int batchNum = 0; episodeIds.Any(); batchNum++)
{
var batch = episodeIds.Take(BatchSize);
await channel.WriteAsync(getEpisodeBatchAsync(batchNum, parent, batch));
episodeIds = episodeIds.Skip(BatchSize);
}
}
private async Task
> getEpisodeBatchAsync(int batchNum, Item parent, IEnumerable childrenIds)
{
try
{
List- episodeBatch = await Api.GetCatalogProductsAsync(childrenIds, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS);
setSeries(parent, episodeBatch);
if (batchNum == 0)
episodeBatch.Add(parent);
Serilog.Log.Logger.Debug($"Batch {batchNum}: {episodeBatch.Count} results\t({{parent}})", parent);
return episodeBatch;
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error fetching batch of episodes. {@DebugInfo}", new
{
ParentId = parent.Asin,
ParentTitle = parent.Title,
BatchNumber = batchNum,
ChildIdBatch = childrenIds
});
throw;
}
}
private static void setSeries(Item parent, IEnumerable
- children)
{
//A series parent will always have exactly 1 Series
parent.Series = new[]
{
new Series
{
Asin = parent.Asin,
Sequence = "-1",
Title = parent.TitleWithSubtitle
}
};
foreach (var child in children)
{
// use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime
child.PurchaseDate = parent.PurchaseDate;
// parent is essentially a series
child.Series = new[]
{
new Series
{
Asin = parent.Asin,
// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive for malformed data from audible
Sequence = parent.Relationships.FirstOrDefault(r => r.Asin == child.Asin)?.Sort?.ToString() ?? "0",
Title = parent.TitleWithSubtitle
}
};
}
}
#endregion
}
}