tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parallelize getChildEpisodesAsync

Browse Source
This commit is contained in:
Michael Bucari-Tovo 2022-05-24 21:17:59 -06:00
parent 242909b542
commit 8283f19d6b
1 changed files with 28 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
Source/AudibleUtilities/ApiExtended.cs
View File

@ -121,93 +121,25 @@ namespace AudibleUtilities
Serilog.Log.Logger.Debug("Begin initial library scan"); Serilog.Log.Logger.Debug("Begin initial library scan");
List<Task<List<Item>>> getChildEpisodesTasks = new();
await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions)) await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions))
{ {
if (item.IsEpisodes && importEpisodes) if (item.IsEpisodes && importEpisodes)
{ {
var children = await getEpisodeChildrenAsync(item); //Get child episodes asynchronously and await all at the end
getChildEpisodesTasks.Add(getChildEpisodesAsync(item));
// actual individual episode, not the parent of a series.
// for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic
if (!children.Any())
{
items.Add(item);
continue;
}
foreach (var child in children)
{
// use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime
child.PurchaseDate = item.PurchaseDate;
// parent is essentially a series
child.Series = new Series[]
{
new Series
{
Asin = item.Asin,
// This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive for malformed data from audible
Sequence = item.Relationships.FirstOrDefault(r => r.Asin == child.Asin).Sort.ToString(),
Title = item.TitleWithSubtitle
}
};
// overload (read: abuse) IsEpisodes flag
child.Relationships = new Relationship[]
{
new Relationship
{
RelationshipToProduct = RelationshipToProduct.Child,
RelationshipType = RelationshipType.Episode
}
};
}
items.AddRange(children);
} }
else if (!item.IsEpisodes) else if (!item.IsEpisodes)
items.Add(item); items.Add(item);
} }
//asait and all all episides from all parents
foreach (var epList in await Task.WhenAll(getChildEpisodesTasks))
items.AddRange(epList);
Serilog.Log.Logger.Debug("Scan complete"); Serilog.Log.Logger.Debug("Scan complete");
#if DEBUG
//System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
#endif
var validators = new List<IValidator>();
validators.AddRange(getValidators());
foreach (var v in validators)
{
var exceptions = v.Validate(items);
if (exceptions is not null && exceptions.Any())
throw new AggregateException(exceptions);
}
return items;
}
private async Task<List<Item>> getItemsAsync2(LibraryOptions libraryOptions, bool importEpisodes)
{
var items = new List<Item>();
#if DEBUG
//// this will not work for multi accounts
//var library_json = "library.json";
//library_json = System.IO.Path.GetFullPath(library_json);
//if (System.IO.File.Exists(library_json))
//{
// items = AudibleApi.Common.Converter.FromJson<List<Item>>(System.IO.File.ReadAllText(library_json));
//}
#endif
Serilog.Log.Logger.Debug("Begin initial library scan");
if (!items.Any())
items = await Api.GetAllLibraryItemsAsync(libraryOptions);
Serilog.Log.Logger.Debug("Initial library scan complete. Begin episode scan");
await manageEpisodesAsync(items, importEpisodes);
Serilog.Log.Logger.Debug("Episode scan complete");
#if DEBUG #if DEBUG
//System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items)); //System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
#endif #endif
@ -225,55 +157,15 @@ namespace AudibleUtilities
} }
#region episodes and podcasts #region episodes and podcasts
private async Task manageEpisodesAsync(List<Item> items, bool importEpisodes)
{
// add podcasts and episodes to list. If fail, don't let it de-rail the rest of the import
try
{
// get parents
var parents = items.Where(i => i.IsEpisodes).ToList();
#if DEBUG
//var parentsDebug = parents.Select(i => i.ToJson()).Aggregate((a, b) => $"{a}\r\n\r\n{b}");
//System.IO.File.WriteAllText("parents.json", parentsDebug);
#endif
if (!parents.Any()) private async Task<List<Item>> getChildEpisodesAsync(Item parent)
return;
Serilog.Log.Logger.Information($"{parents.Count} series of shows/podcasts found");
// remove episode parents. even if the following stuff fails, these will still be removed from the collection
items.RemoveAll(i => i.IsEpisodes);
if (importEpisodes)
{
// add children
var children = await getEpisodesAsync(parents);
Serilog.Log.Logger.Information($"{children.Count} episodes of shows/podcasts found");
items.AddRange(children);
}
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "Error adding podcasts and episodes");
}
}
private async Task<List<Item>> getEpisodesAsync(List<Item> parents)
{
var results = new List<Item>();
foreach (var parent in parents)
{ {
var children = await getEpisodeChildrenAsync(parent); var children = await getEpisodeChildrenAsync(parent);
// actual individual episode, not the parent of a series. // actual individual episode, not the parent of a series.
// for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic // for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic
if (!children.Any()) if (!children.Any())
{ return new List<Item>() { parent };
results.Add(parent);
continue;
}
foreach (var child in children) foreach (var child in children)
{ {
@ -300,11 +192,7 @@ namespace AudibleUtilities
} }
}; };
} }
return children;
results.AddRange(children);
}
return results;
} }
private async Task<List<Item>> getEpisodeChildrenAsync(Item parent) private async Task<List<Item>> getEpisodeChildrenAsync(Item parent)