From 8283f19d6bbbdc90bfb69192342355f6cadd5807 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Tue, 24 May 2022 21:17:59 -0600 Subject: [PATCH] Parallelize getChildEpisodesAsync --- Source/AudibleUtilities/ApiExtended.cs | 168 +++++-------------------- 1 file changed, 28 insertions(+), 140 deletions(-) diff --git a/Source/AudibleUtilities/ApiExtended.cs b/Source/AudibleUtilities/ApiExtended.cs index 4d2b8fc9..aa2625e9 100644 --- a/Source/AudibleUtilities/ApiExtended.cs +++ b/Source/AudibleUtilities/ApiExtended.cs @@ -121,93 +121,25 @@ namespace AudibleUtilities Serilog.Log.Logger.Debug("Begin initial library scan"); + List>> getChildEpisodesTasks = new(); + await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions)) { if (item.IsEpisodes && importEpisodes) { - var children = await getEpisodeChildrenAsync(item); - - // actual individual episode, not the parent of a series. - // for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic - if (!children.Any()) - { - items.Add(item); - continue; - } - - foreach (var child in children) - { - // use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime - child.PurchaseDate = item.PurchaseDate; - // parent is essentially a series - child.Series = new Series[] - { - new Series - { - Asin = item.Asin, - // This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive for malformed data from audible - Sequence = item.Relationships.FirstOrDefault(r => r.Asin == child.Asin).Sort.ToString(), - Title = item.TitleWithSubtitle - } - }; - // overload (read: abuse) IsEpisodes flag - child.Relationships = new Relationship[] - { - new Relationship - { - RelationshipToProduct = RelationshipToProduct.Child, - RelationshipType = RelationshipType.Episode - } - }; - } - items.AddRange(children); + //Get child episodes asynchronously and await all at the end + getChildEpisodesTasks.Add(getChildEpisodesAsync(item)); } else if (!item.IsEpisodes) items.Add(item); } + //asait and all all episides from all parents + foreach (var epList in await Task.WhenAll(getChildEpisodesTasks)) + items.AddRange(epList); + Serilog.Log.Logger.Debug("Scan complete"); -#if DEBUG -//System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items)); -#endif - - var validators = new List(); - validators.AddRange(getValidators()); - foreach (var v in validators) - { - var exceptions = v.Validate(items); - if (exceptions is not null && exceptions.Any()) - throw new AggregateException(exceptions); - } - - return items; - } - - private async Task> getItemsAsync2(LibraryOptions libraryOptions, bool importEpisodes) - { - var items = new List(); -#if DEBUG - //// this will not work for multi accounts - //var library_json = "library.json"; - //library_json = System.IO.Path.GetFullPath(library_json); - //if (System.IO.File.Exists(library_json)) - //{ - // items = AudibleApi.Common.Converter.FromJson>(System.IO.File.ReadAllText(library_json)); - //} -#endif - - Serilog.Log.Logger.Debug("Begin initial library scan"); - - if (!items.Any()) - items = await Api.GetAllLibraryItemsAsync(libraryOptions); - - Serilog.Log.Logger.Debug("Initial library scan complete. Begin episode scan"); - - await manageEpisodesAsync(items, importEpisodes); - - Serilog.Log.Logger.Debug("Episode scan complete"); - #if DEBUG //System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items)); #endif @@ -225,63 +157,23 @@ namespace AudibleUtilities } #region episodes and podcasts - private async Task manageEpisodesAsync(List items, bool importEpisodes) + + private async Task> getChildEpisodesAsync(Item parent) { - // add podcasts and episodes to list. If fail, don't let it de-rail the rest of the import - try + var children = await getEpisodeChildrenAsync(parent); + + // actual individual episode, not the parent of a series. + // for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic + if (!children.Any()) + return new List() { parent }; + + foreach (var child in children) { - // get parents - var parents = items.Where(i => i.IsEpisodes).ToList(); -#if DEBUG -//var parentsDebug = parents.Select(i => i.ToJson()).Aggregate((a, b) => $"{a}\r\n\r\n{b}"); -//System.IO.File.WriteAllText("parents.json", parentsDebug); -#endif - - if (!parents.Any()) - return; - - Serilog.Log.Logger.Information($"{parents.Count} series of shows/podcasts found"); - - // remove episode parents. even if the following stuff fails, these will still be removed from the collection - items.RemoveAll(i => i.IsEpisodes); - - if (importEpisodes) + // use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime + child.PurchaseDate = parent.PurchaseDate; + // parent is essentially a series + child.Series = new Series[] { - // add children - var children = await getEpisodesAsync(parents); - Serilog.Log.Logger.Information($"{children.Count} episodes of shows/podcasts found"); - items.AddRange(children); - } - } - catch (Exception ex) - { - Serilog.Log.Logger.Error(ex, "Error adding podcasts and episodes"); - } - } - - private async Task> getEpisodesAsync(List parents) - { - var results = new List(); - - foreach (var parent in parents) - { - var children = await getEpisodeChildrenAsync(parent); - - // actual individual episode, not the parent of a series. - // for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic - if (!children.Any()) - { - results.Add(parent); - continue; - } - - foreach (var child in children) - { - // use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime - child.PurchaseDate = parent.PurchaseDate; - // parent is essentially a series - child.Series = new Series[] - { new Series { Asin = parent.Asin, @@ -289,22 +181,18 @@ namespace AudibleUtilities Sequence = parent.Relationships.FirstOrDefault(r => r.Asin == child.Asin).Sort.ToString(), Title = parent.TitleWithSubtitle } - }; - // overload (read: abuse) IsEpisodes flag - child.Relationships = new Relationship[] - { + }; + // overload (read: abuse) IsEpisodes flag + child.Relationships = new Relationship[] + { new Relationship { RelationshipToProduct = RelationshipToProduct.Child, RelationshipType = RelationshipType.Episode } - }; - } - - results.AddRange(children); + }; } - - return results; + return children; } private async Task> getEpisodeChildrenAsync(Item parent)