From 20ca4e073928da88d081f4ec4fec556dccd712c0 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Wed, 25 May 2022 16:49:22 -0600 Subject: [PATCH 1/5] Refactor for clarity. --- Source/AudibleUtilities/ApiExtended.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/AudibleUtilities/ApiExtended.cs b/Source/AudibleUtilities/ApiExtended.cs index 11ff3c39..17abe31d 100644 --- a/Source/AudibleUtilities/ApiExtended.cs +++ b/Source/AudibleUtilities/ApiExtended.cs @@ -125,7 +125,7 @@ namespace AudibleUtilities int count = 0; - await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions)) + await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions, 5, 4)) { if (item.IsEpisodes && importEpisodes) { From c57b184a0977161a8110563c3971bf6135c8db8c Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Wed, 25 May 2022 16:51:25 -0600 Subject: [PATCH 2/5] Remove test params --- Source/AudibleUtilities/ApiExtended.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/AudibleUtilities/ApiExtended.cs b/Source/AudibleUtilities/ApiExtended.cs index 17abe31d..11ff3c39 100644 --- a/Source/AudibleUtilities/ApiExtended.cs +++ b/Source/AudibleUtilities/ApiExtended.cs @@ -125,7 +125,7 @@ namespace AudibleUtilities int count = 0; - await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions, 5, 4)) + await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions)) { if (item.IsEpisodes && importEpisodes) { From a8f41841bd8048a68068e8599d133e389bb1d45b Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Wed, 25 May 2022 20:43:12 -0600 Subject: [PATCH 3/5] Throttle episode scanning to 10 concurrent scans. --- Source/AudibleUtilities/ApiExtended.cs | 57 +++++++++++++++----------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/Source/AudibleUtilities/ApiExtended.cs b/Source/AudibleUtilities/ApiExtended.cs index 11ff3c39..4194875d 100644 --- a/Source/AudibleUtilities/ApiExtended.cs +++ b/Source/AudibleUtilities/ApiExtended.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using System.Threading.Tasks; using AudibleApi; using AudibleApi.Common; @@ -123,14 +124,15 @@ namespace AudibleUtilities List>> getChildEpisodesTasks = new(); - int count = 0; + int count = 0, maxConcurrentEpisodeScans = 10; + using SemaphoreSlim concurrencySemaphore = new(maxConcurrentEpisodeScans); await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions)) { if (item.IsEpisodes && importEpisodes) { //Get child episodes asynchronously and await all at the end - getChildEpisodesTasks.Add(getChildEpisodesAsync(item)); + getChildEpisodesTasks.Add(getChildEpisodesAsync(concurrencySemaphore, item)); } else if (!item.IsEpisodes) items.Add(item); @@ -138,7 +140,7 @@ namespace AudibleUtilities count++; } - Serilog.Log.Logger.Debug("Library scan complete. Found {count} books. Waiting on episode scans to complete", count); + Serilog.Log.Logger.Debug("Library scan complete. Found {count} books and series. Waiting on {getChildEpisodesTasksCount} series episode scans to complete", count, getChildEpisodesTasks.Count); //await and add all episides from all parents foreach (var epList in await Task.WhenAll(getChildEpisodesTasks)) @@ -163,24 +165,28 @@ namespace AudibleUtilities #region episodes and podcasts - private async Task> getChildEpisodesAsync(Item parent) + private async Task> getChildEpisodesAsync(SemaphoreSlim concurrencySemaphore, Item parent) { - Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent); + await concurrencySemaphore.WaitAsync(); - var children = await getEpisodeChildrenAsync(parent); - - // actual individual episode, not the parent of a series. - // for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic - if (!children.Any()) - return new List() { parent }; - - foreach (var child in children) + try { - // use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime - child.PurchaseDate = parent.PurchaseDate; - // parent is essentially a series - child.Series = new Series[] + Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent); + + var children = await getEpisodeChildrenAsync(parent); + + // actual individual episode, not the parent of a series. + // for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic + if (!children.Any()) + return new List() { parent }; + + foreach (var child in children) { + // use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime + child.PurchaseDate = parent.PurchaseDate; + // parent is essentially a series + child.Series = new Series[] + { new Series { Asin = parent.Asin, @@ -188,18 +194,23 @@ namespace AudibleUtilities Sequence = parent.Relationships.FirstOrDefault(r => r.Asin == child.Asin).Sort.ToString(), Title = parent.TitleWithSubtitle } - }; - // overload (read: abuse) IsEpisodes flag - child.Relationships = new Relationship[] - { + }; + // overload (read: abuse) IsEpisodes flag + child.Relationships = new Relationship[] + { new Relationship { RelationshipToProduct = RelationshipToProduct.Child, RelationshipType = RelationshipType.Episode } - }; + }; + } + return children; + } + finally + { + concurrencySemaphore.Release(); } - return children; } private async Task> getEpisodeChildrenAsync(Item parent) From d92d892dc78144008547bfdf1f3d2bbe89e18fc4 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Wed, 25 May 2022 20:45:18 -0600 Subject: [PATCH 4/5] logging --- Source/AudibleUtilities/ApiExtended.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Source/AudibleUtilities/ApiExtended.cs b/Source/AudibleUtilities/ApiExtended.cs index 4194875d..1dbc4ced 100644 --- a/Source/AudibleUtilities/ApiExtended.cs +++ b/Source/AudibleUtilities/ApiExtended.cs @@ -120,7 +120,7 @@ namespace AudibleUtilities { var items = new List(); - Serilog.Log.Logger.Debug("Begin library scan"); + Serilog.Log.Logger.Debug("Beginning library scan."); List>> getChildEpisodesTasks = new(); @@ -140,13 +140,13 @@ namespace AudibleUtilities count++; } - Serilog.Log.Logger.Debug("Library scan complete. Found {count} books and series. Waiting on {getChildEpisodesTasksCount} series episode scans to complete", count, getChildEpisodesTasks.Count); + Serilog.Log.Logger.Debug("Library scan complete. Found {count} books and series. Waiting on {getChildEpisodesTasksCount} series episode scans to complete.", count, getChildEpisodesTasks.Count); //await and add all episides from all parents foreach (var epList in await Task.WhenAll(getChildEpisodesTasks)) items.AddRange(epList); - Serilog.Log.Logger.Debug("Scan complete"); + Serilog.Log.Logger.Debug("Completed library scan."); #if DEBUG //System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items)); @@ -205,6 +205,9 @@ namespace AudibleUtilities } }; } + + Serilog.Log.Logger.Debug("Completed episode scan for {parent}", parent); + return children; } finally From 2e8acfdeeffbfa39d9ee16546478a61fe5107cd0 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Thu, 26 May 2022 08:44:39 -0600 Subject: [PATCH 5/5] Limnit episode concurrency to 5 --- Source/AudibleUtilities/ApiExtended.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/AudibleUtilities/ApiExtended.cs b/Source/AudibleUtilities/ApiExtended.cs index 1dbc4ced..70d338b6 100644 --- a/Source/AudibleUtilities/ApiExtended.cs +++ b/Source/AudibleUtilities/ApiExtended.cs @@ -124,7 +124,7 @@ namespace AudibleUtilities List>> getChildEpisodesTasks = new(); - int count = 0, maxConcurrentEpisodeScans = 10; + int count = 0, maxConcurrentEpisodeScans = 5; using SemaphoreSlim concurrencySemaphore = new(maxConcurrentEpisodeScans); await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions))