tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #260 from Mbucari/master

Browse Source 
Throttle episode scanning to 10 concurrent scans.
This commit is contained in:
rmcrackan 2022-05-26 11:29:18 -04:00 committed by GitHub
commit 99b77decff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 39 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
Source/AudibleUtilities/ApiExtended.cs
View File

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using AudibleApi;
using AudibleApi.Common;
@ -119,18 +120,19 @@ namespace AudibleUtilities
{
var items = new List<Item>();
Serilog.Log.Logger.Debug("Begin library scan");
Serilog.Log.Logger.Debug("Beginning library scan.");
List<Task<List<Item>>> getChildEpisodesTasks = new();
int count = 0;
int count = 0, maxConcurrentEpisodeScans = 5;
using SemaphoreSlim concurrencySemaphore = new(maxConcurrentEpisodeScans);
await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions))
{
if (item.IsEpisodes && importEpisodes)
{
//Get child episodes asynchronously and await all at the end
getChildEpisodesTasks.Add(getChildEpisodesAsync(item));
getChildEpisodesTasks.Add(getChildEpisodesAsync(concurrencySemaphore, item));
}
else if (!item.IsEpisodes)
items.Add(item);
@ -138,13 +140,13 @@ namespace AudibleUtilities
count++;
}
Serilog.Log.Logger.Debug("Library scan complete. Found {count} books. Waiting on episode scans to complete", count);
Serilog.Log.Logger.Debug("Library scan complete. Found {count} books and series. Waiting on {getChildEpisodesTasksCount} series episode scans to complete.", count, getChildEpisodesTasks.Count);
//await and add all episides from all parents
foreach (var epList in await Task.WhenAll(getChildEpisodesTasks))
items.AddRange(epList);
Serilog.Log.Logger.Debug("Scan complete");
Serilog.Log.Logger.Debug("Completed library scan.");
#if DEBUG
//System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
@ -163,24 +165,28 @@ namespace AudibleUtilities
#region episodes and podcasts
private async Task<List<Item>> getChildEpisodesAsync(Item parent)
private async Task<List<Item>> getChildEpisodesAsync(SemaphoreSlim concurrencySemaphore, Item parent)
{
Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent);
await concurrencySemaphore.WaitAsync();
var children = await getEpisodeChildrenAsync(parent);
// actual individual episode, not the parent of a series.
// for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic
if (!children.Any())
return new List<Item>() { parent };
foreach (var child in children)
try
{
// use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime
child.PurchaseDate = parent.PurchaseDate;
// parent is essentially a series
child.Series = new Series[]
Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent);
var children = await getEpisodeChildrenAsync(parent);
// actual individual episode, not the parent of a series.
// for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic
if (!children.Any())
return new List<Item>() { parent };
foreach (var child in children)
{
// use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime
child.PurchaseDate = parent.PurchaseDate;
// parent is essentially a series
child.Series = new Series[]
{
new Series
{
Asin = parent.Asin,
@ -188,18 +194,26 @@ namespace AudibleUtilities
Sequence = parent.Relationships.FirstOrDefault(r => r.Asin == child.Asin).Sort.ToString(),
Title = parent.TitleWithSubtitle
}
};
// overload (read: abuse) IsEpisodes flag
child.Relationships = new Relationship[]
{
};
// overload (read: abuse) IsEpisodes flag
child.Relationships = new Relationship[]
{
new Relationship
{
RelationshipToProduct = RelationshipToProduct.Child,
RelationshipType = RelationshipType.Episode
}
};
};
}
Serilog.Log.Logger.Debug("Completed episode scan for {parent}", parent);
return children;
}
finally
{
concurrencySemaphore.Release();
}
return children;
}
private async Task<List<Item>> getEpisodeChildrenAsync(Item parent)