From cdb27ef712d7e23cace8d885dac27c1a8ec16d73 Mon Sep 17 00:00:00 2001 From: Mbucari Date: Fri, 3 Mar 2023 15:06:06 -0700 Subject: [PATCH 1/9] Add last downloaded info to exports --- Source/ApplicationServices/LibraryExporter.cs | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/Source/ApplicationServices/LibraryExporter.cs b/Source/ApplicationServices/LibraryExporter.cs index 01e336af..713e2317 100644 --- a/Source/ApplicationServices/LibraryExporter.cs +++ b/Source/ApplicationServices/LibraryExporter.cs @@ -106,6 +106,12 @@ namespace ApplicationServices [Name("Language")] public string Language { get; set; } + + [Name("LastDownloaded")] + public DateTime? LastDownloaded { get; set; } + + [Name("LastDownloadedVersion")] + public string LastDownloadedVersion { get; set; } } public static class LibToDtos { @@ -140,7 +146,10 @@ namespace ApplicationServices PdfStatus = a.Book.UserDefinedItem.PdfStatus.ToString(), ContentType = a.Book.ContentType.ToString(), AudioFormat = a.Book.AudioFormat.ToString(), - Language = a.Book.Language + Language = a.Book.Language, + LastDownloaded = a.Book.UserDefinedItem.LastDownloaded, + LastDownloadedVersion = a.Book.UserDefinedItem.LastDownloadedVersion?.ToString() ?? "", + }).ToList(); } public static class LibraryExporter @@ -212,7 +221,9 @@ namespace ApplicationServices nameof(ExportDto.PdfStatus), nameof(ExportDto.ContentType), nameof(ExportDto.AudioFormat), - nameof(ExportDto.Language) + nameof(ExportDto.Language), + nameof(ExportDto.LastDownloaded), + nameof(ExportDto.LastDownloadedVersion), }; var col = 0; foreach (var c in columns) @@ -238,9 +249,9 @@ namespace ApplicationServices row.CreateCell(col++).SetCellValue(dto.Account); - var dateAddedCell = row.CreateCell(col++); - dateAddedCell.CellStyle = dateStyle; - dateAddedCell.SetCellValue(dto.DateAdded); + var dateCell = row.CreateCell(col++); + dateCell.CellStyle = dateStyle; + dateCell.SetCellValue(dto.DateAdded); row.CreateCell(col++).SetCellValue(dto.AudibleProductId); row.CreateCell(col++).SetCellValue(dto.Locale); @@ -281,6 +292,15 @@ namespace ApplicationServices row.CreateCell(col++).SetCellValue(dto.AudioFormat); row.CreateCell(col++).SetCellValue(dto.Language); + if (dto.LastDownloaded.HasValue) + { + dateCell = row.CreateCell(col); + dateCell.CellStyle = dateStyle; + dateCell.SetCellValue(dto.LastDownloaded.Value); + } + + row.CreateCell(++col).SetCellValue(dto.LastDownloadedVersion); + rowIndex++; } From da36f9414dae11bd475e62cb53cb910983d390a5 Mon Sep 17 00:00:00 2001 From: Mbucari Date: Mon, 6 Mar 2023 16:49:52 -0700 Subject: [PATCH 2/9] Improve library scan performance --- Source/AudibleUtilities/ApiExtended.cs | 319 +++++++++++++------------ 1 file changed, 168 insertions(+), 151 deletions(-) diff --git a/Source/AudibleUtilities/ApiExtended.cs b/Source/AudibleUtilities/ApiExtended.cs index 376bde2c..52585b00 100644 --- a/Source/AudibleUtilities/ApiExtended.cs +++ b/Source/AudibleUtilities/ApiExtended.cs @@ -2,8 +2,9 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; +using System.Diagnostics; using AudibleApi; using AudibleApi.Common; using Dinah.Core; @@ -19,6 +20,9 @@ namespace AudibleUtilities { public Api Api { get; private set; } + private const int MaxConcurrency = 10; + private const int BatchSize = 50; + private ApiExtended(Api api) => Api = api; /// Get api from existing tokens else login with 'eager' choice. External browser url is provided. Response can be external browser login or continuing with native api callbacks. @@ -85,35 +89,49 @@ namespace AudibleUtilities private async Task> getItemsAsync(LibraryOptions libraryOptions, bool importEpisodes) { - var items = new List(); - Serilog.Log.Logger.Debug("Beginning library scan."); - List>> getChildEpisodesTasks = new(); + var episodeChannel = Channel.CreateBounded>>( + new BoundedChannelOptions(1) + { + SingleReader = true, + SingleWriter = false, + FullMode = BoundedChannelFullMode.Wait + }); - int count = 0, maxConcurrentEpisodeScans = 5; - using SemaphoreSlim concurrencySemaphore = new(maxConcurrentEpisodeScans); + int count = 0; + List items = new(); + List seriesItems = new(); - await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions)) + var sw = Stopwatch.StartNew(); + + await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions, BatchSize, MaxConcurrency)) { if ((item.IsEpisodes || item.IsSeriesParent) && importEpisodes) - { - //Get child episodes asynchronously and await all at the end - getChildEpisodesTasks.Add(getChildEpisodesAsync(concurrencySemaphore, item)); - } + seriesItems.Add(item); else if (!item.IsEpisodes && !item.IsSeriesParent) items.Add(item); count++; } - Serilog.Log.Logger.Debug("Library scan complete. Found {count} books and series. Waiting on {getChildEpisodesTasksCount} series episode scans to complete.", count, getChildEpisodesTasks.Count); + Serilog.Log.Logger.Debug("Library scan complete. Found {count} books and series. Waiting on series episode scans to complete.", count); + Serilog.Log.Logger.Debug("Beginning episode scan."); - //await and add all episodes from all parents - foreach (var epList in await Task.WhenAll(getChildEpisodesTasks)) - items.AddRange(epList); + count = 0; + var episodeDlTask = scanAllSeries(seriesItems, episodeChannel.Writer); - Serilog.Log.Logger.Debug("Completed library scan."); + await foreach (var ep in getAllEpisodesAsync(episodeChannel.Reader, MaxConcurrency)) + { + items.Add(ep); + count++; + } + + await episodeDlTask; + + sw.Stop(); + Serilog.Log.Logger.Debug("Episode scan complete. Found {count} episodes and series.", count); + Serilog.Log.Logger.Debug($"Completed library scan in {sw.Elapsed.TotalMilliseconds:F0} ms."); #if DEBUG //// this will not work for multi accounts @@ -146,165 +164,164 @@ namespace AudibleUtilities #region episodes and podcasts - private async Task> getChildEpisodesAsync(SemaphoreSlim concurrencySemaphore, Item parent) + private async IAsyncEnumerable getAllEpisodesAsync(ChannelReader>> allEpisodes, int maxConcurrency) { - await concurrencySemaphore.WaitAsync(); + List>> concurentGets = new(); + for (int i = 0; i < maxConcurrency && await allEpisodes.WaitToReadAsync(); i++) + concurentGets.Add(await allEpisodes.ReadAsync()); + + while (concurentGets.Count > 0) + { + var completed = await Task.WhenAny(concurentGets); + concurentGets.Remove(completed); + + if (await allEpisodes.WaitToReadAsync()) + concurentGets.Add(await allEpisodes.ReadAsync()); + + foreach (var item in completed.Result) + yield return item; + } + } + + private async Task scanAllSeries(IEnumerable seriesItems, ChannelWriter>> allEpisodes) + { try { - Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent); + List episodeScanTasks = new(); - List children; - - if (parent.IsEpisodes) + foreach (var item in seriesItems) { - //The 'parent' is a single episode that was added to the library. - //Get the episode's parent and add it to the database. + if (item.IsEpisodes) + await allEpisodes.WriteAsync(getEpisodeParentAsync(item)); + else if (item.IsSeriesParent) + episodeScanTasks.Add(getParentEpisodesAsync(allEpisodes, item)); + } - Serilog.Log.Logger.Debug("Supplied Parent is an episode. Beginning parent scan for {parent}", parent); + await Task.WhenAll(episodeScanTasks); + } + finally { allEpisodes.Complete(); } + } - children = new() { parent }; + private async Task> getEpisodeParentAsync(Item episode) + { + //Item is a single episode that was added to the library. + //Get the episode's parent and add it to the database. - var parentAsins = parent.Relationships - .Where(r => r.RelationshipToProduct == RelationshipToProduct.Parent) - .Select(p => p.Asin); + Serilog.Log.Logger.Debug("Supplied Parent is an episode. Beginning parent scan for {parent}", episode); - var seriesParents = await Api.GetCatalogProductsAsync(parentAsins, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS); + List children = new() { episode }; - int numSeriesParents = seriesParents.Count(p => p.IsSeriesParent); - if (numSeriesParents != 1) - { - //There should only ever be 1 top-level parent per episode. If not, log - //so we can figure out what to do about those special cases, and don't - //import the episode. - JsonSerializerSettings Settings = new() - { - MetadataPropertyHandling = MetadataPropertyHandling.Ignore, - DateParseHandling = DateParseHandling.None, - Converters = - { - new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } - }, - }; - Serilog.Log.Logger.Error($"Found {numSeriesParents} parents for {parent.Asin}\r\nEpisode Product:\r\n{JsonConvert.SerializeObject(parent, Formatting.None, Settings)}"); - return new List(); + var parentAsins = episode.Relationships + .Where(r => r.RelationshipToProduct == RelationshipToProduct.Parent) + .Select(p => p.Asin); + + var seriesParents = await Api.GetCatalogProductsAsync(parentAsins, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS); + + int numSeriesParents = seriesParents.Count(p => p.IsSeriesParent); + if (numSeriesParents != 1) + { + //There should only ever be 1 top-level parent per episode. If not, log + //so we can figure out what to do about those special cases, and don't + //import the episode. + JsonSerializerSettings Settings = new() + { + MetadataPropertyHandling = MetadataPropertyHandling.Ignore, + DateParseHandling = DateParseHandling.None, + Converters = { + new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal } } + }; + Serilog.Log.Logger.Error($"Found {numSeriesParents} parents for {episode.Asin}\r\nEpisode Product:\r\n{JsonConvert.SerializeObject(episode, Formatting.None, Settings)}"); + return new(); + } - var realParent = seriesParents.Single(p => p.IsSeriesParent); - realParent.PurchaseDate = parent.PurchaseDate; + var parent = seriesParents.Single(p => p.IsSeriesParent); + parent.PurchaseDate = episode.PurchaseDate; - Serilog.Log.Logger.Debug("Completed parent scan for {parent}", parent); - parent = realParent; - } - else + setSeries(parent, children); + children.Add(parent); + + Serilog.Log.Logger.Debug("Completed parent scan for {episode}", episode); + + return children; + } + + private async Task getParentEpisodesAsync(ChannelWriter>> channel, Item parent) + { + Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent); + + var episodeIds = parent.Relationships + .Where(r => r.RelationshipToProduct == RelationshipToProduct.Child && r.RelationshipType == RelationshipType.Episode) + .Select(r => r.Asin); + + for (int batchNum = 0; episodeIds.Any(); batchNum++) + { + var batch = episodeIds.Take(BatchSize); + + await channel.WriteAsync(getEpisodeBatchAsync(batchNum, parent, batch)); + + episodeIds = episodeIds.Skip(BatchSize); + } + } + + private async Task> getEpisodeBatchAsync(int batchNum, Item parent, IEnumerable childrenIds) + { + try + { + List episodeBatch = await Api.GetCatalogProductsAsync(childrenIds, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS); + + setSeries(parent, episodeBatch); + + if (batchNum == 0) + episodeBatch.Add(parent); + + Serilog.Log.Logger.Debug($"Batch {batchNum}: {episodeBatch.Count} results\t({{parent}})", parent); + + return episodeBatch; + } + catch (Exception ex) + { + Serilog.Log.Logger.Error(ex, "Error fetching batch of episodes. {@DebugInfo}", new { - children = await getEpisodeChildrenAsync(parent); - if (!children.Any()) - return new(); - } + ParentId = parent.Asin, + ParentTitle = parent.Title, + BatchNumber = batchNum, + ChildIdBatch = childrenIds + }); + throw; + } + } - //A series parent will always have exactly 1 Series - parent.Series = new Series[] + private static void setSeries(Item parent, IEnumerable children) + { + //A series parent will always have exactly 1 Series + parent.Series = new[] + { + new Series + { + Asin = parent.Asin, + Sequence = "-1", + Title = parent.TitleWithSubtitle + } + }; + + foreach (var child in children) + { + // use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime + child.PurchaseDate = parent.PurchaseDate; + // parent is essentially a series + child.Series = new[] { new Series { Asin = parent.Asin, - Sequence = "-1", + // This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive for malformed data from audible + Sequence = parent.Relationships.FirstOrDefault(r => r.Asin == child.Asin)?.Sort?.ToString() ?? "0", Title = parent.TitleWithSubtitle } }; - - foreach (var child in children) - { - // use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime - child.PurchaseDate = parent.PurchaseDate; - // parent is essentially a series - child.Series = new Series[] - { - new Series - { - Asin = parent.Asin, - // This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive for malformed data from audible - Sequence = parent.Relationships.FirstOrDefault(r => r.Asin == child.Asin)?.Sort?.ToString() ?? "0", - Title = parent.TitleWithSubtitle - } - }; - } - - children.Add(parent); - - Serilog.Log.Logger.Debug("Completed episode scan for {parent}", parent); - - return children; } - finally - { - concurrencySemaphore.Release(); - } - } - - private async Task> getEpisodeChildrenAsync(Item parent) - { - var childrenIds = parent.Relationships - .Where(r => r.RelationshipToProduct == RelationshipToProduct.Child && r.RelationshipType == RelationshipType.Episode) - .Select(r => r.Asin) - .ToList(); - - // fetch children in batches - const int batchSize = 20; - - var results = new List(); - - for (var i = 1; ; i++) - { - var idBatch = childrenIds.Skip((i - 1) * batchSize).Take(batchSize).ToList(); - if (!idBatch.Any()) - break; - - List childrenBatch; - try - { - childrenBatch = await Api.GetCatalogProductsAsync(idBatch, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS); -#if DEBUG - //var childrenBatchDebug = childrenBatch.Select(i => i.ToJson()).Aggregate((a, b) => $"{a}\r\n\r\n{b}"); - //System.IO.File.WriteAllText($"children of {parent.Asin}.json", childrenBatchDebug); -#endif - } - catch (Exception ex) - { - Serilog.Log.Logger.Error(ex, "Error fetching batch of episodes. {@DebugInfo}", new - { - ParentId = parent.Asin, - ParentTitle = parent.Title, - BatchNumber = i, - ChildIdBatch = idBatch - }); - throw; - } - - Serilog.Log.Logger.Debug($"Batch {i}: {childrenBatch.Count} results\t({{parent}})", parent); - // the service returned no results. probably indicates an error. stop running batches - if (!childrenBatch.Any()) - break; - - results.AddRange(childrenBatch); - } - - Serilog.Log.Logger.Debug("Parent episodes/podcasts series. Children found. {@DebugInfo}", new - { - ParentId = parent.Asin, - ParentTitle = parent.Title, - ChildCount = childrenIds.Count - }); - - if (childrenIds.Count != results.Count) - { - var ex = new ApplicationException($"Mis-match: Children defined by parent={childrenIds.Count}. Children returned by batches={results.Count}"); - Serilog.Log.Logger.Error(ex, "{parent} - Quantity of series episodes defined by parent does not match quantity returned by batch fetching.", parent); - throw ex; - } - - return results; } #endregion } From bd49db83e469b16cdbf2efaf30b419f38dc1014b Mon Sep 17 00:00:00 2001 From: MBucari Date: Mon, 6 Mar 2023 18:39:48 -0700 Subject: [PATCH 3/9] Improve library scan performance --- Source/AudibleUtilities/ApiExtended.cs | 59 +++++++++++++++++--------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/Source/AudibleUtilities/ApiExtended.cs b/Source/AudibleUtilities/ApiExtended.cs index 52585b00..1f89f511 100644 --- a/Source/AudibleUtilities/ApiExtended.cs +++ b/Source/AudibleUtilities/ApiExtended.cs @@ -91,20 +91,13 @@ namespace AudibleUtilities { Serilog.Log.Logger.Debug("Beginning library scan."); - var episodeChannel = Channel.CreateBounded>>( - new BoundedChannelOptions(1) - { - SingleReader = true, - SingleWriter = false, - FullMode = BoundedChannelFullMode.Wait - }); - int count = 0; List items = new(); List seriesItems = new(); var sw = Stopwatch.StartNew(); + //Scan the library for all added books, and add ay episode-type items to seriesItems to be scanned for episodes/parents await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions, BatchSize, MaxConcurrency)) { if ((item.IsEpisodes || item.IsSeriesParent) && importEpisodes) @@ -119,15 +112,24 @@ namespace AudibleUtilities Serilog.Log.Logger.Debug("Beginning episode scan."); count = 0; - var episodeDlTask = scanAllSeries(seriesItems, episodeChannel.Writer); - await foreach (var ep in getAllEpisodesAsync(episodeChannel.Reader, MaxConcurrency)) + //'get' Tasks are activated when they are written to the channel. To avoid more concurrency than is desired, the + //channel is bounded with a capacity of 1. Channel write operations are blocked until the current item is read + var episodeChannel = Channel.CreateBounded>>(new BoundedChannelOptions(1) { SingleReader = true }); + + //Start scanning for all episodes. Episode batch 'get' Tasks are written to the channel. + var scanAllSeriesTask = scanAllSeries(seriesItems, episodeChannel.Writer); + + //Read all episodes from the channel and add them to the import items. + //This method blocks until episodeChannel.Writer is closed by scanAllSeries() + await foreach (var ep in getAllEpisodesAsync(episodeChannel.Reader)) { items.Add(ep); count++; } - await episodeDlTask; + //Be sure to await the scanAllSeries Task so that any exceptions are thrown + await scanAllSeriesTask; sw.Stop(); Serilog.Log.Logger.Debug("Episode scan complete. Found {count} episodes and series.", count); @@ -164,27 +166,37 @@ namespace AudibleUtilities #region episodes and podcasts - private async IAsyncEnumerable getAllEpisodesAsync(ChannelReader>> allEpisodes, int maxConcurrency) + /// + /// Read get tasks from the and await results. This method maintains + /// a list of up to get tasks. When any of the get tasks completes, + /// the Items are yielded, that task is removed from the list, and a new get task is read from + /// the channel. + /// + private async IAsyncEnumerable getAllEpisodesAsync(ChannelReader>> channel) { List>> concurentGets = new(); - for (int i = 0; i < maxConcurrency && await allEpisodes.WaitToReadAsync(); i++) - concurentGets.Add(await allEpisodes.ReadAsync()); + for (int i = 0; i < MaxConcurrency && await channel.WaitToReadAsync(); i++) + concurentGets.Add(await channel.ReadAsync()); while (concurentGets.Count > 0) { var completed = await Task.WhenAny(concurentGets); concurentGets.Remove(completed); - if (await allEpisodes.WaitToReadAsync()) - concurentGets.Add(await allEpisodes.ReadAsync()); + if (await channel.WaitToReadAsync()) + concurentGets.Add(await channel.ReadAsync()); foreach (var item in completed.Result) yield return item; } } - private async Task scanAllSeries(IEnumerable seriesItems, ChannelWriter>> allEpisodes) + /// + /// Gets all child episodes and episode parents belonging to in batches and + /// writes the get tasks to . + /// + private async Task scanAllSeries(IEnumerable seriesItems, ChannelWriter>> channel) { try { @@ -193,14 +205,15 @@ namespace AudibleUtilities foreach (var item in seriesItems) { if (item.IsEpisodes) - await allEpisodes.WriteAsync(getEpisodeParentAsync(item)); + await channel.WriteAsync(getEpisodeParentAsync(item)); else if (item.IsSeriesParent) - episodeScanTasks.Add(getParentEpisodesAsync(allEpisodes, item)); + episodeScanTasks.Add(getParentEpisodesAsync(item, channel)); } + //episodeScanTasks complete only after all episode batch 'gets' have been written to the channel await Task.WhenAll(episodeScanTasks); } - finally { allEpisodes.Complete(); } + finally { channel.Complete(); } } private async Task> getEpisodeParentAsync(Item episode) @@ -247,7 +260,11 @@ namespace AudibleUtilities return children; } - private async Task getParentEpisodesAsync(ChannelWriter>> channel, Item parent) + /// + /// Gets all episodes belonging to in batches of and writes the batch get tasks to + /// This method only completes after all episode batch 'gets' have been written to the channel + /// + private async Task getParentEpisodesAsync(Item parent, ChannelWriter>> channel) { Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent); From f6dcc0db1db8fca42c76d42f288b5494ea54afd1 Mon Sep 17 00:00:00 2001 From: Mbucari Date: Tue, 7 Mar 2023 16:00:23 -0700 Subject: [PATCH 4/9] Add AbsentFromLastScan --- Source/DataLayer/EfClasses/LibraryBook.cs | 1 + ...08013410_AddAbsentFromLastScan.Designer.cs | 413 ++++++++++++++++++ .../20230308013410_AddAbsentFromLastScan.cs | 29 ++ .../LibationContextModelSnapshot.cs | 3 + Source/DtoImporterService/ImportItem.cs | 2 + .../DtoImporterService/LibraryBookImporter.cs | 32 +- 6 files changed, 477 insertions(+), 3 deletions(-) create mode 100644 Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.Designer.cs create mode 100644 Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.cs diff --git a/Source/DataLayer/EfClasses/LibraryBook.cs b/Source/DataLayer/EfClasses/LibraryBook.cs index 02bb5472..449a2477 100644 --- a/Source/DataLayer/EfClasses/LibraryBook.cs +++ b/Source/DataLayer/EfClasses/LibraryBook.cs @@ -12,6 +12,7 @@ namespace DataLayer public string Account { get; private set; } public bool IsDeleted { get; set; } + public bool AbsentFromLastScan { get; set; } private LibraryBook() { } public LibraryBook(Book book, DateTime dateAdded, string account) diff --git a/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.Designer.cs b/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.Designer.cs new file mode 100644 index 00000000..02b58087 --- /dev/null +++ b/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.Designer.cs @@ -0,0 +1,413 @@ +// +using System; +using DataLayer; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DataLayer.Migrations +{ + [DbContext(typeof(LibationContext))] + [Migration("20230308013410_AddAbsentFromLastScan")] + partial class AddAbsentFromLastScan + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "7.0.3"); + + modelBuilder.Entity("DataLayer.Book", b => + { + b.Property("BookId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AudibleProductId") + .HasColumnType("TEXT"); + + b.Property("CategoryId") + .HasColumnType("INTEGER"); + + b.Property("ContentType") + .HasColumnType("INTEGER"); + + b.Property("DatePublished") + .HasColumnType("TEXT"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("IsAbridged") + .HasColumnType("INTEGER"); + + b.Property("Language") + .HasColumnType("TEXT"); + + b.Property("LengthInMinutes") + .HasColumnType("INTEGER"); + + b.Property("Locale") + .HasColumnType("TEXT"); + + b.Property("PictureId") + .HasColumnType("TEXT"); + + b.Property("PictureLarge") + .HasColumnType("TEXT"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.Property("_audioFormat") + .HasColumnType("INTEGER"); + + b.HasKey("BookId"); + + b.HasIndex("AudibleProductId"); + + b.HasIndex("CategoryId"); + + b.ToTable("Books"); + }); + + modelBuilder.Entity("DataLayer.BookContributor", b => + { + b.Property("BookId") + .HasColumnType("INTEGER"); + + b.Property("ContributorId") + .HasColumnType("INTEGER"); + + b.Property("Role") + .HasColumnType("INTEGER"); + + b.Property("Order") + .HasColumnType("INTEGER"); + + b.HasKey("BookId", "ContributorId", "Role"); + + b.HasIndex("BookId"); + + b.HasIndex("ContributorId"); + + b.ToTable("BookContributor"); + }); + + modelBuilder.Entity("DataLayer.Category", b => + { + b.Property("CategoryId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AudibleCategoryId") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("ParentCategoryCategoryId") + .HasColumnType("INTEGER"); + + b.HasKey("CategoryId"); + + b.HasIndex("AudibleCategoryId"); + + b.HasIndex("ParentCategoryCategoryId"); + + b.ToTable("Categories"); + + b.HasData( + new + { + CategoryId = -1, + AudibleCategoryId = "", + Name = "" + }); + }); + + modelBuilder.Entity("DataLayer.Contributor", b => + { + b.Property("ContributorId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AudibleContributorId") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.HasKey("ContributorId"); + + b.HasIndex("Name"); + + b.ToTable("Contributors"); + + b.HasData( + new + { + ContributorId = -1, + Name = "" + }); + }); + + modelBuilder.Entity("DataLayer.LibraryBook", b => + { + b.Property("BookId") + .HasColumnType("INTEGER"); + + b.Property("AbsentFromLastScan") + .HasColumnType("INTEGER"); + + b.Property("Account") + .HasColumnType("TEXT"); + + b.Property("DateAdded") + .HasColumnType("TEXT"); + + b.Property("IsDeleted") + .HasColumnType("INTEGER"); + + b.HasKey("BookId"); + + b.ToTable("LibraryBooks"); + }); + + modelBuilder.Entity("DataLayer.Series", b => + { + b.Property("SeriesId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AudibleSeriesId") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.HasKey("SeriesId"); + + b.HasIndex("AudibleSeriesId"); + + b.ToTable("Series"); + }); + + modelBuilder.Entity("DataLayer.SeriesBook", b => + { + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.Property("BookId") + .HasColumnType("INTEGER"); + + b.Property("Order") + .HasColumnType("TEXT"); + + b.HasKey("SeriesId", "BookId"); + + b.HasIndex("BookId"); + + b.HasIndex("SeriesId"); + + b.ToTable("SeriesBook"); + }); + + modelBuilder.Entity("DataLayer.Book", b => + { + b.HasOne("DataLayer.Category", "Category") + .WithMany() + .HasForeignKey("CategoryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("DataLayer.Rating", "Rating", b1 => + { + b1.Property("BookId") + .HasColumnType("INTEGER"); + + b1.Property("OverallRating") + .HasColumnType("REAL"); + + b1.Property("PerformanceRating") + .HasColumnType("REAL"); + + b1.Property("StoryRating") + .HasColumnType("REAL"); + + b1.HasKey("BookId"); + + b1.ToTable("Books"); + + b1.WithOwner() + .HasForeignKey("BookId"); + }); + + b.OwnsMany("DataLayer.Supplement", "Supplements", b1 => + { + b1.Property("SupplementId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b1.Property("BookId") + .HasColumnType("INTEGER"); + + b1.Property("Url") + .HasColumnType("TEXT"); + + b1.HasKey("SupplementId"); + + b1.HasIndex("BookId"); + + b1.ToTable("Supplement"); + + b1.WithOwner("Book") + .HasForeignKey("BookId"); + + b1.Navigation("Book"); + }); + + b.OwnsOne("DataLayer.UserDefinedItem", "UserDefinedItem", b1 => + { + b1.Property("BookId") + .HasColumnType("INTEGER"); + + b1.Property("BookStatus") + .HasColumnType("INTEGER"); + + b1.Property("LastDownloaded") + .HasColumnType("TEXT"); + + b1.Property("LastDownloadedVersion") + .HasColumnType("TEXT"); + + b1.Property("PdfStatus") + .HasColumnType("INTEGER"); + + b1.Property("Tags") + .HasColumnType("TEXT"); + + b1.HasKey("BookId"); + + b1.ToTable("UserDefinedItem", (string)null); + + b1.WithOwner("Book") + .HasForeignKey("BookId"); + + b1.OwnsOne("DataLayer.Rating", "Rating", b2 => + { + b2.Property("UserDefinedItemBookId") + .HasColumnType("INTEGER"); + + b2.Property("OverallRating") + .HasColumnType("REAL"); + + b2.Property("PerformanceRating") + .HasColumnType("REAL"); + + b2.Property("StoryRating") + .HasColumnType("REAL"); + + b2.HasKey("UserDefinedItemBookId"); + + b2.ToTable("UserDefinedItem"); + + b2.WithOwner() + .HasForeignKey("UserDefinedItemBookId"); + }); + + b1.Navigation("Book"); + + b1.Navigation("Rating"); + }); + + b.Navigation("Category"); + + b.Navigation("Rating"); + + b.Navigation("Supplements"); + + b.Navigation("UserDefinedItem"); + }); + + modelBuilder.Entity("DataLayer.BookContributor", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithMany("ContributorsLink") + .HasForeignKey("BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DataLayer.Contributor", "Contributor") + .WithMany("BooksLink") + .HasForeignKey("ContributorId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Book"); + + b.Navigation("Contributor"); + }); + + modelBuilder.Entity("DataLayer.Category", b => + { + b.HasOne("DataLayer.Category", "ParentCategory") + .WithMany() + .HasForeignKey("ParentCategoryCategoryId"); + + b.Navigation("ParentCategory"); + }); + + modelBuilder.Entity("DataLayer.LibraryBook", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithOne() + .HasForeignKey("DataLayer.LibraryBook", "BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Book"); + }); + + modelBuilder.Entity("DataLayer.SeriesBook", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithMany("SeriesLink") + .HasForeignKey("BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DataLayer.Series", "Series") + .WithMany("BooksLink") + .HasForeignKey("SeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Book"); + + b.Navigation("Series"); + }); + + modelBuilder.Entity("DataLayer.Book", b => + { + b.Navigation("ContributorsLink"); + + b.Navigation("SeriesLink"); + }); + + modelBuilder.Entity("DataLayer.Contributor", b => + { + b.Navigation("BooksLink"); + }); + + modelBuilder.Entity("DataLayer.Series", b => + { + b.Navigation("BooksLink"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.cs b/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.cs new file mode 100644 index 00000000..0630e2ff --- /dev/null +++ b/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DataLayer.Migrations +{ + /// + public partial class AddAbsentFromLastScan : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "AbsentFromLastScan", + table: "LibraryBooks", + type: "INTEGER", + nullable: false, + defaultValue: false); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "AbsentFromLastScan", + table: "LibraryBooks"); + } + } +} diff --git a/Source/DataLayer/Migrations/LibationContextModelSnapshot.cs b/Source/DataLayer/Migrations/LibationContextModelSnapshot.cs index 747513d6..2a0a1869 100644 --- a/Source/DataLayer/Migrations/LibationContextModelSnapshot.cs +++ b/Source/DataLayer/Migrations/LibationContextModelSnapshot.cs @@ -157,6 +157,9 @@ namespace DataLayer.Migrations b.Property("BookId") .HasColumnType("INTEGER"); + b.Property("AbsentFromLastScan") + .HasColumnType("INTEGER"); + b.Property("Account") .HasColumnType("TEXT"); diff --git a/Source/DtoImporterService/ImportItem.cs b/Source/DtoImporterService/ImportItem.cs index 990ee48f..f01908e8 100644 --- a/Source/DtoImporterService/ImportItem.cs +++ b/Source/DtoImporterService/ImportItem.cs @@ -8,5 +8,7 @@ namespace DtoImporterService public Item DtoItem { get; set; } public string AccountId { get; set; } public string LocaleName { get; set; } + public override string ToString() + => DtoItem is null ? base.ToString() : $"[{DtoItem.ProductId}] {DtoItem.Title}"; } } diff --git a/Source/DtoImporterService/LibraryBookImporter.cs b/Source/DtoImporterService/LibraryBookImporter.cs index 38581b7c..77ba8aad 100644 --- a/Source/DtoImporterService/LibraryBookImporter.cs +++ b/Source/DtoImporterService/LibraryBookImporter.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using AudibleUtilities; using DataLayer; +using Dinah.Core; using Dinah.Core.Collections.Generic; namespace DtoImporterService @@ -40,9 +41,8 @@ namespace DtoImporterService // // CURRENT SOLUTION: don't re-insert - var currentLibraryProductIds = DbContext.LibraryBooks.Select(l => l.Book.AudibleProductId).ToList(); var newItems = importItems - .Where(dto => !currentLibraryProductIds.Contains(dto.DtoItem.ProductId)) + .ExceptBy(DbContext.LibraryBooks.Select(lb => lb.Book.AudibleProductId), imp => imp.DtoItem.ProductId) .ToList(); // if 2 accounts try to import the same book in the same transaction: error since we're only tracking and pulling by asin. @@ -55,7 +55,11 @@ namespace DtoImporterService var libraryBook = new LibraryBook( bookImporter.Cache[newItem.DtoItem.ProductId], newItem.DtoItem.DateAdded, - newItem.AccountId); + newItem.AccountId) + { + AbsentFromLastScan = isPlusTitleUnavailable(newItem) + }; + try { DbContext.LibraryBooks.Add(libraryBook); @@ -66,8 +70,30 @@ namespace DtoImporterService } } + //If an existing Book wasn't found in the import, the owning LibraryBook's Book will be null. + foreach (var nullBook in DbContext.LibraryBooks.AsEnumerable().Where(lb => lb.Book is null)) + nullBook.AbsentFromLastScan = true; + + //Join importItems on LibraryBooks before iterating over LibraryBooks to avoid + //quadratic complexity caused by searching all of importItems for each LibraryBook. + //Join uses hashing, so complexity should approach O(N) instead of O(N^2). + var items_lbs + = importItems + .Join(DbContext.LibraryBooks, o => (o.AccountId, o.DtoItem.ProductId), i => (i.Account, i.Book?.AudibleProductId), (o, i) => (o, i)); + + foreach ((ImportItem item, LibraryBook lb) in items_lbs) + lb.AbsentFromLastScan = isPlusTitleUnavailable(item); + var qtyNew = hash.Count; return qtyNew; } + + + //This SEEMS to work to detect plus titles which are no longer available. + //I have my doubts it won't yield false negatives, but I have more + //confidence that it won't yield many/any false positives. + private static bool isPlusTitleUnavailable(ImportItem item) + => item.DtoItem.IsAyce is true + && !item.DtoItem.Plans.Any(p => p.PlanName.ContainsInsensitive("Minerva") || p.PlanName.ContainsInsensitive("Free")); } } From 3ebd4ce24381df4fa01e7716a6bea440dfa87700 Mon Sep 17 00:00:00 2001 From: MBucari Date: Tue, 7 Mar 2023 19:36:15 -0700 Subject: [PATCH 5/9] Show AbsentFromLastScan book status in grid --- Source/ApplicationServices/LibraryCommands.cs | 1 + .../DataLayer/QueryObjects/LibraryBookQueries.cs | 5 +++-- Source/LibationAvalonia/Assets/LibationStyles.xaml | 2 ++ Source/LibationAvalonia/ViewModels/GridEntry.cs | 1 - .../ViewModels/LiberateButtonStatus.cs | 14 ++++++++++++-- .../ViewModels/LibraryBookEntry.cs | 4 ++-- Source/LibationAvalonia/ViewModels/SeriesEntry.cs | 3 +-- .../Views/MainWindow.VisibleBooks.cs | 1 + .../LibationAvalonia/Views/ProductsDisplay.axaml | 10 +++++++--- .../Views/ProductsDisplay.axaml.cs | 6 +++--- Source/LibationWinForms/Form1.VisibleBooks.cs | 2 +- .../GridView/LiberateButtonStatus.cs | 12 +++++++++++- .../LiberateDataGridViewImageButtonColumn.cs | 14 ++++++++++++-- .../LibationWinForms/GridView/LibraryBookEntry.cs | 2 +- .../LibationWinForms/GridView/ProductsDisplay.cs | 3 ++- Source/LibationWinForms/GridView/ProductsGrid.cs | 2 +- Source/LibationWinForms/GridView/SeriesEntry.cs | 2 +- 17 files changed, 61 insertions(+), 23 deletions(-) diff --git a/Source/ApplicationServices/LibraryCommands.cs b/Source/ApplicationServices/LibraryCommands.cs index d166a158..e5a9b04f 100644 --- a/Source/ApplicationServices/LibraryCommands.cs +++ b/Source/ApplicationServices/LibraryCommands.cs @@ -467,6 +467,7 @@ namespace ApplicationServices var results = libraryBooks .AsParallel() + .Where(lb => !lb.AbsentFromLastScan) .Select(lb => Liberated_Status(lb.Book)) .ToList(); var booksFullyBackedUp = results.Count(r => r == LiberatedStatus.Liberated); diff --git a/Source/DataLayer/QueryObjects/LibraryBookQueries.cs b/Source/DataLayer/QueryObjects/LibraryBookQueries.cs index 24063f0c..f10350f5 100644 --- a/Source/DataLayer/QueryObjects/LibraryBookQueries.cs +++ b/Source/DataLayer/QueryObjects/LibraryBookQueries.cs @@ -107,8 +107,9 @@ namespace DataLayer => bookList .Where( lb => - lb.Book.UserDefinedItem.BookStatus is LiberatedStatus.NotLiberated or LiberatedStatus.PartialDownload - || lb.Book.UserDefinedItem.PdfStatus is LiberatedStatus.NotLiberated or LiberatedStatus.PartialDownload + !lb.AbsentFromLastScan && + (lb.Book.UserDefinedItem.BookStatus is LiberatedStatus.NotLiberated or LiberatedStatus.PartialDownload + || lb.Book.UserDefinedItem.PdfStatus is LiberatedStatus.NotLiberated or LiberatedStatus.PartialDownload) ); } } diff --git a/Source/LibationAvalonia/Assets/LibationStyles.xaml b/Source/LibationAvalonia/Assets/LibationStyles.xaml index 7b3ae474..e3569ba2 100644 --- a/Source/LibationAvalonia/Assets/LibationStyles.xaml +++ b/Source/LibationAvalonia/Assets/LibationStyles.xaml @@ -8,6 +8,8 @@ + +