diff --git a/AaxDecrypter/AaxDecrypter.csproj b/AaxDecrypter/AaxDecrypter.csproj index 32994916..6940da8d 100644 --- a/AaxDecrypter/AaxDecrypter.csproj +++ b/AaxDecrypter/AaxDecrypter.csproj @@ -6,7 +6,7 @@ - + diff --git a/AppScaffolding/AppScaffolding.csproj b/AppScaffolding/AppScaffolding.csproj index 27f6b566..22abde40 100644 --- a/AppScaffolding/AppScaffolding.csproj +++ b/AppScaffolding/AppScaffolding.csproj @@ -3,7 +3,7 @@ net6.0-windows - 6.8.4.1 + 7.0.0.1 @@ -11,7 +11,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/ApplicationServices/LibraryCommands.cs b/ApplicationServices/LibraryCommands.cs index 9609a25c..f58fc3d0 100644 --- a/ApplicationServices/LibraryCommands.cs +++ b/ApplicationServices/LibraryCommands.cs @@ -89,6 +89,9 @@ namespace ApplicationServices var totalCount = importItems.Count; Log.Logger.Information($"GetAllLibraryItems: Total count {totalCount}"); + if (totalCount == 0) + return default; + Log.Logger.Information("Begin long-running import"); logTime($"pre {nameof(importIntoDbAsync)}"); var newCount = await importIntoDbAsync(importItems); diff --git a/AudibleUtilities/AudibleUtilities.csproj b/AudibleUtilities/AudibleUtilities.csproj index cb36d510..ca3ccb51 100644 --- a/AudibleUtilities/AudibleUtilities.csproj +++ b/AudibleUtilities/AudibleUtilities.csproj @@ -5,7 +5,7 @@ - + diff --git a/DataLayer/Configurations/BookContributorConfig.cs b/DataLayer/Configurations/BookContributorConfig.cs index f3bc09c7..5fd789ab 100644 --- a/DataLayer/Configurations/BookContributorConfig.cs +++ b/DataLayer/Configurations/BookContributorConfig.cs @@ -9,8 +9,8 @@ namespace DataLayer.Configurations { entity.HasKey(bc => new { bc.BookId, bc.ContributorId, bc.Role }); - entity.HasIndex(b => b.BookId); - entity.HasIndex(b => b.ContributorId); + entity.HasIndex(bc => bc.BookId); + entity.HasIndex(bc => bc.ContributorId); entity .HasOne(bc => bc.Book) diff --git a/DataLayer/Configurations/LibraryBookConfig.cs b/DataLayer/Configurations/LibraryBookConfig.cs index 5f1e3736..c503b9d1 100644 --- a/DataLayer/Configurations/LibraryBookConfig.cs +++ b/DataLayer/Configurations/LibraryBookConfig.cs @@ -21,12 +21,12 @@ namespace DataLayer.Configurations // - update LibraryBook import code // - would likely challenge assumptions throughout Libation which have been true up until now - entity.HasKey(b => b.BookId); + entity.HasKey(lb => lb.BookId); entity - .HasOne(le => le.Book) + .HasOne(lb => lb.Book) .WithOne() - .HasForeignKey(le => le.BookId); + .HasForeignKey(lb => lb.BookId); } } } \ No newline at end of file diff --git a/DataLayer/Configurations/SeriesBookConfig.cs b/DataLayer/Configurations/SeriesBookConfig.cs index c848fa9a..0e541486 100644 --- a/DataLayer/Configurations/SeriesBookConfig.cs +++ b/DataLayer/Configurations/SeriesBookConfig.cs @@ -7,10 +7,10 @@ namespace DataLayer.Configurations { public void Configure(EntityTypeBuilder entity) { - entity.HasKey(bc => new { bc.SeriesId, bc.BookId }); + entity.HasKey(sb => new { sb.SeriesId, sb.BookId }); - entity.HasIndex(b => b.SeriesId); - entity.HasIndex(b => b.BookId); + entity.HasIndex(sb => sb.SeriesId); + entity.HasIndex(sb => sb.BookId); entity .HasOne(sb => sb.Series) diff --git a/DataLayer/Configurations/SeriesConfig.cs b/DataLayer/Configurations/SeriesConfig.cs index 7d067cf4..70530b85 100644 --- a/DataLayer/Configurations/SeriesConfig.cs +++ b/DataLayer/Configurations/SeriesConfig.cs @@ -7,8 +7,8 @@ namespace DataLayer.Configurations { public void Configure(EntityTypeBuilder entity) { - entity.HasKey(b => b.SeriesId); - entity.HasIndex(b => b.AudibleSeriesId); + entity.HasKey(s => s.SeriesId); + entity.HasIndex(s => s.AudibleSeriesId); entity .Metadata diff --git a/DataLayer/EfClasses/Contributor.cs b/DataLayer/EfClasses/Contributor.cs index f050c9aa..9c88d96b 100644 --- a/DataLayer/EfClasses/Contributor.cs +++ b/DataLayer/EfClasses/Contributor.cs @@ -31,16 +31,12 @@ namespace DataLayer public string AudibleContributorId { get; private set; } private Contributor() { } - public Contributor(string name) + public Contributor(string name, string audibleContributorId = null) { - ArgumentValidator.EnsureNotNullOrWhiteSpace(name, nameof(name)); + Name = ArgumentValidator.EnsureNotNullOrWhiteSpace(name, nameof(name)); _booksLink = new HashSet(); - Name = name; - } - public Contributor(string name, string audibleContributorId) : this(name) - { // don't overwrite with null or whitespace but not an error if (!string.IsNullOrWhiteSpace(audibleContributorId)) AudibleContributorId = audibleContributorId; diff --git a/DataLayer/QueryObjects/BookQueries.cs b/DataLayer/QueryObjects/BookQueries.cs index 581c810e..e54bb42f 100644 --- a/DataLayer/QueryObjects/BookQueries.cs +++ b/DataLayer/QueryObjects/BookQueries.cs @@ -19,7 +19,8 @@ namespace DataLayer public static Book GetBook(this IQueryable books, string productId) => books .GetBooks() - .SingleOrDefault(b => b.AudibleProductId == productId); + // 'Single' is more accurate but 'First' is faster and less error prone + .FirstOrDefault(b => b.AudibleProductId == productId); /// This is still IQueryable. YOU MUST CALL ToList() YOURSELF public static IQueryable GetBooks(this IQueryable books, Expression> predicate) diff --git a/DtoImporterService/BookImporter.cs b/DtoImporterService/BookImporter.cs index e4f619ed..5e8d468e 100644 --- a/DtoImporterService/BookImporter.cs +++ b/DtoImporterService/BookImporter.cs @@ -4,62 +4,53 @@ using System.Linq; using AudibleApi.Common; using AudibleUtilities; using DataLayer; +using Dinah.Core.Collections.Generic; namespace DtoImporterService { public class BookImporter : ItemsImporterBase { - public BookImporter(LibationContext context) : base(context) { } + protected override IValidator Validator => new BookValidator(); - public override IEnumerable Validate(IEnumerable importItems) => new BookValidator().Validate(importItems.Select(i => i.DtoItem)); + public Dictionary Cache { get; private set; } = new(); + + private ContributorImporter contributorImporter { get; } + private SeriesImporter seriesImporter { get; } + private CategoryImporter categoryImporter { get; } + + public BookImporter(LibationContext context) : base(context) + { + contributorImporter = new ContributorImporter(DbContext); + seriesImporter = new SeriesImporter(DbContext); + categoryImporter = new CategoryImporter(DbContext); + } protected override int DoImport(IEnumerable importItems) { // pre-req.s - new ContributorImporter(DbContext).Import(importItems); - new SeriesImporter(DbContext).Import(importItems); - new CategoryImporter(DbContext).Import(importItems); - - // get distinct - var productIds = importItems.Select(i => i.DtoItem.ProductId).Distinct().ToList(); - - // load db existing => .Local - loadLocal_books(productIds); - + contributorImporter.Import(importItems); + seriesImporter.Import(importItems); + categoryImporter.Import(importItems); + + // load db existing => hash table + loadLocal_books(importItems); + // upsert var qtyNew = upsertBooks(importItems); return qtyNew; } - private void loadLocal_books(List productIds) + private void loadLocal_books(IEnumerable importItems) { - // if this context has already loaded books, don't need to reload them. vestige from when context was long-lived. in practice, we now typically use a fresh context. this is quick though so no harm in leaving it. - var localProductIds = DbContext.Books.Local.Select(b => b.AudibleProductId).ToList(); - var remainingProductIds = productIds - .Except(localProductIds) + // get distinct + var productIds = importItems + .Select(i => i.DtoItem.ProductId) + .Distinct() .ToList(); - #region // explanation of DbContext.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList(); - /* - articles suggest loading to Local with - context.Books.Load(); - we want Books and associated fields - context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList(); - this is emulating Load() but with also getting associated fields - - from: Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions - // Summary: - // Enumerates the query. When using Entity Framework, this causes the results of - // the query to be loaded into the associated context. This is equivalent to calling - // ToList and then throwing away the list (without the overhead of actually creating - // the list). - public static void Load([NotNullAttribute] this IQueryable source); - */ - #endregion - - // GetBooks() eager loads Series, category, et al - if (remainingProductIds.Any()) - DbContext.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList(); + Cache = DbContext.Books + .GetBooks(b => productIds.Contains(b.AudibleProductId)) + .ToDictionarySafe(b => b.AudibleProductId); } private int upsertBooks(IEnumerable importItems) @@ -68,8 +59,7 @@ namespace DtoImporterService foreach (var item in importItems) { - var book = DbContext.Books.Local.FirstOrDefault(p => p.AudibleProductId == item.DtoItem.ProductId); - if (book is null) + if (!Cache.TryGetValue(item.DtoItem.ProductId, out var book)) { book = createNewBook(item); qtyNew++; @@ -94,8 +84,7 @@ namespace DtoImporterService // nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db var authors = item .Authors - // This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive - .Select(a => DbContext.Contributors.Local.FirstOrDefault(c => a.Name == c.Name)) + .Select(a => contributorImporter.Cache[a.Name]) .ToList(); var narrators @@ -105,8 +94,7 @@ namespace DtoImporterService // nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db : item .Narrators - // This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive - .Select(n => DbContext.Contributors.Local.FirstOrDefault(c => n.Name == c.Name)) + .Select(n => contributorImporter.Cache[n.Name]) .ToList(); // categories are laid out for a breadcrumb. category is 1st, subcategory is 2nd @@ -120,8 +108,7 @@ namespace DtoImporterService // 2+ : item.Categories[1].CategoryId; - // This should properly be SingleOrDefault() not FirstOrDefault(), but FirstOrDefault is defensive - var category = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == lastCategory); + var category = categoryImporter.Cache[lastCategory]; Book book; try @@ -137,6 +124,7 @@ namespace DtoImporterService category, importItem.LocaleName) ).Entity; + Cache.Add(book.AudibleProductId, book); } catch (Exception ex) { @@ -157,8 +145,7 @@ namespace DtoImporterService var publisherName = item.Publisher; if (!string.IsNullOrWhiteSpace(publisherName)) { - // This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive - var publisher = DbContext.Contributors.Local.FirstOrDefault(c => publisherName == c.Name); + var publisher = contributorImporter.Cache[publisherName]; book.ReplacePublisher(publisher); } @@ -189,7 +176,7 @@ namespace DtoImporterService { foreach (var seriesEntry in item.Series) { - var series = DbContext.Series.Local.FirstOrDefault(s => seriesEntry.SeriesId == s.AudibleSeriesId); + var series = seriesImporter.Cache[seriesEntry.SeriesId]; book.UpsertSeries(series, seriesEntry.Sequence); } } diff --git a/DtoImporterService/CategoryImporter.cs b/DtoImporterService/CategoryImporter.cs index eeba9413..5a3434c6 100644 --- a/DtoImporterService/CategoryImporter.cs +++ b/DtoImporterService/CategoryImporter.cs @@ -4,14 +4,17 @@ using System.Linq; using AudibleApi.Common; using AudibleUtilities; using DataLayer; +using Dinah.Core.Collections.Generic; namespace DtoImporterService { public class CategoryImporter : ItemsImporterBase { - public CategoryImporter(LibationContext context) : base(context) { } + protected override IValidator Validator => new CategoryValidator(); - public override IEnumerable Validate(IEnumerable importItems) => new CategoryValidator().Validate(importItems.Select(i => i.DtoItem)); + public Dictionary Cache { get; private set; } = new(); + + public CategoryImporter(LibationContext context) : base(context) { } protected override int DoImport(IEnumerable importItems) { @@ -19,7 +22,9 @@ namespace DtoImporterService var categoryIds = importItems .Select(i => i.DtoItem) .GetCategoriesDistinct() - .Select(c => c.CategoryId).ToList(); + .Select(c => c.CategoryId) + .Distinct() + .ToList(); // load db existing => .Local loadLocal_categories(categoryIds); @@ -38,15 +43,10 @@ namespace DtoImporterService // must include default/empty/missing categoryIds.Add(Category.GetEmpty().AudibleCategoryId); - var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId).ToList(); - var remainingCategoryIds = categoryIds - .Distinct() - .Except(localIds) - .ToList(); - // load existing => local - if (remainingCategoryIds.Any()) - DbContext.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId)).ToList(); + Cache = DbContext.Categories + .Where(c => categoryIds.Contains(c.AudibleCategoryId)) + .ToDictionarySafe(c => c.AudibleCategoryId); } // only use after loading contributors => local @@ -67,22 +67,11 @@ namespace DtoImporterService Category parentCategory = null; if (i == 1) - // should be "Single()" but user is getting a strange error - parentCategory = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == pair[0].CategoryId); + Cache.TryGetValue(pair[0].CategoryId, out parentCategory); - // should be "SingleOrDefault()" but user is getting a strange error - var category = DbContext.Categories.Local.FirstOrDefault(c => c.AudibleCategoryId == id); - if (category is null) + if (!Cache.TryGetValue(id, out var category)) { - try - { - category = DbContext.Categories.Add(new Category(new AudibleCategoryId(id), name)).Entity; - } - catch (Exception ex) - { - Serilog.Log.Logger.Error(ex, "Error adding category. {@DebugInfo}", new { id, name }); - throw; - } + category = addCategory(id, name); qtyNew++; } @@ -92,5 +81,24 @@ namespace DtoImporterService return qtyNew; } + + private Category addCategory(string id, string name) + { + try + { + var category = new Category(new AudibleCategoryId(id), name); + + var entityEntry = DbContext.Categories.Add(category); + var entity = entityEntry.Entity; + + Cache.Add(entity.AudibleCategoryId, entity); + return entity; + } + catch (Exception ex) + { + Serilog.Log.Logger.Error(ex, "Error adding category. {@DebugInfo}", new { id, name }); + throw; + } + } } } diff --git a/DtoImporterService/ContributorImporter.cs b/DtoImporterService/ContributorImporter.cs index e37a160e..dd3e273f 100644 --- a/DtoImporterService/ContributorImporter.cs +++ b/DtoImporterService/ContributorImporter.cs @@ -4,14 +4,17 @@ using System.Linq; using AudibleApi.Common; using AudibleUtilities; using DataLayer; +using Dinah.Core.Collections.Generic; namespace DtoImporterService { public class ContributorImporter : ItemsImporterBase { - public ContributorImporter(LibationContext context) : base(context) { } + protected override IValidator Validator => new ContributorValidator(); - public override IEnumerable Validate(IEnumerable importItems) => new ContributorValidator().Validate(importItems.Select(i => i.DtoItem)); + public Dictionary Cache { get; private set; } = new(); + + public ContributorImporter(LibationContext context) : base(context) { } protected override int DoImport(IEnumerable importItems) { @@ -50,78 +53,61 @@ namespace DtoImporterService // must include default/empty/missing contributorNames.Add(Contributor.GetEmpty().Name); - //// BAD: very inefficient - // var x = context.Contributors.Local.Where(c => !contribNames.Contains(c.Name)); - - // GOOD: Except() is efficient. Due to hashing, it's close to O(n) - var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList(); - var remainingContribNames = contributorNames - .Distinct() - .Except(localNames) - .ToList(); - // load existing => local - if (remainingContribNames.Any()) - DbContext.Contributors.Where(c => remainingContribNames.Contains(c.Name)).ToList(); + Cache = DbContext.Contributors + .Where(c => contributorNames.Contains(c.Name)) + .ToDictionarySafe(c => c.Name); } - // only use after loading contributors => local private int upsertPeople(List people) { - var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList(); - var newPeople = people - .Select(p => p.Name) - .Distinct() - .Except(localNames) - .ToList(); + var hash = people + // new people only + .Where(p => !Cache.ContainsKey(p.Name)) + // remove duplicates by Name. first in wins + .ToDictionarySafe(p => p.Name); - var groupby = people.GroupBy( - p => p.Name, - p => p, - (key, g) => new { Name = key, People = g.ToList() } - ); - foreach (var name in newPeople) + foreach (var kvp in hash) { - // This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive - var p = groupby.FirstOrDefault(g => g.Name == name).People.First(); - - try - { - DbContext.Contributors.Add(new Contributor(p.Name, p.Asin)); - } - catch (Exception ex) - { - Serilog.Log.Logger.Error(ex, "Error adding person. {@DebugInfo}", new { p?.Name, p?.Asin }); - throw; - } + var person = kvp.Value; + addContributor(person.Name, person.Asin); } - return newPeople.Count; + return hash.Count; } // only use after loading contributors => local private int upsertPublishers(List publishers) { - var localNames = DbContext.Contributors.Local.Select(c => c.Name).ToList(); - var newPublishers = publishers - .Distinct() - .Except(localNames) - .ToList(); + var hash = publishers + // new publishers only + .Where(p => !Cache.ContainsKey(p)) + // remove duplicates + .ToHashSet(); - foreach (var pub in newPublishers) - { - try - { - DbContext.Contributors.Add(new Contributor(pub)); - } - catch (Exception ex) - { - Serilog.Log.Logger.Error(ex, "Error adding publisher. {@DebugInfo}", new { pub }); - throw; - } - } + foreach (var pub in hash) + addContributor(pub); - return newPublishers.Count; + return hash.Count; } - } + + private Contributor addContributor(string name, string id = null) + { + try + { + var newContrib = new Contributor(name); + + var entityEntry = DbContext.Contributors.Add(newContrib); + var entity = entityEntry.Entity; + + Cache.Add(entity.Name, entity); + return entity; + } + catch (Exception ex) + { + Serilog.Log.Logger.Error(ex, "Error adding contributor. {@DebugInfo}", new { name, id }); + throw; + } + } + } } diff --git a/DtoImporterService/ImporterBase.cs b/DtoImporterService/ImporterBase.cs index 5b4984fe..76f1132e 100644 --- a/DtoImporterService/ImporterBase.cs +++ b/DtoImporterService/ImporterBase.cs @@ -53,5 +53,9 @@ namespace DtoImporterService public abstract class ItemsImporterBase : ImporterBase> { protected ItemsImporterBase(LibationContext context) : base(context) { } + + protected abstract IValidator Validator { get; } + public sealed override IEnumerable Validate(IEnumerable importItems) + => Validator.Validate(importItems.Select(i => i.DtoItem)); } } diff --git a/DtoImporterService/LibraryBookImporter.cs b/DtoImporterService/LibraryBookImporter.cs index 5b5e5aa5..634d8494 100644 --- a/DtoImporterService/LibraryBookImporter.cs +++ b/DtoImporterService/LibraryBookImporter.cs @@ -3,18 +3,24 @@ using System.Collections.Generic; using System.Linq; using AudibleUtilities; using DataLayer; +using Dinah.Core.Collections.Generic; namespace DtoImporterService { public class LibraryBookImporter : ItemsImporterBase { - public LibraryBookImporter(LibationContext context) : base(context) { } + protected override IValidator Validator => new LibraryValidator(); - public override IEnumerable Validate(IEnumerable importItems) => new LibraryValidator().Validate(importItems.Select(i => i.DtoItem)); + private BookImporter bookImporter { get; } + + public LibraryBookImporter(LibationContext context) : base(context) + { + bookImporter = new BookImporter(DbContext); + } protected override int DoImport(IEnumerable importItems) { - new BookImporter(DbContext).Import(importItems); + bookImporter.Import(importItems); var qtyNew = upsertLibraryBooks(importItems); return qtyNew; @@ -36,25 +42,18 @@ namespace DtoImporterService var currentLibraryProductIds = DbContext.LibraryBooks.Select(l => l.Book.AudibleProductId).ToList(); var newItems = importItems - .Where(dto => !currentLibraryProductIds - .Contains(dto.DtoItem.ProductId)) + .Where(dto => !currentLibraryProductIds.Contains(dto.DtoItem.ProductId)) .ToList(); // if 2 accounts try to import the same book in the same transaction: error since we're only tracking and pulling by asin. // just use the first - var groupby = newItems.GroupBy( - i => i.DtoItem.ProductId, - i => i, - (key, g) => new { ProductId = key, ImportItems = g.ToList() } - ) - .ToList(); - foreach (var gb in groupby) - { - var newItem = gb.ImportItems.First(); + var hash = newItems.ToDictionarySafe(dto => dto.DtoItem.ProductId); + foreach (var kvp in hash) + { + var newItem = kvp.Value; var libraryBook = new LibraryBook( - // This should properly be Single() not FirstOrDefault(), but FirstOrDefault is defensive - DbContext.Books.Local.FirstOrDefault(b => b.AudibleProductId == newItem.DtoItem.ProductId), + bookImporter.Cache[newItem.DtoItem.ProductId], newItem.DtoItem.DateAdded, newItem.AccountId); try @@ -67,7 +66,7 @@ namespace DtoImporterService } } - var qtyNew = groupby.Count; + var qtyNew = hash.Count; return qtyNew; } } diff --git a/DtoImporterService/SeriesImporter.cs b/DtoImporterService/SeriesImporter.cs index 6c4a5414..67f5d487 100644 --- a/DtoImporterService/SeriesImporter.cs +++ b/DtoImporterService/SeriesImporter.cs @@ -4,14 +4,17 @@ using System.Linq; using AudibleApi.Common; using AudibleUtilities; using DataLayer; +using Dinah.Core.Collections.Generic; namespace DtoImporterService { public class SeriesImporter : ItemsImporterBase { - public SeriesImporter(LibationContext context) : base(context) { } + protected override IValidator Validator => new SeriesValidator(); - public override IEnumerable Validate(IEnumerable importItems) => new SeriesValidator().Validate(importItems.Select(i => i.DtoItem)); + public Dictionary Cache { get; private set; } = new(); + + public SeriesImporter(LibationContext context) : base(context) { } protected override int DoImport(IEnumerable importItems) { @@ -31,15 +34,12 @@ namespace DtoImporterService private void loadLocal_series(List series) { - var seriesIds = series.Select(s => s.SeriesId).ToList(); - var localIds = DbContext.Series.Local.Select(s => s.AudibleSeriesId).ToList(); - var remainingSeriesIds = seriesIds - .Distinct() - .Except(localIds) - .ToList(); + var seriesIds = series.Select(s => s.SeriesId).Distinct().ToList(); - if (remainingSeriesIds.Any()) - DbContext.Series.Where(s => remainingSeriesIds.Contains(s.AudibleSeriesId)).ToList(); + if (seriesIds.Any()) + Cache = DbContext.Series + .Where(s => seriesIds.Contains(s.AudibleSeriesId)) + .ToDictionarySafe(s => s.AudibleSeriesId); } private int upsertSeries(List requestedSeries) @@ -48,18 +48,10 @@ namespace DtoImporterService foreach (var s in requestedSeries) { - var series = DbContext.Series.Local.FirstOrDefault(c => c.AudibleSeriesId == s.SeriesId); - if (series is null) + // AudibleApi.Common.Series.SeriesId == DataLayer.AudibleSeriesId + if (!Cache.TryGetValue(s.SeriesId, out var series)) { - try - { - series = DbContext.Series.Add(new DataLayer.Series(new AudibleSeriesId(s.SeriesId))).Entity; - } - catch (Exception ex) - { - Serilog.Log.Logger.Error(ex, "Error adding series. {@DebugInfo}", new { s?.SeriesId }); - throw; - } + series = addSeries(s.SeriesId); qtyNew++; } series.UpdateName(s.SeriesName); @@ -67,5 +59,24 @@ namespace DtoImporterService return qtyNew; } + + private DataLayer.Series addSeries(string seriesId) + { + try + { + var series = new DataLayer.Series(new AudibleSeriesId(seriesId)); + + var entityEntry = DbContext.Series.Add(series); + var entity = entityEntry.Entity; + + Cache.Add(entity.AudibleSeriesId, entity); + return entity; + } + catch (Exception ex) + { + Serilog.Log.Logger.Error(ex, "Error adding series. {@DebugInfo}", new { seriesId }); + throw; + } + } } } diff --git a/DtoImporterService/_importer notes.txt b/DtoImporterService/_importer notes.txt new file mode 100644 index 00000000..8d0678ff --- /dev/null +++ b/DtoImporterService/_importer notes.txt @@ -0,0 +1,37 @@ +* Local (eg DbContext.Books.Local): indexes/hashes PK and nothing else. Local.Find(PK) is fast. All other searches (eg FirstOrDefault) have awful performance. It deceptively *feels* like we get this partially for free since added/modified entries live here. +* live db: for all importers, fields used for lookup are indexed + +Using BookImporter as an example: since AudibleProductId is indexed, hitting the live db is much faster than using Local. Fastest is putting all in a local hash table + +Note: GetBook/GetBooks eager loads Series, category, et al + +for 1,200 iterations +* load to LocalView + DbContext.Books.Local.FirstOrDefault(p => p.AudibleProductId == item.DtoItem.ProductId) + 27,125 ms +* read from live db + DbContext.Books.GetBook(item.DtoItem.ProductId) + 12,224 ms +* load to hash table: Dictionary + dictionary[item.DtoItem.ProductId]; + 1 ms (yes: ONE) + +With hashtable, somehow memory usage was not significantly affected + +----------------------------------- + +why were we using Local to begin with? + +articles suggest loading to Local with + context.Books.Load(); +this loads this table but not associated fields +we want Books and associated fields + context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList(); +this is emulating Load() but with also getting associated fields + from: Microsoft.EntityFrameworkCore.EntityFrameworkQueryableExtensions + // Summary: + // Enumerates the query. When using Entity Framework, this causes the results of + // the query to be loaded into the associated context. This is equivalent to calling + // ToList and then throwing away the list (without the overhead of actually creating + // the list). + public static void Load([NotNullAttribute] this IQueryable source); diff --git a/FileManager/FileManager.csproj b/FileManager/FileManager.csproj index 2a6d94ba..e3ebc9a8 100644 --- a/FileManager/FileManager.csproj +++ b/FileManager/FileManager.csproj @@ -5,7 +5,7 @@ - + diff --git a/LibationWinForms/LibationWinForms.csproj b/LibationWinForms/LibationWinForms.csproj index bdf6a16c..58b0df72 100644 --- a/LibationWinForms/LibationWinForms.csproj +++ b/LibationWinForms/LibationWinForms.csproj @@ -28,7 +28,7 @@ - + diff --git a/LibationWinForms/Program.cs b/LibationWinForms/Program.cs index db700ae9..9b4af346 100644 --- a/LibationWinForms/Program.cs +++ b/LibationWinForms/Program.cs @@ -228,7 +228,6 @@ namespace LibationWinForms var libhackFiles = Directory.EnumerateDirectories(config.Books, "*.libhack", SearchOption.AllDirectories); using var context = ApplicationServices.DbContexts.GetContext(); - context.Books.Load(); var jArr = JArray.Parse(File.ReadAllText(filePaths)); @@ -248,7 +247,7 @@ namespace LibationWinForms if (fileType == FileType.Unknown || fileType == FileType.AAXC) continue; - var book = context.Books.Local.FirstOrDefault(b => b.AudibleProductId == asin); + var book = context.Books.FirstOrDefault(b => b.AudibleProductId == asin); if (book is null) continue;