From de34e5c795ec85d1d7d1989bbfec61c1f734381a Mon Sep 17 00:00:00 2001 From: Robert McRackan Date: Fri, 3 Sep 2021 16:35:31 -0400 Subject: [PATCH] import speed improvements --- ApplicationServices/LibraryCommands.cs | 35 +++++-------------- DataLayer/DataLayer.csproj | 2 +- DataLayer/QueryObjects/GenericPaging.cs | 19 ---------- DtoImporterService/BookImporter.cs | 12 +++---- DtoImporterService/CategoryImporter.cs | 2 +- DtoImporterService/PerfLogger.cs | 34 ++++++++++++++++++ InternalUtilities/InternalUtilities.csproj | 2 +- LibationLauncher/LibationLauncher.csproj | 2 +- LibationWinForms/LibationWinForms.csproj | 2 +- .../InternalUtilities.Tests.csproj | 4 +-- .../LibationSearchEngine.Tests.csproj | 4 +-- 11 files changed, 58 insertions(+), 60 deletions(-) delete mode 100644 DataLayer/QueryObjects/GenericPaging.cs create mode 100644 DtoImporterService/PerfLogger.cs diff --git a/ApplicationServices/LibraryCommands.cs b/ApplicationServices/LibraryCommands.cs index 7fd373b1..a782ec7b 100644 --- a/ApplicationServices/LibraryCommands.cs +++ b/ApplicationServices/LibraryCommands.cs @@ -8,6 +8,7 @@ using Dinah.Core; using DtoImporterService; using InternalUtilities; using Serilog; +using static DtoImporterService.PerfLogger; namespace ApplicationServices { @@ -16,12 +17,12 @@ namespace ApplicationServices private static LibraryOptions.ResponseGroupOptions LibraryResponseGroups = LibraryOptions.ResponseGroupOptions.ALL_OPTIONS; public static async Task> FindInactiveBooks(Func loginCallbackFactoryFunc, List existingLibrary, params Account[] accounts) - { + { //These are the minimum response groups required for the //library scanner to pass all validation and filtering. - LibraryResponseGroups = + LibraryResponseGroups = LibraryOptions.ResponseGroupOptions.ProductAttrs | - LibraryOptions.ResponseGroupOptions.ProductDesc | + LibraryOptions.ResponseGroupOptions.ProductDesc | LibraryOptions.ResponseGroupOptions.Relationships; if (accounts is null || accounts.Length == 0) @@ -59,31 +60,16 @@ namespace ApplicationServices Log.Logger.Error(ex, "Error importing library"); throw; } - finally - { + finally + { LibraryResponseGroups = LibraryOptions.ResponseGroupOptions.ALL_OPTIONS; - } - } - - static System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch(); - record timeLogEntry(string msg, long totalElapsed, long delta); - static List __log { get; } = new List(); - static void logTime(string s) - { - var totalElapsed = sw.ElapsedMilliseconds; - - var prev = __log.Last().totalElapsed; - var delta = totalElapsed - prev; - - __log.Add(new(s, totalElapsed, delta)); + } } #region FULL LIBRARY scan and import public static async Task<(int totalCount, int newCount)> ImportAccountAsync(Func loginCallbackFactoryFunc, params Account[] accounts) { - __log.Clear(); - __log.Add(new("begin", 0, 0)); - sw.Restart(); + logRestart(); if (accounts is null || accounts.Length == 0) return (0, 0); @@ -129,10 +115,7 @@ namespace ApplicationServices } finally { - sw.Stop(); - var logOutput - = $"{nameof(timeLogEntry.msg)}\t{nameof(timeLogEntry.totalElapsed)}\t{nameof(timeLogEntry.delta)}\r\n" - + __log.Select(t => $"{t.msg}\t{t.totalElapsed}\t{t.delta}").Aggregate((a, b) => $"{a}\r\n{b}"); + stop(); var putBreakPointHere = logOutput; } } diff --git a/DataLayer/DataLayer.csproj b/DataLayer/DataLayer.csproj index 0cd306cb..5b6f5608 100644 --- a/DataLayer/DataLayer.csproj +++ b/DataLayer/DataLayer.csproj @@ -12,7 +12,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/DataLayer/QueryObjects/GenericPaging.cs b/DataLayer/QueryObjects/GenericPaging.cs deleted file mode 100644 index 389ec491..00000000 --- a/DataLayer/QueryObjects/GenericPaging.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Linq; - -namespace DataLayer -{ - public static class GenericPaging - { - public static IQueryable Page(this IQueryable query, int pageNumZeroStart, int pageSize) - { - if (pageSize < 1) - throw new ArgumentOutOfRangeException(nameof(pageSize), "pageSize must be at least 1"); - - if (pageNumZeroStart > 0) - query = query.Skip(pageNumZeroStart * pageSize); - - return query.Take(pageSize); - } - } -} \ No newline at end of file diff --git a/DtoImporterService/BookImporter.cs b/DtoImporterService/BookImporter.cs index 6e7b96a1..28d45f5d 100644 --- a/DtoImporterService/BookImporter.cs +++ b/DtoImporterService/BookImporter.cs @@ -19,13 +19,13 @@ namespace DtoImporterService new ContributorImporter(DbContext).Import(importItems); new SeriesImporter(DbContext).Import(importItems); new CategoryImporter(DbContext).Import(importItems); - + // get distinct - var productIds = importItems.Select(i => i.DtoItem.ProductId).ToList(); - + var productIds = importItems.Select(i => i.DtoItem.ProductId).Distinct().ToList(); + // load db existing => .Local loadLocal_books(productIds); - + // upsert var qtyNew = upsertBooks(importItems); return qtyNew; @@ -33,9 +33,9 @@ namespace DtoImporterService private void loadLocal_books(List productIds) { - var localProductIds = DbContext.Books.Local.Select(b => b.AudibleProductId); + // if this context has already loaded books, don't need to reload them. vestige from when context was long-lived. in practice, we now typically use a fresh context. this is quick though so no harm in leaving it. + var localProductIds = DbContext.Books.Local.Select(b => b.AudibleProductId).ToList(); var remainingProductIds = productIds - .Distinct() .Except(localProductIds) .ToList(); diff --git a/DtoImporterService/CategoryImporter.cs b/DtoImporterService/CategoryImporter.cs index cf66fd5e..f778e813 100644 --- a/DtoImporterService/CategoryImporter.cs +++ b/DtoImporterService/CategoryImporter.cs @@ -35,7 +35,7 @@ namespace DtoImporterService private void loadLocal_categories(List categoryIds) { - var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId); + var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId).ToList(); var remainingCategoryIds = categoryIds .Distinct() .Except(localIds) diff --git a/DtoImporterService/PerfLogger.cs b/DtoImporterService/PerfLogger.cs new file mode 100644 index 00000000..7fcc5bbd --- /dev/null +++ b/DtoImporterService/PerfLogger.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; + +namespace DtoImporterService +{ + public record timeLogEntry(string msg, long totalElapsed, long delta); + public static class PerfLogger + { + private static Stopwatch sw = new Stopwatch(); + private static List __log { get; } = new List(); + + public static void logTime(string s) + { + var totalElapsed = sw.ElapsedMilliseconds; + + var prev = __log.Last().totalElapsed; + var delta = totalElapsed - prev; + + __log.Add(new(s, totalElapsed, delta)); + } + public static void logRestart() + { + __log.Clear(); + __log.Add(new("begin", 0, 0)); + sw.Restart(); + } + public static void stop() => sw.Stop(); + public static string logOutput => + $"{nameof(timeLogEntry.msg)}\t{nameof(timeLogEntry.totalElapsed)}\t{nameof(timeLogEntry.delta)}\r\n" + + __log.Select(t => $"{t.msg}\t{t.totalElapsed}\t{t.delta}").Aggregate((a, b) => $"{a}\r\n{b}"); + } +} diff --git a/InternalUtilities/InternalUtilities.csproj b/InternalUtilities/InternalUtilities.csproj index 577245f3..c96bf718 100644 --- a/InternalUtilities/InternalUtilities.csproj +++ b/InternalUtilities/InternalUtilities.csproj @@ -5,7 +5,7 @@ - + diff --git a/LibationLauncher/LibationLauncher.csproj b/LibationLauncher/LibationLauncher.csproj index 9213d30f..06038a13 100644 --- a/LibationLauncher/LibationLauncher.csproj +++ b/LibationLauncher/LibationLauncher.csproj @@ -13,7 +13,7 @@ win-x64 - 5.6.3.10 + 5.6.4.1 diff --git a/LibationWinForms/LibationWinForms.csproj b/LibationWinForms/LibationWinForms.csproj index f5aeb1a8..4f6d0f3c 100644 --- a/LibationWinForms/LibationWinForms.csproj +++ b/LibationWinForms/LibationWinForms.csproj @@ -13,7 +13,7 @@ - + diff --git a/_Tests/InternalUtilities.Tests/InternalUtilities.Tests.csproj b/_Tests/InternalUtilities.Tests/InternalUtilities.Tests.csproj index c2dd7783..d3b146e6 100644 --- a/_Tests/InternalUtilities.Tests/InternalUtilities.Tests.csproj +++ b/_Tests/InternalUtilities.Tests/InternalUtilities.Tests.csproj @@ -10,8 +10,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj b/_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj index 4417cbca..bfb4122f 100644 --- a/_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj +++ b/_Tests/LibationSearchEngine.Tests/LibationSearchEngine.Tests.csproj @@ -10,8 +10,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive