import speed improvements
This commit is contained in:
Robert McRackan
parent
8ffcefd6ae
commit
de34e5c795
@ -8,6 +8,7 @@ using Dinah.Core;
|
||||
using DtoImporterService;
|
||||
using InternalUtilities;
|
||||
using Serilog;
|
||||
using static DtoImporterService.PerfLogger;
|
||||
|
||||
namespace ApplicationServices
|
||||
{
|
||||
@ -16,12 +17,12 @@ namespace ApplicationServices
|
||||
private static LibraryOptions.ResponseGroupOptions LibraryResponseGroups = LibraryOptions.ResponseGroupOptions.ALL_OPTIONS;
|
||||
|
||||
public static async Task<List<LibraryBook>> FindInactiveBooks(Func<Account, ILoginCallback> loginCallbackFactoryFunc, List<LibraryBook> existingLibrary, params Account[] accounts)
|
||||
{
|
||||
{
|
||||
//These are the minimum response groups required for the
|
||||
//library scanner to pass all validation and filtering.
|
||||
LibraryResponseGroups =
|
||||
LibraryResponseGroups =
|
||||
LibraryOptions.ResponseGroupOptions.ProductAttrs |
|
||||
LibraryOptions.ResponseGroupOptions.ProductDesc |
|
||||
LibraryOptions.ResponseGroupOptions.ProductDesc |
|
||||
LibraryOptions.ResponseGroupOptions.Relationships;
|
||||
|
||||
if (accounts is null || accounts.Length == 0)
|
||||
@ -59,31 +60,16 @@ namespace ApplicationServices
|
||||
Log.Logger.Error(ex, "Error importing library");
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
finally
|
||||
{
|
||||
LibraryResponseGroups = LibraryOptions.ResponseGroupOptions.ALL_OPTIONS;
|
||||
}
|
||||
}
|
||||
|
||||
static System.Diagnostics.Stopwatch sw = new System.Diagnostics.Stopwatch();
|
||||
record timeLogEntry(string msg, long totalElapsed, long delta);
|
||||
static List<timeLogEntry> __log { get; } = new List<timeLogEntry>();
|
||||
static void logTime(string s)
|
||||
{
|
||||
var totalElapsed = sw.ElapsedMilliseconds;
|
||||
|
||||
var prev = __log.Last().totalElapsed;
|
||||
var delta = totalElapsed - prev;
|
||||
|
||||
__log.Add(new(s, totalElapsed, delta));
|
||||
}
|
||||
}
|
||||
|
||||
#region FULL LIBRARY scan and import
|
||||
public static async Task<(int totalCount, int newCount)> ImportAccountAsync(Func<Account, ILoginCallback> loginCallbackFactoryFunc, params Account[] accounts)
|
||||
{
|
||||
__log.Clear();
|
||||
__log.Add(new("begin", 0, 0));
|
||||
sw.Restart();
|
||||
logRestart();
|
||||
|
||||
if (accounts is null || accounts.Length == 0)
|
||||
return (0, 0);
|
||||
@ -129,10 +115,7 @@ namespace ApplicationServices
|
||||
}
|
||||
finally
|
||||
{
|
||||
sw.Stop();
|
||||
var logOutput
|
||||
= $"{nameof(timeLogEntry.msg)}\t{nameof(timeLogEntry.totalElapsed)}\t{nameof(timeLogEntry.delta)}\r\n"
|
||||
+ __log.Select(t => $"{t.msg}\t{t.totalElapsed}\t{t.delta}").Aggregate((a, b) => $"{a}\r\n{b}");
|
||||
stop();
|
||||
var putBreakPointHere = logOutput;
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Dinah.EntityFrameworkCore" Version="1.0.2.2" />
|
||||
<PackageReference Include="Dinah.EntityFrameworkCore" Version="1.0.5.1" />
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="5.0.9">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace DataLayer
|
||||
{
|
||||
public static class GenericPaging
|
||||
{
|
||||
public static IQueryable<T> Page<T>(this IQueryable<T> query, int pageNumZeroStart, int pageSize)
|
||||
{
|
||||
if (pageSize < 1)
|
||||
throw new ArgumentOutOfRangeException(nameof(pageSize), "pageSize must be at least 1");
|
||||
|
||||
if (pageNumZeroStart > 0)
|
||||
query = query.Skip(pageNumZeroStart * pageSize);
|
||||
|
||||
return query.Take(pageSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -19,13 +19,13 @@ namespace DtoImporterService
|
||||
new ContributorImporter(DbContext).Import(importItems);
|
||||
new SeriesImporter(DbContext).Import(importItems);
|
||||
new CategoryImporter(DbContext).Import(importItems);
|
||||
|
||||
|
||||
// get distinct
|
||||
var productIds = importItems.Select(i => i.DtoItem.ProductId).ToList();
|
||||
|
||||
var productIds = importItems.Select(i => i.DtoItem.ProductId).Distinct().ToList();
|
||||
|
||||
// load db existing => .Local
|
||||
loadLocal_books(productIds);
|
||||
|
||||
|
||||
// upsert
|
||||
var qtyNew = upsertBooks(importItems);
|
||||
return qtyNew;
|
||||
@ -33,9 +33,9 @@ namespace DtoImporterService
|
||||
|
||||
private void loadLocal_books(List<string> productIds)
|
||||
{
|
||||
var localProductIds = DbContext.Books.Local.Select(b => b.AudibleProductId);
|
||||
// if this context has already loaded books, don't need to reload them. vestige from when context was long-lived. in practice, we now typically use a fresh context. this is quick though so no harm in leaving it.
|
||||
var localProductIds = DbContext.Books.Local.Select(b => b.AudibleProductId).ToList();
|
||||
var remainingProductIds = productIds
|
||||
.Distinct()
|
||||
.Except(localProductIds)
|
||||
.ToList();
|
||||
|
||||
|
||||
@ -35,7 +35,7 @@ namespace DtoImporterService
|
||||
|
||||
private void loadLocal_categories(List<string> categoryIds)
|
||||
{
|
||||
var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId);
|
||||
var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId).ToList();
|
||||
var remainingCategoryIds = categoryIds
|
||||
.Distinct()
|
||||
.Except(localIds)
|
||||
|
||||
34
DtoImporterService/PerfLogger.cs
Normal file
34
DtoImporterService/PerfLogger.cs
Normal file
@ -0,0 +1,34 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
|
||||
namespace DtoImporterService
|
||||
{
|
||||
public record timeLogEntry(string msg, long totalElapsed, long delta);
|
||||
public static class PerfLogger
|
||||
{
|
||||
private static Stopwatch sw = new Stopwatch();
|
||||
private static List<timeLogEntry> __log { get; } = new List<timeLogEntry>();
|
||||
|
||||
public static void logTime(string s)
|
||||
{
|
||||
var totalElapsed = sw.ElapsedMilliseconds;
|
||||
|
||||
var prev = __log.Last().totalElapsed;
|
||||
var delta = totalElapsed - prev;
|
||||
|
||||
__log.Add(new(s, totalElapsed, delta));
|
||||
}
|
||||
public static void logRestart()
|
||||
{
|
||||
__log.Clear();
|
||||
__log.Add(new("begin", 0, 0));
|
||||
sw.Restart();
|
||||
}
|
||||
public static void stop() => sw.Stop();
|
||||
public static string logOutput =>
|
||||
$"{nameof(timeLogEntry.msg)}\t{nameof(timeLogEntry.totalElapsed)}\t{nameof(timeLogEntry.delta)}\r\n"
|
||||
+ __log.Select(t => $"{t.msg}\t{t.totalElapsed}\t{t.delta}").Aggregate((a, b) => $"{a}\r\n{b}");
|
||||
}
|
||||
}
|
||||
@ -5,7 +5,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AudibleApi" Version="1.1.1.1" />
|
||||
<PackageReference Include="AudibleApi" Version="1.1.5.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
<!-- <PublishSingleFile>true</PublishSingleFile> -->
|
||||
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
|
||||
|
||||
<Version>5.6.3.10</Version>
|
||||
<Version>5.6.4.1</Version>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Dinah.Core.WindowsDesktop" Version="1.0.2.1" />
|
||||
<PackageReference Include="Dinah.Core.WindowsDesktop" Version="1.0.5.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@ -10,8 +10,8 @@
|
||||
<PackageReference Include="FluentAssertions" Version="6.1.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
|
||||
<PackageReference Include="Moq" Version="4.16.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.2.6" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.2.6" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.2.7" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.2.7" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.1.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
|
||||
@ -10,8 +10,8 @@
|
||||
<PackageReference Include="FluentAssertions" Version="6.1.0" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
|
||||
<PackageReference Include="Moq" Version="4.16.1" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.2.6" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.2.6" />
|
||||
<PackageReference Include="MSTest.TestAdapter" Version="2.2.7" />
|
||||
<PackageReference Include="MSTest.TestFramework" Version="2.2.7" />
|
||||
<PackageReference Include="coverlet.collector" Version="3.1.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user