From f6dcc0db1db8fca42c76d42f288b5494ea54afd1 Mon Sep 17 00:00:00 2001 From: Mbucari Date: Tue, 7 Mar 2023 16:00:23 -0700 Subject: [PATCH] Add AbsentFromLastScan --- Source/DataLayer/EfClasses/LibraryBook.cs | 1 + ...08013410_AddAbsentFromLastScan.Designer.cs | 413 ++++++++++++++++++ .../20230308013410_AddAbsentFromLastScan.cs | 29 ++ .../LibationContextModelSnapshot.cs | 3 + Source/DtoImporterService/ImportItem.cs | 2 + .../DtoImporterService/LibraryBookImporter.cs | 32 +- 6 files changed, 477 insertions(+), 3 deletions(-) create mode 100644 Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.Designer.cs create mode 100644 Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.cs diff --git a/Source/DataLayer/EfClasses/LibraryBook.cs b/Source/DataLayer/EfClasses/LibraryBook.cs index 02bb5472..449a2477 100644 --- a/Source/DataLayer/EfClasses/LibraryBook.cs +++ b/Source/DataLayer/EfClasses/LibraryBook.cs @@ -12,6 +12,7 @@ namespace DataLayer public string Account { get; private set; } public bool IsDeleted { get; set; } + public bool AbsentFromLastScan { get; set; } private LibraryBook() { } public LibraryBook(Book book, DateTime dateAdded, string account) diff --git a/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.Designer.cs b/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.Designer.cs new file mode 100644 index 00000000..02b58087 --- /dev/null +++ b/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.Designer.cs @@ -0,0 +1,413 @@ +// +using System; +using DataLayer; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DataLayer.Migrations +{ + [DbContext(typeof(LibationContext))] + [Migration("20230308013410_AddAbsentFromLastScan")] + partial class AddAbsentFromLastScan + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "7.0.3"); + + modelBuilder.Entity("DataLayer.Book", b => + { + b.Property("BookId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AudibleProductId") + .HasColumnType("TEXT"); + + b.Property("CategoryId") + .HasColumnType("INTEGER"); + + b.Property("ContentType") + .HasColumnType("INTEGER"); + + b.Property("DatePublished") + .HasColumnType("TEXT"); + + b.Property("Description") + .HasColumnType("TEXT"); + + b.Property("IsAbridged") + .HasColumnType("INTEGER"); + + b.Property("Language") + .HasColumnType("TEXT"); + + b.Property("LengthInMinutes") + .HasColumnType("INTEGER"); + + b.Property("Locale") + .HasColumnType("TEXT"); + + b.Property("PictureId") + .HasColumnType("TEXT"); + + b.Property("PictureLarge") + .HasColumnType("TEXT"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.Property("_audioFormat") + .HasColumnType("INTEGER"); + + b.HasKey("BookId"); + + b.HasIndex("AudibleProductId"); + + b.HasIndex("CategoryId"); + + b.ToTable("Books"); + }); + + modelBuilder.Entity("DataLayer.BookContributor", b => + { + b.Property("BookId") + .HasColumnType("INTEGER"); + + b.Property("ContributorId") + .HasColumnType("INTEGER"); + + b.Property("Role") + .HasColumnType("INTEGER"); + + b.Property("Order") + .HasColumnType("INTEGER"); + + b.HasKey("BookId", "ContributorId", "Role"); + + b.HasIndex("BookId"); + + b.HasIndex("ContributorId"); + + b.ToTable("BookContributor"); + }); + + modelBuilder.Entity("DataLayer.Category", b => + { + b.Property("CategoryId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AudibleCategoryId") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("ParentCategoryCategoryId") + .HasColumnType("INTEGER"); + + b.HasKey("CategoryId"); + + b.HasIndex("AudibleCategoryId"); + + b.HasIndex("ParentCategoryCategoryId"); + + b.ToTable("Categories"); + + b.HasData( + new + { + CategoryId = -1, + AudibleCategoryId = "", + Name = "" + }); + }); + + modelBuilder.Entity("DataLayer.Contributor", b => + { + b.Property("ContributorId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AudibleContributorId") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.HasKey("ContributorId"); + + b.HasIndex("Name"); + + b.ToTable("Contributors"); + + b.HasData( + new + { + ContributorId = -1, + Name = "" + }); + }); + + modelBuilder.Entity("DataLayer.LibraryBook", b => + { + b.Property("BookId") + .HasColumnType("INTEGER"); + + b.Property("AbsentFromLastScan") + .HasColumnType("INTEGER"); + + b.Property("Account") + .HasColumnType("TEXT"); + + b.Property("DateAdded") + .HasColumnType("TEXT"); + + b.Property("IsDeleted") + .HasColumnType("INTEGER"); + + b.HasKey("BookId"); + + b.ToTable("LibraryBooks"); + }); + + modelBuilder.Entity("DataLayer.Series", b => + { + b.Property("SeriesId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AudibleSeriesId") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.HasKey("SeriesId"); + + b.HasIndex("AudibleSeriesId"); + + b.ToTable("Series"); + }); + + modelBuilder.Entity("DataLayer.SeriesBook", b => + { + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.Property("BookId") + .HasColumnType("INTEGER"); + + b.Property("Order") + .HasColumnType("TEXT"); + + b.HasKey("SeriesId", "BookId"); + + b.HasIndex("BookId"); + + b.HasIndex("SeriesId"); + + b.ToTable("SeriesBook"); + }); + + modelBuilder.Entity("DataLayer.Book", b => + { + b.HasOne("DataLayer.Category", "Category") + .WithMany() + .HasForeignKey("CategoryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("DataLayer.Rating", "Rating", b1 => + { + b1.Property("BookId") + .HasColumnType("INTEGER"); + + b1.Property("OverallRating") + .HasColumnType("REAL"); + + b1.Property("PerformanceRating") + .HasColumnType("REAL"); + + b1.Property("StoryRating") + .HasColumnType("REAL"); + + b1.HasKey("BookId"); + + b1.ToTable("Books"); + + b1.WithOwner() + .HasForeignKey("BookId"); + }); + + b.OwnsMany("DataLayer.Supplement", "Supplements", b1 => + { + b1.Property("SupplementId") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b1.Property("BookId") + .HasColumnType("INTEGER"); + + b1.Property("Url") + .HasColumnType("TEXT"); + + b1.HasKey("SupplementId"); + + b1.HasIndex("BookId"); + + b1.ToTable("Supplement"); + + b1.WithOwner("Book") + .HasForeignKey("BookId"); + + b1.Navigation("Book"); + }); + + b.OwnsOne("DataLayer.UserDefinedItem", "UserDefinedItem", b1 => + { + b1.Property("BookId") + .HasColumnType("INTEGER"); + + b1.Property("BookStatus") + .HasColumnType("INTEGER"); + + b1.Property("LastDownloaded") + .HasColumnType("TEXT"); + + b1.Property("LastDownloadedVersion") + .HasColumnType("TEXT"); + + b1.Property("PdfStatus") + .HasColumnType("INTEGER"); + + b1.Property("Tags") + .HasColumnType("TEXT"); + + b1.HasKey("BookId"); + + b1.ToTable("UserDefinedItem", (string)null); + + b1.WithOwner("Book") + .HasForeignKey("BookId"); + + b1.OwnsOne("DataLayer.Rating", "Rating", b2 => + { + b2.Property("UserDefinedItemBookId") + .HasColumnType("INTEGER"); + + b2.Property("OverallRating") + .HasColumnType("REAL"); + + b2.Property("PerformanceRating") + .HasColumnType("REAL"); + + b2.Property("StoryRating") + .HasColumnType("REAL"); + + b2.HasKey("UserDefinedItemBookId"); + + b2.ToTable("UserDefinedItem"); + + b2.WithOwner() + .HasForeignKey("UserDefinedItemBookId"); + }); + + b1.Navigation("Book"); + + b1.Navigation("Rating"); + }); + + b.Navigation("Category"); + + b.Navigation("Rating"); + + b.Navigation("Supplements"); + + b.Navigation("UserDefinedItem"); + }); + + modelBuilder.Entity("DataLayer.BookContributor", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithMany("ContributorsLink") + .HasForeignKey("BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DataLayer.Contributor", "Contributor") + .WithMany("BooksLink") + .HasForeignKey("ContributorId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Book"); + + b.Navigation("Contributor"); + }); + + modelBuilder.Entity("DataLayer.Category", b => + { + b.HasOne("DataLayer.Category", "ParentCategory") + .WithMany() + .HasForeignKey("ParentCategoryCategoryId"); + + b.Navigation("ParentCategory"); + }); + + modelBuilder.Entity("DataLayer.LibraryBook", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithOne() + .HasForeignKey("DataLayer.LibraryBook", "BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Book"); + }); + + modelBuilder.Entity("DataLayer.SeriesBook", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithMany("SeriesLink") + .HasForeignKey("BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DataLayer.Series", "Series") + .WithMany("BooksLink") + .HasForeignKey("SeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Book"); + + b.Navigation("Series"); + }); + + modelBuilder.Entity("DataLayer.Book", b => + { + b.Navigation("ContributorsLink"); + + b.Navigation("SeriesLink"); + }); + + modelBuilder.Entity("DataLayer.Contributor", b => + { + b.Navigation("BooksLink"); + }); + + modelBuilder.Entity("DataLayer.Series", b => + { + b.Navigation("BooksLink"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.cs b/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.cs new file mode 100644 index 00000000..0630e2ff --- /dev/null +++ b/Source/DataLayer/Migrations/20230308013410_AddAbsentFromLastScan.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DataLayer.Migrations +{ + /// + public partial class AddAbsentFromLastScan : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "AbsentFromLastScan", + table: "LibraryBooks", + type: "INTEGER", + nullable: false, + defaultValue: false); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "AbsentFromLastScan", + table: "LibraryBooks"); + } + } +} diff --git a/Source/DataLayer/Migrations/LibationContextModelSnapshot.cs b/Source/DataLayer/Migrations/LibationContextModelSnapshot.cs index 747513d6..2a0a1869 100644 --- a/Source/DataLayer/Migrations/LibationContextModelSnapshot.cs +++ b/Source/DataLayer/Migrations/LibationContextModelSnapshot.cs @@ -157,6 +157,9 @@ namespace DataLayer.Migrations b.Property("BookId") .HasColumnType("INTEGER"); + b.Property("AbsentFromLastScan") + .HasColumnType("INTEGER"); + b.Property("Account") .HasColumnType("TEXT"); diff --git a/Source/DtoImporterService/ImportItem.cs b/Source/DtoImporterService/ImportItem.cs index 990ee48f..f01908e8 100644 --- a/Source/DtoImporterService/ImportItem.cs +++ b/Source/DtoImporterService/ImportItem.cs @@ -8,5 +8,7 @@ namespace DtoImporterService public Item DtoItem { get; set; } public string AccountId { get; set; } public string LocaleName { get; set; } + public override string ToString() + => DtoItem is null ? base.ToString() : $"[{DtoItem.ProductId}] {DtoItem.Title}"; } } diff --git a/Source/DtoImporterService/LibraryBookImporter.cs b/Source/DtoImporterService/LibraryBookImporter.cs index 38581b7c..77ba8aad 100644 --- a/Source/DtoImporterService/LibraryBookImporter.cs +++ b/Source/DtoImporterService/LibraryBookImporter.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using AudibleUtilities; using DataLayer; +using Dinah.Core; using Dinah.Core.Collections.Generic; namespace DtoImporterService @@ -40,9 +41,8 @@ namespace DtoImporterService // // CURRENT SOLUTION: don't re-insert - var currentLibraryProductIds = DbContext.LibraryBooks.Select(l => l.Book.AudibleProductId).ToList(); var newItems = importItems - .Where(dto => !currentLibraryProductIds.Contains(dto.DtoItem.ProductId)) + .ExceptBy(DbContext.LibraryBooks.Select(lb => lb.Book.AudibleProductId), imp => imp.DtoItem.ProductId) .ToList(); // if 2 accounts try to import the same book in the same transaction: error since we're only tracking and pulling by asin. @@ -55,7 +55,11 @@ namespace DtoImporterService var libraryBook = new LibraryBook( bookImporter.Cache[newItem.DtoItem.ProductId], newItem.DtoItem.DateAdded, - newItem.AccountId); + newItem.AccountId) + { + AbsentFromLastScan = isPlusTitleUnavailable(newItem) + }; + try { DbContext.LibraryBooks.Add(libraryBook); @@ -66,8 +70,30 @@ namespace DtoImporterService } } + //If an existing Book wasn't found in the import, the owning LibraryBook's Book will be null. + foreach (var nullBook in DbContext.LibraryBooks.AsEnumerable().Where(lb => lb.Book is null)) + nullBook.AbsentFromLastScan = true; + + //Join importItems on LibraryBooks before iterating over LibraryBooks to avoid + //quadratic complexity caused by searching all of importItems for each LibraryBook. + //Join uses hashing, so complexity should approach O(N) instead of O(N^2). + var items_lbs + = importItems + .Join(DbContext.LibraryBooks, o => (o.AccountId, o.DtoItem.ProductId), i => (i.Account, i.Book?.AudibleProductId), (o, i) => (o, i)); + + foreach ((ImportItem item, LibraryBook lb) in items_lbs) + lb.AbsentFromLastScan = isPlusTitleUnavailable(item); + var qtyNew = hash.Count; return qtyNew; } + + + //This SEEMS to work to detect plus titles which are no longer available. + //I have my doubts it won't yield false negatives, but I have more + //confidence that it won't yield many/any false positives. + private static bool isPlusTitleUnavailable(ImportItem item) + => item.DtoItem.IsAyce is true + && !item.DtoItem.Plans.Any(p => p.PlanName.ContainsInsensitive("Minerva") || p.PlanName.ContainsInsensitive("Free")); } }