Improved Category Ladders

Source/DataLayer/Configurations/BookConfig.cs

@@ -21,10 +21,7 @@ namespace DataLayer.Configurations
             entity.Ignore(nameof(Book.Narrators));
             entity.Ignore(nameof(Book.AudioFormat));
             entity.Ignore(nameof(Book.TitleWithSubtitle));
-            //// these don't seem to matter
+            entity.Ignore(b => b.Categories);
-            //entity.Ignore(nameof(Book.AuthorNames));
-            //entity.Ignore(nameof(Book.NarratorNames));
-            //entity.Ignore(nameof(Book.HasPdfs));
 
             // OwnsMany: "Can only ever appear on navigation properties of other entity types.
             //  Are automatically loaded, and can only be tracked by a DbContext alongside their owner."
@@ -58,24 +55,6 @@ namespace DataLayer.Configurations
                     // owns it 1:1, store in same table
                     b_udi.OwnsOne(udi => udi.Rating);
                 });
-
-            entity
-                .Metadata
-                .FindNavigation(nameof(Book.ContributorsLink))
-                // PropertyAccessMode.Field : Contributions is a get-only property, not a field, so use its backing field
-                .SetPropertyAccessMode(PropertyAccessMode.Field);
-
-            entity
-                .Metadata
-                .FindNavigation(nameof(Book.SeriesLink))
-                // PropertyAccessMode.Field : Series is a get-only property, not a field, so use its backing field
-                .SetPropertyAccessMode(PropertyAccessMode.Field);
-
-            entity
-                .Metadata
-                .FindNavigation(nameof(Book.CategoriesLink))
-				// PropertyAccessMode.Field : Categories is a get-only property, not a field, so use its backing field
-				.SetPropertyAccessMode(PropertyAccessMode.Field);
         }
     }
 }

Source/DataLayer/EfClasses/Book.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.Linq;
 using Dinah.Core;
 using Microsoft.EntityFrameworkCore;
@@ -90,8 +91,8 @@ namespace DataLayer
 
             // non-ef-ctor init.s
             UserDefinedItem = new UserDefinedItem(this);
-            _contributorsLink = new HashSet<BookContributor>();
+            ContributorsLink = new HashSet<BookContributor>();
-            _categoriesLink = new HashSet<BookCategory>();
+            CategoriesLink = new HashSet<BookCategory>();
             _seriesLink = new HashSet<SeriesBook>();
             _supplements = new HashSet<Supplement>();
 
@@ -113,19 +114,11 @@ namespace DataLayer
         }
 
         #region contributors, authors, narrators
-		// use uninitialised backing fields - this means we can detect if the collection was loaded
+        internal HashSet<BookContributor> ContributorsLink { get; private set; }
-		private HashSet<BookContributor> _contributorsLink;
-        // i'd like this to be internal but migration throws this exception when i try:
-        //   Value cannot be null.
-        //   Parameter name: property
-        public IEnumerable<BookContributor> ContributorsLink
-            => _contributorsLink?
-                .OrderBy(bc => bc.Order)
-                .ToList();
 
-        public IEnumerable<Contributor> Authors => getContributions(Role.Author).Select(bc => bc.Contributor).ToList();
+        public IEnumerable<Contributor> Authors => ContributorsLink.ByRole(Role.Author).Select(bc => bc.Contributor).ToList();
-        public IEnumerable<Contributor> Narrators => getContributions(Role.Narrator).Select(bc => bc.Contributor).ToList();
+        public IEnumerable<Contributor> Narrators => ContributorsLink.ByRole(Role.Narrator).Select(bc => bc.Contributor).ToList();
-        public string Publisher => getContributions(Role.Publisher).SingleOrDefault()?.Contributor.Name;
+        public string Publisher => ContributorsLink.ByRole(Role.Publisher).SingleOrDefault()?.Contributor.Name;
 
         public void ReplaceAuthors(IEnumerable<Contributor> authors, DbContext context = null)
             => replaceContributors(authors, Role.Author, context);
@@ -138,15 +131,19 @@ namespace DataLayer
             ArgumentValidator.EnsureEnumerableNotNullOrEmpty(newContributors, nameof(newContributors));
 
             // the edge cases of doing local-loaded vs remote-only got weird. just load it
-            if (_contributorsLink is null)
+            if (ContributorsLink is null)
                 getEntry(context).Collection(s => s.ContributorsLink).Load();
 
-			var roleContributions = getContributions(role);
+            var isIdentical
-            var isIdentical = roleContributions.Select(c => c.Contributor).SequenceEqual(newContributors);
+                = ContributorsLink
+                .ByRole(role)
+                .Select(c => c.Contributor)
+                .SequenceEqual(newContributors);
+
             if (isIdentical)
                 return;
 
-            _contributorsLink.RemoveWhere(bc => bc.Role == role);
+            ContributorsLink.RemoveWhere(bc => bc.Role == role);
             addNewContributors(newContributors, role);
         }
 
@@ -155,14 +152,9 @@ namespace DataLayer
             byte order = 0;
             var newContributionsEnum = newContributors.Select(c => new BookContributor(this, c, role, order++));
             var newContributions = new HashSet<BookContributor>(newContributionsEnum);
-            _contributorsLink.UnionWith(newContributions);
+            ContributorsLink.UnionWith(newContributions);
         }
 
-        private List<BookContributor> getContributions(Role role)
-            => ContributorsLink
-                .Where(a => a.Role == role)
-                .OrderBy(a => a.Order)
-                .ToList();
         #endregion
 
         private Microsoft.EntityFrameworkCore.ChangeTracking.EntityEntry<Book> getEntry(DbContext context)
@@ -176,25 +168,28 @@ namespace DataLayer
 
             return entry;
         }
+
         #region categories
-		private HashSet<BookCategory> _categoriesLink;
+        internal HashSet<BookCategory> CategoriesLink { get; private set; }
-		public IEnumerable<BookCategory> CategoriesLink => _categoriesLink?.ToList();
-		public void UpsertCategories(CategoryLadder ladder)
-		{
-			ArgumentValidator.EnsureNotNull(ladder, nameof(ladder));
 
-			var singleBookCategory = _categoriesLink.SingleOrDefault(bc => bc.CategoryLadder.Equals(ladder));
+        private ReadOnlyCollection<BookCategory> _categoriesReadOnly;
-
+        public ReadOnlyCollection<BookCategory> Categories
-			if (singleBookCategory is null)
-				_categoriesLink.Add(new BookCategory(this, ladder));
-            else
         {
-                for (var i = 0; i < ladder._categories.Count; i++)
+            get
             {
-                    //Update the category name
+                if (_categoriesReadOnly?.SequenceEqual(CategoriesLink) is not true)
-					singleBookCategory.CategoryLadder._categories[i].Name = ladder._categories[i].Name;
+                    _categoriesReadOnly = CategoriesLink.ToList().AsReadOnly();
+                return _categoriesReadOnly;
             }
         }
+		public void SetCategoryLadders(IEnumerable<CategoryLadder> ladders)
+		{
+			ArgumentValidator.EnsureNotNull(ladders, nameof(ladders));
+
+			//Replace all existing category ladders.
+            //Some books make have duplocate ladders
+			CategoriesLink.Clear();
+            CategoriesLink.UnionWith(ladders.Distinct().Select(l => new BookCategory(this, l)));
 		}
 		#endregion
 

Source/DataLayer/EfClasses/Category.cs

@@ -1,6 +1,9 @@
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.Linq;
 using Dinah.Core;
 
+#nullable enable
 namespace DataLayer
 {
     public class AudibleCategoryId
@@ -16,12 +19,21 @@ namespace DataLayer
     public class Category
     {
         internal int CategoryId { get; private set; }
-        public string AudibleCategoryId { get; private set; }
+        public string? AudibleCategoryId { get; private set; }
 
-        public string Name { get; internal set; }
+        public string? Name { get; internal set; }
 
         internal List<CategoryLadder> _categoryLadders = new();
-        public IReadOnlyCollection<CategoryLadder> CategoryLadders => _categoryLadders.AsReadOnly();
+        private ReadOnlyCollection<CategoryLadder>? _categoryLaddersReadOnly;
+        public ReadOnlyCollection<CategoryLadder> CategoryLadders
+		{
+			get
+			{
+				if (_categoryLaddersReadOnly?.SequenceEqual(_categoryLadders) is not true)
+					_categoryLaddersReadOnly = _categoryLadders.AsReadOnly();
+				return _categoryLaddersReadOnly;
+			}
+		}
 
 		private Category() { }
         /// <summary>special id class b/c it's too easy to get string order mixed up</summary>

Source/DataLayer/EfClasses/CategoryLadder.cs

@@ -4,6 +4,7 @@ using System.Collections.Generic;
 using System.Collections.ObjectModel;
 using System.Linq;
 
+#nullable enable
 namespace DataLayer
 {
 	public class CategoryLadder : IEquatable<CategoryLadder>
@@ -11,10 +12,19 @@ namespace DataLayer
 		internal int CategoryLadderId { get; private set; }
 
 		internal List<Category> _categories;
-		public ReadOnlyCollection<Category> Categories => _categories.AsReadOnly();
+		private ReadOnlyCollection<Category>? _categoriesReadOnly;
+		public ReadOnlyCollection<Category> Categories
+		{
+			get
+			{
+				if (_categoriesReadOnly?.SequenceEqual(_categories) is not true)
+					_categoriesReadOnly = _categories.AsReadOnly();
+				return _categoriesReadOnly;
+			}
+		}
 
-		private HashSet<BookCategory> _booksLink;
+		private HashSet<BookCategory>? _booksLink;
-		public IEnumerable<BookCategory> BooksLink => _booksLink?.ToList();
+		public IEnumerable<BookCategory>? BooksLink => _booksLink?.ToList();
 		private CategoryLadder() { _categories = new(); }
 		public CategoryLadder(List<Category> categories)
 		{
@@ -32,16 +42,16 @@ namespace DataLayer
 			return hashCode.ToHashCode();
 		}
 
-		public bool Equals(CategoryLadder other)
+		public bool Equals(CategoryLadder? other)
-		{
+			=> other?._categories is not null
-			if (other?._categories is null)
+			&& Equals(other._categories.Select(c => c.AudibleCategoryId));
-				return false;
 
-			return Equals(other._categories.Select(c => c.AudibleCategoryId));
+		public bool Equals(IEnumerable<string?>? categoryIds)
-		}
+			=> categoryIds is not null
-		public bool Equals(IEnumerable<string> categoryIds)
+			&& _categories.Select(c => c.AudibleCategoryId).SequenceEqual(categoryIds);
-			=> _categories.Select(c => c.AudibleCategoryId).SequenceEqual(categoryIds);
+
-		public override bool Equals(object obj) => obj is CategoryLadder other && Equals(other);
+		public override bool Equals(object? obj)
+			=> obj is CategoryLadder other && Equals(other);
 
 		public override string ToString() => string.Join(" > ", _categories.Select(c => c.Name));
 	}

Source/DataLayer/EntityExtensions.cs

@@ -8,6 +8,11 @@ namespace DataLayer
 {
     public static class EntityExtensions
 	{
+		public static IEnumerable<BookContributor> ByRole(this IEnumerable<BookContributor> contributors, Role role)
+			=> contributors
+				.Where(a => a.Role == role)
+				.OrderBy(a => a.Order);
+
 		public static string TitleSortable(this Book book) => Formatters.GetSortName(book.Title + book.Subtitle);
 
         public static string AuthorNames(this Book book) => string.Join(", ", book.Authors.Select(a => a.Name));

Source/DataLayer/Migrations/20230718214617_AddCategoryLadder.Designer.cs

@@ -11,8 +11,8 @@ using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
 namespace DataLayer.Migrations
 {
     [DbContext(typeof(LibationContext))]
-    [Migration("20230717220642_AddCategoriesList")]
+    [Migration("20230718214617_AddCategoryLadder")]
-    partial class AddCategoriesList
+    partial class AddCategoryLadder
     {
         /// <inheritdoc />
         protected override void BuildTargetModel(ModelBuilder modelBuilder)
@@ -144,14 +144,6 @@ namespace DataLayer.Migrations
                     b.HasIndex("AudibleCategoryId");
 
                     b.ToTable("Categories");
-
-                    b.HasData(
-                        new
-                        {
-                            CategoryId = -1,
-                            AudibleCategoryId = "",
-                            Name = ""
-                        });
                 });
 
             modelBuilder.Entity("DataLayer.CategoryLadder", b =>
@@ -163,12 +155,6 @@ namespace DataLayer.Migrations
                     b.HasKey("CategoryLadderId");
 
                     b.ToTable("CategoryLadders");
-
-                    b.HasData(
-                        new
-                        {
-                            CategoryLadderId = -1
-                        });
                 });
 
             modelBuilder.Entity("DataLayer.Contributor", b =>

Source/DataLayer/Migrations/20230718214617_AddCategoryLadder.cs

@@ -5,7 +5,7 @@
 namespace DataLayer.Migrations
 {
     /// <inheritdoc />
-    public partial class AddCategoriesList : Migration
+    public partial class AddCategoryLadder : Migration
     {
         /// <inheritdoc />
         protected override void Up(MigrationBuilder migrationBuilder)
@@ -26,6 +26,11 @@ namespace DataLayer.Migrations
                 name: "IX_Books_CategoryId",
                 table: "Books");
 
+            migrationBuilder.DeleteData(
+                table: "Categories",
+                keyColumn: "CategoryId",
+                keyValue: -1);
+
             migrationBuilder.DropColumn(
                 name: "ParentCategoryCategoryId",
                 table: "Categories");
@@ -94,11 +99,6 @@ namespace DataLayer.Migrations
                         onDelete: ReferentialAction.Cascade);
                 });
 
-            migrationBuilder.InsertData(
-                table: "CategoryLadders",
-                column: "CategoryLadderId",
-                value: -1);
-
             migrationBuilder.CreateIndex(
                 name: "IX_BookCategory_BookId",
                 table: "BookCategory",
@@ -140,12 +140,10 @@ namespace DataLayer.Migrations
                 nullable: false,
                 defaultValue: 0);
 
-            migrationBuilder.UpdateData(
+            migrationBuilder.InsertData(
                 table: "Categories",
-                keyColumn: "CategoryId",
+                columns: new[] { "CategoryId", "AudibleCategoryId", "Name", "ParentCategoryCategoryId" },
-                keyValue: -1,
+                values: new object[] { -1, "", "", null });
-                column: "ParentCategoryCategoryId",
-                value: null);
 
             migrationBuilder.CreateIndex(
                 name: "IX_Categories_ParentCategoryCategoryId",

Source/DataLayer/Migrations/LibationContextModelSnapshot.cs

@@ -141,14 +141,6 @@ namespace DataLayer.Migrations
                     b.HasIndex("AudibleCategoryId");
 
                     b.ToTable("Categories");
-
-                    b.HasData(
-                        new
-                        {
-                            CategoryId = -1,
-                            AudibleCategoryId = "",
-                            Name = ""
-                        });
                 });
 
             modelBuilder.Entity("DataLayer.CategoryLadder", b =>
@@ -160,12 +152,6 @@ namespace DataLayer.Migrations
                     b.HasKey("CategoryLadderId");
 
                     b.ToTable("CategoryLadders");
-
-                    b.HasData(
-                        new
-                        {
-                            CategoryLadderId = -1
-                        });
                 });
 
             modelBuilder.Entity("DataLayer.Contributor", b =>

Source/DataLayer/QueryObjects/CategoryQueries.cs

@@ -0,0 +1,11 @@
+using Microsoft.EntityFrameworkCore;
+using System.Linq;
+
+namespace DataLayer
+{
+	public static class CategoryQueries
+	{
+		public static IQueryable<CategoryLadder> GetCategoryLadders(this LibationContext context)
+			=> context.CategoryLadders.Include(c => c._categories);
+	}
+}

Source/DtoImporterService/BookImporter.cs

@@ -188,13 +188,15 @@ namespace DtoImporterService
 
 			if (item.CategoryLadders is not null)
 			{
+				var ladders = new List<DataLayer.CategoryLadder>();
 				foreach (var ladder in item.CategoryLadders.Select(cl => cl.Ladder).Where(l => l?.Length > 0))
 				{
 					var categoryIds = ladder.Select(l => l.CategoryId).ToList();
-					var cata = categoryImporter.LadderCache.Single(c => c.Equals(categoryIds));
+					ladders.Add(categoryImporter.LadderCache.Single(c => c.Equals(categoryIds)));
-
-					book.UpsertCategories(cata);
 				}
+				//Set all ladders at once so ladders that have been
+				//removed by audible can be removed from the DB
+				book.SetCategoryLadders(ladders);
 			}
 		}
 

Source/DtoImporterService/CategoryImporter.cs

@@ -12,23 +12,15 @@ namespace DtoImporterService
 	{
 		protected override IValidator Validator => new CategoryValidator();
 
-		private Dictionary<string, Category> Cache { get; set; } = new();
+		private Dictionary<string, Category> CategoryCache { get; set; } = new();
 		public HashSet<DataLayer.CategoryLadder> LadderCache { get; private set; } = new();
 
 		public CategoryImporter(LibationContext context) : base(context) { }
 
 		protected override int DoImport(IEnumerable<ImportItem> importItems)
 		{
-			// get distinct
-			var categoryIds = importItems
-				.Select(i => i.DtoItem)
-				.GetCategoriesDistinct()
-				.Select(c => c.CategoryId)
-				.Distinct()
-				.ToList();
-
 			// load db existing => .Local
-			loadLocal_categories(categoryIds);
+			loadLocal_categories();
 
 			// upsert
 			var categoryLadders = importItems
@@ -41,14 +33,11 @@ namespace DtoImporterService
 			return qtyNew;
 		}
 
-		private void loadLocal_categories(List<string> categoryIds)
+		private void loadLocal_categories()
 		{
 			// load existing => local
-			Cache = DbContext.Categories
+			LadderCache = DbContext.GetCategoryLadders().ToHashSet();
-				.Where(c => categoryIds.Contains(c.AudibleCategoryId))
+			CategoryCache = LadderCache.SelectMany(cl => cl.Categories).ToDictionarySafe(c => c.AudibleCategoryId);
-				.ToDictionarySafe(c => c.AudibleCategoryId);
-
-			LadderCache = DbContext.CategoryLadders.ToHashSet();
 		}
 
 		// only use after loading contributors => local
@@ -65,10 +54,9 @@ namespace DtoImporterService
 					var id = ladder[i].CategoryId;
 					var name = ladder[i].CategoryName;
 
-					if (!Cache.TryGetValue(id, out var category))
+					if (!CategoryCache.TryGetValue(id, out var category))
 					{
 						category = addCategory(id, name);
-						qtyNew++;
 					}
 
 					categories.Add(category);
@@ -111,7 +99,7 @@ namespace DtoImporterService
 				var entityEntry = DbContext.Categories.Add(category);
 				var entity = entityEntry.Entity;
 
-				Cache.Add(entity.AudibleCategoryId, entity);
+				CategoryCache.Add(entity.AudibleCategoryId, entity);
 				return entity;
 			}
 			catch (Exception ex)