Add migration to try and fix db for incorrect or missing espiode series entries.

Source/AppScaffolding/LibationScaffolding.cs

@@ -5,10 +5,10 @@ using System.Linq;
 using System.Reflection;
 using ApplicationServices;
 using AudibleUtilities;
-using Dinah.Core;
 using Dinah.Core.IO;
 using Dinah.Core.Logging;
 using LibationFileManager;
+using Microsoft.EntityFrameworkCore;
 using Newtonsoft.Json.Linq;
 using Serilog;
 
@@ -405,25 +405,58 @@ namespace AppScaffolding
 
 		public static void migrate_from_7_10_1(Configuration config)
 		{
-			//This migration removes books and series with SERIES_ prefix that were created
+			//https://github.com/rmcrackan/Libation/issues/270#issuecomment-1152863629
-			//as a hack workaround in 7.10.1. Said workaround was removed in 7.10.2
+			//This migration helps fix databases contaminated with the 7.10.1 hack workaround
-
+			//and those with improperly identified or missing series. This does not solve cases
-			var migrated = config.GetNonString<bool>(nameof(migrate_from_7_10_1));
+			//where individual episodes are in the db with a valid series link, but said series'
-
+			//parents have not been imported into the database.  For those cases, Libation will
-			if (migrated) return;
+			//attempt fixup by retrieving parents from the catalog endpoint
 
 			using var context = DbContexts.GetContext();
 
-			var booksToRemove = context.Books.Where(b => b.AudibleProductId.StartsWith("SERIES_")).ToArray();
+			//This migration removes books and series with SERIES_ prefix that were created
-			var seriesToRemove = context.Series.Where(s => s.AudibleSeriesId.StartsWith("SERIES_")).ToArray();
+			//as a hack workaround in 7.10.1. Said workaround was removed in 7.10.2
-			var lbToRemove = context.LibraryBooks.Where(lb => booksToRemove.Any(b => b == lb.Book)).ToArray();
+			string removeHackSeries = "delete " +
+									  "from series " +
+									  "where AudibleSeriesId like 'SERIES%'";
 
-			context.LibraryBooks.RemoveRange(lbToRemove);
+			string removeHackBooks = "delete " +
-			context.Books.RemoveRange(booksToRemove);
+									 "from books " +
-			context.Series.RemoveRange(seriesToRemove);
+									 "where AudibleProductId like 'SERIES%'";
+
+			//Detect series parents that were added to the database as books with ContentType.Episode,
+			//and change them to ContentType.Parent
+			string updateContentType =
+				"UPDATE books " +
+				"SET contenttype = 4 " +
+				"WHERE audibleproductid IN(SELECT books.audibleproductid " +
+											"FROM books " +
+												"INNER JOIN series " +
+													"ON ( books.audibleproductid = " +
+														"series.audibleseriesid) " +
+											"WHERE books.contenttype = 2)";
+
+			//Then detect series parents that were added to the database as books with ContentType.Parent
+			//but are missing a series link, and add the link (don't know how this happened)
+			string addMissingSeriesLink =
+				"INSERT INTO seriesbook " +
+				"SELECT series.seriesid, " +
+						"books.bookid, " +
+						"'- 1' " +
+				"FROM books " +
+					"LEFT OUTER JOIN seriesbook " +
+									"ON books.bookid = seriesbook.bookid " +
+					"INNER JOIN series " +
+								"ON books.audibleproductid = series.audibleseriesid " +
+				"WHERE books.contenttype = 4 " +
+						"AND seriesbook.seriesid IS NULL";
+
+			context.Database.ExecuteSqlRaw(removeHackSeries);
+			context.Database.ExecuteSqlRaw(removeHackBooks);
+			context.Database.ExecuteSqlRaw(updateContentType);
+			context.Database.ExecuteSqlRaw(addMissingSeriesLink);
 
 			LibraryCommands.SaveContext(context);
-			config.SetObject(nameof(migrate_from_7_10_1), true);
 		}
 	}
 }

Source/ApplicationServices/LibraryCommands.cs

@@ -126,6 +126,22 @@ namespace ApplicationServices
 				if (totalCount == 0)
 					return default;
 
+
+				Log.Logger.Information("Begin scan for orphaned episode parents");
+				var newParents = await findAndAddMissingParents(apiExtendedfunc, accounts);
+				Log.Logger.Information($"Orphan episode scan complete. New parents count {newParents}");
+
+				if (newParents >= 0)
+				{
+					//If any episodes are still orphaned, their series have been
+					//removed from the catalog and wel'll never be able to find them.
+
+					//only do this if findAndAddMissingParents returned >= 0. If it
+					//returned < 0, an error happened and there's still a chance that
+					//a future successful run will find missing parents.
+					removedOrphanedEpisodes();
+				}
+
 				Log.Logger.Information("Begin long-running import");
 				logTime($"pre {nameof(importIntoDbAsync)}");
 				var newCount = await importIntoDbAsync(importItems);
@@ -207,7 +223,7 @@ namespace ApplicationServices
             using var context = DbContexts.GetContext();
             var libraryBookImporter = new LibraryBookImporter(context);
             var newCount = await Task.Run(() => libraryBookImporter.Import(importItems));
-            logTime("importIntoDbAsync -- post Import()");
+			logTime("importIntoDbAsync -- post Import()");
             int qtyChanges = SaveContext(context);
             logTime("importIntoDbAsync -- post SaveChanges");
 
@@ -219,7 +235,84 @@ namespace ApplicationServices
             return newCount;
         }
 
-        public static int SaveContext(LibationContext context)
+		static void removedOrphanedEpisodes()
+		{
+			using var context = DbContexts.GetContext();
+			try
+			{
+				var orphanedEpisodes =
+					context
+					.GetLibrary_Flat_NoTracking(includeParents: true)
+					.FindOrphanedEpisodes();
+
+				context.LibraryBooks.RemoveRange(orphanedEpisodes);
+				context.Books.RemoveRange(orphanedEpisodes.Select(lb => lb.Book));
+
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "An error occured while trying to remove orphaned episodes from the database");
+			}
+		}
+
+		static async Task<int> findAndAddMissingParents(Func<Account, Task<ApiExtended>> apiExtendedfunc, Account[] accounts)
+		{
+			using var context = DbContexts.GetContext();
+
+			var library = context.GetLibrary_Flat_NoTracking(includeParents: true);
+
+			try
+			{
+				var orphanedEpisodes = library.FindOrphanedEpisodes().ToList();
+
+				if (!orphanedEpisodes.Any())
+					return -1;
+
+				var orphanedSeries =
+					orphanedEpisodes
+					.SelectMany(lb => lb.Book.SeriesLink)
+					.DistinctBy(s => s.Series.AudibleSeriesId)
+					.ToList();
+
+				// We're only calling the Catalog endpoint, so it doesn't matter which account we use.
+				var apiExtended = await apiExtendedfunc(accounts[0]);
+
+				var seriesParents = orphanedSeries.Select(o => o.Series.AudibleSeriesId).ToList();
+				var items = await apiExtended.Api.GetCatalogProductsAsync(seriesParents, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS);
+
+				List<ImportItem> newParentsImportItems = new();
+				foreach (var sp in orphanedSeries)
+				{
+					var serie = items.First(i => i.Asin == sp.Series.AudibleSeriesId);
+					var lb = orphanedEpisodes.First(l => l.Book.AudibleProductId == sp.Book.AudibleProductId);
+
+					if (serie.Relationships is null)
+						continue;
+
+					serie.PurchaseDate = new DateTimeOffset(lb.DateAdded);
+					serie.Series = new AudibleApi.Common.Series[]
+					{
+					new AudibleApi.Common.Series{ Asin = serie.Asin, Title = serie.TitleWithSubtitle, Sequence = "-1"}
+					};
+
+					newParentsImportItems.Add(new ImportItem { DtoItem = serie, AccountId = lb.Account, LocaleName = lb.Book.Locale });
+				}
+
+				var newCoutn = new LibraryBookImporter(context)
+					.Import(newParentsImportItems);
+
+				await context.SaveChangesAsync();
+
+				return newCoutn;
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Logger.Error(ex, "An error occured while trying to scan for orphaned episode parents.");
+				return -1;
+			}
+		}
+
+		public static int SaveContext(LibationContext context)
 		{
 			try
 			{

Source/DataLayer/QueryObjects/BookQueries.cs

@@ -44,6 +44,8 @@ namespace DataLayer
 
         public static bool IsEpisodeParent(this Book book)
             => book.ContentType is ContentType.Parent;
-
+        public static bool HasLiberated(this Book book)
+            => book.UserDefinedItem.BookStatus is LiberatedStatus.Liberated ||
+            book.UserDefinedItem.PdfStatus is not null and LiberatedStatus.Liberated;
     }
 }

Source/DataLayer/QueryObjects/LibraryBookQueries.cs

@@ -43,18 +43,41 @@ namespace DataLayer
                 .Include(le => le.Book).ThenInclude(b => b.ContributorsLink).ThenInclude(c => c.Contributor)
                 .Include(le => le.Book).ThenInclude(b => b.Category).ThenInclude(c => c.ParentCategory);
 
+        public static IEnumerable<LibraryBook> FindOrphanedEpisodes(this IEnumerable<LibraryBook> libraryBooks)
+		{
+            var parentedEpisodes = 
+                libraryBooks
+                .Where(lb => lb.Book.IsEpisodeParent())
+                .SelectMany(s => libraryBooks.FindChildren(s));
+
+            return
+                libraryBooks
+                .Where(lb => lb.Book.IsEpisodeChild())
+                .ExceptBy(
+                    parentedEpisodes
+                    .Select(ge => ge.Book.AudibleProductId), ge => ge.Book.AudibleProductId);
+        }
+
 #nullable enable
         public static LibraryBook? FindSeriesParent(this IEnumerable<LibraryBook> libraryBooks, LibraryBook seriesEpisode)
         {
             if (seriesEpisode.Book.SeriesLink is null) return null;
 
-            //Parent books will always have exactly 1 SeriesBook due to how
+            try
-            //they are imported in ApiExtended.getChildEpisodesAsync()
+            {
-            return libraryBooks.FirstOrDefault(
+                //Parent books will always have exactly 1 SeriesBook due to how
-                lb =>
+                //they are imported in ApiExtended.getChildEpisodesAsync()
-                lb.Book.IsEpisodeParent() &&
+                return libraryBooks.FirstOrDefault(
-                seriesEpisode.Book.SeriesLink.Any(
+                    lb =>
-                    s => s.Series.AudibleSeriesId == lb.Book.SeriesLink.Single().Series.AudibleSeriesId));
+                    lb.Book.IsEpisodeParent() &&
+                    seriesEpisode.Book.SeriesLink.Any(
+                        s => s.Series.AudibleSeriesId == lb.Book.SeriesLink.Single().Series.AudibleSeriesId));
+            }
+            catch (System.Exception ex)
+            {
+                Serilog.Log.Error(ex, "Query error in {0}", nameof(FindSeriesParent));
+                return null;
+            }
         }
 #nullable disable
 

Source/LibationWinForms/GridView/ProductsGrid.cs

@@ -114,11 +114,17 @@ namespace LibationWinForms.GridView
 
 		internal void BindToGrid(List<LibraryBook> dbBooks)
 		{
-			var geList = dbBooks.Where(lb => lb.Book.IsProduct()).Select(b => new LibraryBookEntry(b)).Cast<GridEntry>().ToList();
+			var geList = dbBooks
+				.Where(lb => lb.Book.IsProduct())
+				.Select(b => new LibraryBookEntry(b))
+				.Cast<GridEntry>()
+				.ToList();
 
 			var episodes = dbBooks.Where(lb => lb.Book.IsEpisodeChild());
-						
+
-			foreach (var parent in dbBooks.Where(lb => lb.Book.IsEpisodeParent()))
+			var seriesBooks = dbBooks.Where(lb => lb.Book.IsEpisodeParent()).ToList();
+
+			foreach (var parent in seriesBooks)
 			{
 				var seriesEpisodes = episodes.FindChildren(parent);
 
@@ -216,6 +222,7 @@ namespace LibationWinForms.GridView
 			if (existingEpisodeEntry is null)
 			{
 				LibraryBookEntry episodeEntry;
+
 				var seriesEntry = seriesEntries.FindSeriesParent(episodeBook);
 
 				if (seriesEntry is null)
@@ -225,13 +232,14 @@ namespace LibationWinForms.GridView
 
 					if (seriesBook is null)
 					{
-						//This should be impossible because the importer ensures every episode has a parent.
+						//This is only possible if the user's db  has some malformed
-						var ex = new ApplicationException($"Episode's series parent not found in database.");
+						//entries from earlier Libation releases that could not be
-						var seriesLinks = string.Join("\r\n", episodeBook.Book.SeriesLink?.Select(sb => $"{nameof(sb.Series.Name)}={sb.Series.Name}, {nameof(sb.Series.AudibleSeriesId)}={sb.Series.AudibleSeriesId}"));
+						//automatically fixed. Log, but don't throw.
-						Serilog.Log.Logger.Error(ex, "Episode={episodeBook}, Series: {seriesLinks}", episodeBook, seriesLinks);
+						Serilog.Log.Logger.Error("Episode={0}, Episode Series: {1}", episodeBook, episodeBook.Book.SeriesNames());
-						throw ex;
+						return;
 					}
 
+
 					seriesEntry = new SeriesEntry(seriesBook, episodeBook);
 					seriesEntries.Add(seriesEntry);
 

Source/LibationWinForms/GridView/QueryExtensions.cs

@@ -24,12 +24,20 @@ namespace LibationWinForms.GridView
 		{
 			if (seriesEpisode.Book.SeriesLink is null) return null;
 
-			//Parent books will always have exactly 1 SeriesBook due to how
+			try
-			//they are imported in ApiExtended.getChildEpisodesAsync()
+			{
-			return gridEntries.SeriesEntries().FirstOrDefault(
+				//Parent books will always have exactly 1 SeriesBook due to how
-				lb =>
+				//they are imported in ApiExtended.getChildEpisodesAsync()
-				seriesEpisode.Book.SeriesLink.Any(
+				return gridEntries.SeriesEntries().FirstOrDefault(
-					s => s.Series.AudibleSeriesId == lb.LibraryBook.Book.SeriesLink.Single().Series.AudibleSeriesId));
+					lb =>
+					seriesEpisode.Book.SeriesLink.Any(
+						s => s.Series.AudibleSeriesId == lb.LibraryBook.Book.SeriesLink.Single().Series.AudibleSeriesId));
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Error(ex, "Query error in {0}", nameof(FindSeriesParent));
+				return null;
+			}
 		}
 	}
 #nullable disable