diff --git a/Source/AppScaffolding/LibationScaffolding.cs b/Source/AppScaffolding/LibationScaffolding.cs index e16f52f0..91f0f963 100644 --- a/Source/AppScaffolding/LibationScaffolding.cs +++ b/Source/AppScaffolding/LibationScaffolding.cs @@ -5,10 +5,10 @@ using System.Linq; using System.Reflection; using ApplicationServices; using AudibleUtilities; -using Dinah.Core; using Dinah.Core.IO; using Dinah.Core.Logging; using LibationFileManager; +using Microsoft.EntityFrameworkCore; using Newtonsoft.Json.Linq; using Serilog; @@ -405,25 +405,58 @@ namespace AppScaffolding public static void migrate_from_7_10_1(Configuration config) { - //This migration removes books and series with SERIES_ prefix that were created - //as a hack workaround in 7.10.1. Said workaround was removed in 7.10.2 - - var migrated = config.GetNonString(nameof(migrate_from_7_10_1)); - - if (migrated) return; + //https://github.com/rmcrackan/Libation/issues/270#issuecomment-1152863629 + //This migration helps fix databases contaminated with the 7.10.1 hack workaround + //and those with improperly identified or missing series. This does not solve cases + //where individual episodes are in the db with a valid series link, but said series' + //parents have not been imported into the database. For those cases, Libation will + //attempt fixup by retrieving parents from the catalog endpoint using var context = DbContexts.GetContext(); - var booksToRemove = context.Books.Where(b => b.AudibleProductId.StartsWith("SERIES_")).ToArray(); - var seriesToRemove = context.Series.Where(s => s.AudibleSeriesId.StartsWith("SERIES_")).ToArray(); - var lbToRemove = context.LibraryBooks.Where(lb => booksToRemove.Any(b => b == lb.Book)).ToArray(); + //This migration removes books and series with SERIES_ prefix that were created + //as a hack workaround in 7.10.1. Said workaround was removed in 7.10.2 + string removeHackSeries = "delete " + + "from series " + + "where AudibleSeriesId like 'SERIES%'"; - context.LibraryBooks.RemoveRange(lbToRemove); - context.Books.RemoveRange(booksToRemove); - context.Series.RemoveRange(seriesToRemove); + string removeHackBooks = "delete " + + "from books " + + "where AudibleProductId like 'SERIES%'"; + + //Detect series parents that were added to the database as books with ContentType.Episode, + //and change them to ContentType.Parent + string updateContentType = + "UPDATE books " + + "SET contenttype = 4 " + + "WHERE audibleproductid IN(SELECT books.audibleproductid " + + "FROM books " + + "INNER JOIN series " + + "ON ( books.audibleproductid = " + + "series.audibleseriesid) " + + "WHERE books.contenttype = 2)"; + + //Then detect series parents that were added to the database as books with ContentType.Parent + //but are missing a series link, and add the link (don't know how this happened) + string addMissingSeriesLink = + "INSERT INTO seriesbook " + + "SELECT series.seriesid, " + + "books.bookid, " + + "'- 1' " + + "FROM books " + + "LEFT OUTER JOIN seriesbook " + + "ON books.bookid = seriesbook.bookid " + + "INNER JOIN series " + + "ON books.audibleproductid = series.audibleseriesid " + + "WHERE books.contenttype = 4 " + + "AND seriesbook.seriesid IS NULL"; + + context.Database.ExecuteSqlRaw(removeHackSeries); + context.Database.ExecuteSqlRaw(removeHackBooks); + context.Database.ExecuteSqlRaw(updateContentType); + context.Database.ExecuteSqlRaw(addMissingSeriesLink); LibraryCommands.SaveContext(context); - config.SetObject(nameof(migrate_from_7_10_1), true); } } } diff --git a/Source/ApplicationServices/LibraryCommands.cs b/Source/ApplicationServices/LibraryCommands.cs index 42f751e3..cd6165b5 100644 --- a/Source/ApplicationServices/LibraryCommands.cs +++ b/Source/ApplicationServices/LibraryCommands.cs @@ -126,6 +126,22 @@ namespace ApplicationServices if (totalCount == 0) return default; + + Log.Logger.Information("Begin scan for orphaned episode parents"); + var newParents = await findAndAddMissingParents(apiExtendedfunc, accounts); + Log.Logger.Information($"Orphan episode scan complete. New parents count {newParents}"); + + if (newParents >= 0) + { + //If any episodes are still orphaned, their series have been + //removed from the catalog and wel'll never be able to find them. + + //only do this if findAndAddMissingParents returned >= 0. If it + //returned < 0, an error happened and there's still a chance that + //a future successful run will find missing parents. + removedOrphanedEpisodes(); + } + Log.Logger.Information("Begin long-running import"); logTime($"pre {nameof(importIntoDbAsync)}"); var newCount = await importIntoDbAsync(importItems); @@ -207,7 +223,7 @@ namespace ApplicationServices using var context = DbContexts.GetContext(); var libraryBookImporter = new LibraryBookImporter(context); var newCount = await Task.Run(() => libraryBookImporter.Import(importItems)); - logTime("importIntoDbAsync -- post Import()"); + logTime("importIntoDbAsync -- post Import()"); int qtyChanges = SaveContext(context); logTime("importIntoDbAsync -- post SaveChanges"); @@ -219,7 +235,84 @@ namespace ApplicationServices return newCount; } - public static int SaveContext(LibationContext context) + static void removedOrphanedEpisodes() + { + using var context = DbContexts.GetContext(); + try + { + var orphanedEpisodes = + context + .GetLibrary_Flat_NoTracking(includeParents: true) + .FindOrphanedEpisodes(); + + context.LibraryBooks.RemoveRange(orphanedEpisodes); + context.Books.RemoveRange(orphanedEpisodes.Select(lb => lb.Book)); + + } + catch (Exception ex) + { + Serilog.Log.Logger.Error(ex, "An error occured while trying to remove orphaned episodes from the database"); + } + } + + static async Task findAndAddMissingParents(Func> apiExtendedfunc, Account[] accounts) + { + using var context = DbContexts.GetContext(); + + var library = context.GetLibrary_Flat_NoTracking(includeParents: true); + + try + { + var orphanedEpisodes = library.FindOrphanedEpisodes().ToList(); + + if (!orphanedEpisodes.Any()) + return -1; + + var orphanedSeries = + orphanedEpisodes + .SelectMany(lb => lb.Book.SeriesLink) + .DistinctBy(s => s.Series.AudibleSeriesId) + .ToList(); + + // We're only calling the Catalog endpoint, so it doesn't matter which account we use. + var apiExtended = await apiExtendedfunc(accounts[0]); + + var seriesParents = orphanedSeries.Select(o => o.Series.AudibleSeriesId).ToList(); + var items = await apiExtended.Api.GetCatalogProductsAsync(seriesParents, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS); + + List newParentsImportItems = new(); + foreach (var sp in orphanedSeries) + { + var serie = items.First(i => i.Asin == sp.Series.AudibleSeriesId); + var lb = orphanedEpisodes.First(l => l.Book.AudibleProductId == sp.Book.AudibleProductId); + + if (serie.Relationships is null) + continue; + + serie.PurchaseDate = new DateTimeOffset(lb.DateAdded); + serie.Series = new AudibleApi.Common.Series[] + { + new AudibleApi.Common.Series{ Asin = serie.Asin, Title = serie.TitleWithSubtitle, Sequence = "-1"} + }; + + newParentsImportItems.Add(new ImportItem { DtoItem = serie, AccountId = lb.Account, LocaleName = lb.Book.Locale }); + } + + var newCoutn = new LibraryBookImporter(context) + .Import(newParentsImportItems); + + await context.SaveChangesAsync(); + + return newCoutn; + } + catch (Exception ex) + { + Serilog.Log.Logger.Error(ex, "An error occured while trying to scan for orphaned episode parents."); + return -1; + } + } + + public static int SaveContext(LibationContext context) { try { diff --git a/Source/DataLayer/QueryObjects/BookQueries.cs b/Source/DataLayer/QueryObjects/BookQueries.cs index efcbdc36..85f9153b 100644 --- a/Source/DataLayer/QueryObjects/BookQueries.cs +++ b/Source/DataLayer/QueryObjects/BookQueries.cs @@ -44,6 +44,8 @@ namespace DataLayer public static bool IsEpisodeParent(this Book book) => book.ContentType is ContentType.Parent; - + public static bool HasLiberated(this Book book) + => book.UserDefinedItem.BookStatus is LiberatedStatus.Liberated || + book.UserDefinedItem.PdfStatus is not null and LiberatedStatus.Liberated; } } diff --git a/Source/DataLayer/QueryObjects/LibraryBookQueries.cs b/Source/DataLayer/QueryObjects/LibraryBookQueries.cs index 9c60f3fb..b18506e3 100644 --- a/Source/DataLayer/QueryObjects/LibraryBookQueries.cs +++ b/Source/DataLayer/QueryObjects/LibraryBookQueries.cs @@ -43,18 +43,41 @@ namespace DataLayer .Include(le => le.Book).ThenInclude(b => b.ContributorsLink).ThenInclude(c => c.Contributor) .Include(le => le.Book).ThenInclude(b => b.Category).ThenInclude(c => c.ParentCategory); + public static IEnumerable FindOrphanedEpisodes(this IEnumerable libraryBooks) + { + var parentedEpisodes = + libraryBooks + .Where(lb => lb.Book.IsEpisodeParent()) + .SelectMany(s => libraryBooks.FindChildren(s)); + + return + libraryBooks + .Where(lb => lb.Book.IsEpisodeChild()) + .ExceptBy( + parentedEpisodes + .Select(ge => ge.Book.AudibleProductId), ge => ge.Book.AudibleProductId); + } + #nullable enable public static LibraryBook? FindSeriesParent(this IEnumerable libraryBooks, LibraryBook seriesEpisode) { if (seriesEpisode.Book.SeriesLink is null) return null; - //Parent books will always have exactly 1 SeriesBook due to how - //they are imported in ApiExtended.getChildEpisodesAsync() - return libraryBooks.FirstOrDefault( - lb => - lb.Book.IsEpisodeParent() && - seriesEpisode.Book.SeriesLink.Any( - s => s.Series.AudibleSeriesId == lb.Book.SeriesLink.Single().Series.AudibleSeriesId)); + try + { + //Parent books will always have exactly 1 SeriesBook due to how + //they are imported in ApiExtended.getChildEpisodesAsync() + return libraryBooks.FirstOrDefault( + lb => + lb.Book.IsEpisodeParent() && + seriesEpisode.Book.SeriesLink.Any( + s => s.Series.AudibleSeriesId == lb.Book.SeriesLink.Single().Series.AudibleSeriesId)); + } + catch (System.Exception ex) + { + Serilog.Log.Error(ex, "Query error in {0}", nameof(FindSeriesParent)); + return null; + } } #nullable disable diff --git a/Source/LibationWinForms/GridView/ProductsGrid.cs b/Source/LibationWinForms/GridView/ProductsGrid.cs index eea3dc67..eacf9a8e 100644 --- a/Source/LibationWinForms/GridView/ProductsGrid.cs +++ b/Source/LibationWinForms/GridView/ProductsGrid.cs @@ -114,11 +114,17 @@ namespace LibationWinForms.GridView internal void BindToGrid(List dbBooks) { - var geList = dbBooks.Where(lb => lb.Book.IsProduct()).Select(b => new LibraryBookEntry(b)).Cast().ToList(); + var geList = dbBooks + .Where(lb => lb.Book.IsProduct()) + .Select(b => new LibraryBookEntry(b)) + .Cast() + .ToList(); var episodes = dbBooks.Where(lb => lb.Book.IsEpisodeChild()); - - foreach (var parent in dbBooks.Where(lb => lb.Book.IsEpisodeParent())) + + var seriesBooks = dbBooks.Where(lb => lb.Book.IsEpisodeParent()).ToList(); + + foreach (var parent in seriesBooks) { var seriesEpisodes = episodes.FindChildren(parent); @@ -216,6 +222,7 @@ namespace LibationWinForms.GridView if (existingEpisodeEntry is null) { LibraryBookEntry episodeEntry; + var seriesEntry = seriesEntries.FindSeriesParent(episodeBook); if (seriesEntry is null) @@ -225,13 +232,14 @@ namespace LibationWinForms.GridView if (seriesBook is null) { - //This should be impossible because the importer ensures every episode has a parent. - var ex = new ApplicationException($"Episode's series parent not found in database."); - var seriesLinks = string.Join("\r\n", episodeBook.Book.SeriesLink?.Select(sb => $"{nameof(sb.Series.Name)}={sb.Series.Name}, {nameof(sb.Series.AudibleSeriesId)}={sb.Series.AudibleSeriesId}")); - Serilog.Log.Logger.Error(ex, "Episode={episodeBook}, Series: {seriesLinks}", episodeBook, seriesLinks); - throw ex; + //This is only possible if the user's db has some malformed + //entries from earlier Libation releases that could not be + //automatically fixed. Log, but don't throw. + Serilog.Log.Logger.Error("Episode={0}, Episode Series: {1}", episodeBook, episodeBook.Book.SeriesNames()); + return; } + seriesEntry = new SeriesEntry(seriesBook, episodeBook); seriesEntries.Add(seriesEntry); diff --git a/Source/LibationWinForms/GridView/QueryExtensions.cs b/Source/LibationWinForms/GridView/QueryExtensions.cs index d084aec7..1fb52bdf 100644 --- a/Source/LibationWinForms/GridView/QueryExtensions.cs +++ b/Source/LibationWinForms/GridView/QueryExtensions.cs @@ -24,12 +24,20 @@ namespace LibationWinForms.GridView { if (seriesEpisode.Book.SeriesLink is null) return null; - //Parent books will always have exactly 1 SeriesBook due to how - //they are imported in ApiExtended.getChildEpisodesAsync() - return gridEntries.SeriesEntries().FirstOrDefault( - lb => - seriesEpisode.Book.SeriesLink.Any( - s => s.Series.AudibleSeriesId == lb.LibraryBook.Book.SeriesLink.Single().Series.AudibleSeriesId)); + try + { + //Parent books will always have exactly 1 SeriesBook due to how + //they are imported in ApiExtended.getChildEpisodesAsync() + return gridEntries.SeriesEntries().FirstOrDefault( + lb => + seriesEpisode.Book.SeriesLink.Any( + s => s.Series.AudibleSeriesId == lb.LibraryBook.Book.SeriesLink.Single().Series.AudibleSeriesId)); + } + catch (Exception ex) + { + Serilog.Log.Error(ex, "Query error in {0}", nameof(FindSeriesParent)); + return null; + } } } #nullable disable