tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add migration to try and fix db for incorrect or missing espiode series entries.

Browse Source
This commit is contained in:
Michael Bucari-Tovo 2022-06-11 12:41:20 -06:00
parent 984119c7ee
commit 1ae5f99bf0
6 changed files with 205 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
Source/AppScaffolding/LibationScaffolding.cs
View File

@ -5,10 +5,10 @@ using System.Linq;
using System.Reflection;
using ApplicationServices;
using AudibleUtilities;
using Dinah.Core;
using Dinah.Core.IO;
using Dinah.Core.Logging;
using LibationFileManager;
using Microsoft.EntityFrameworkCore;
using Newtonsoft.Json.Linq;
using Serilog;
@ -405,25 +405,58 @@ namespace AppScaffolding
public static void migrate_from_7_10_1(Configuration config)
{
//This migration removes books and series with SERIES_ prefix that were created
//as a hack workaround in 7.10.1. Said workaround was removed in 7.10.2
var migrated = config.GetNonString<bool>(nameof(migrate_from_7_10_1));
if (migrated) return;
//https://github.com/rmcrackan/Libation/issues/270#issuecomment-1152863629
//This migration helps fix databases contaminated with the 7.10.1 hack workaround
//and those with improperly identified or missing series. This does not solve cases
//where individual episodes are in the db with a valid series link, but said series'
//parents have not been imported into the database. For those cases, Libation will
//attempt fixup by retrieving parents from the catalog endpoint
using var context = DbContexts.GetContext();
var booksToRemove = context.Books.Where(b => b.AudibleProductId.StartsWith("SERIES_")).ToArray();
var seriesToRemove = context.Series.Where(s => s.AudibleSeriesId.StartsWith("SERIES_")).ToArray();
var lbToRemove = context.LibraryBooks.Where(lb => booksToRemove.Any(b => b == lb.Book)).ToArray();
//This migration removes books and series with SERIES_ prefix that were created
//as a hack workaround in 7.10.1. Said workaround was removed in 7.10.2
string removeHackSeries = "delete " +
"from series " +
"where AudibleSeriesId like 'SERIES%'";
context.LibraryBooks.RemoveRange(lbToRemove);
context.Books.RemoveRange(booksToRemove);
context.Series.RemoveRange(seriesToRemove);
string removeHackBooks = "delete " +
"from books " +
"where AudibleProductId like 'SERIES%'";
//Detect series parents that were added to the database as books with ContentType.Episode,
//and change them to ContentType.Parent
string updateContentType =
"UPDATE books " +
"SET contenttype = 4 " +
"WHERE audibleproductid IN(SELECT books.audibleproductid " +
"FROM books " +
"INNER JOIN series " +
"ON ( books.audibleproductid = " +
"series.audibleseriesid) " +
"WHERE books.contenttype = 2)";
//Then detect series parents that were added to the database as books with ContentType.Parent
//but are missing a series link, and add the link (don't know how this happened)
string addMissingSeriesLink =
"INSERT INTO seriesbook " +
"SELECT series.seriesid, " +
"books.bookid, " +
"'- 1' " +
"FROM books " +
"LEFT OUTER JOIN seriesbook " +
"ON books.bookid = seriesbook.bookid " +
"INNER JOIN series " +
"ON books.audibleproductid = series.audibleseriesid " +
"WHERE books.contenttype = 4 " +
"AND seriesbook.seriesid IS NULL";
context.Database.ExecuteSqlRaw(removeHackSeries);
context.Database.ExecuteSqlRaw(removeHackBooks);
context.Database.ExecuteSqlRaw(updateContentType);
context.Database.ExecuteSqlRaw(addMissingSeriesLink);
LibraryCommands.SaveContext(context);
config.SetObject(nameof(migrate_from_7_10_1), true);
}
}
}

93
Source/ApplicationServices/LibraryCommands.cs
View File

@ -126,6 +126,22 @@ namespace ApplicationServices
if (totalCount == 0)
return default;
Log.Logger.Information("Begin scan for orphaned episode parents");
var newParents = await findAndAddMissingParents(apiExtendedfunc, accounts);
Log.Logger.Information($"Orphan episode scan complete. New parents count {newParents}");
if (newParents >= 0)
{
//If any episodes are still orphaned, their series have been
//removed from the catalog and wel'll never be able to find them.
//only do this if findAndAddMissingParents returned >= 0. If it
//returned < 0, an error happened and there's still a chance that
//a future successful run will find missing parents.
removedOrphanedEpisodes();
}
Log.Logger.Information("Begin long-running import");
logTime($"pre {nameof(importIntoDbAsync)}");
var newCount = await importIntoDbAsync(importItems);
@ -219,6 +235,83 @@ namespace ApplicationServices
return newCount;
}
static void removedOrphanedEpisodes()
{
using var context = DbContexts.GetContext();
try
{
var orphanedEpisodes =
context
.GetLibrary_Flat_NoTracking(includeParents: true)
.FindOrphanedEpisodes();
context.LibraryBooks.RemoveRange(orphanedEpisodes);
context.Books.RemoveRange(orphanedEpisodes.Select(lb => lb.Book));
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "An error occured while trying to remove orphaned episodes from the database");
}
}
static async Task<int> findAndAddMissingParents(Func<Account, Task<ApiExtended>> apiExtendedfunc, Account[] accounts)
{
using var context = DbContexts.GetContext();
var library = context.GetLibrary_Flat_NoTracking(includeParents: true);
try
{
var orphanedEpisodes = library.FindOrphanedEpisodes().ToList();
if (!orphanedEpisodes.Any())
return -1;
var orphanedSeries =
orphanedEpisodes
.SelectMany(lb => lb.Book.SeriesLink)
.DistinctBy(s => s.Series.AudibleSeriesId)
.ToList();
// We're only calling the Catalog endpoint, so it doesn't matter which account we use.
var apiExtended = await apiExtendedfunc(accounts[0]);
var seriesParents = orphanedSeries.Select(o => o.Series.AudibleSeriesId).ToList();
var items = await apiExtended.Api.GetCatalogProductsAsync(seriesParents, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS);
List<ImportItem> newParentsImportItems = new();
foreach (var sp in orphanedSeries)
{
var serie = items.First(i => i.Asin == sp.Series.AudibleSeriesId);
var lb = orphanedEpisodes.First(l => l.Book.AudibleProductId == sp.Book.AudibleProductId);
if (serie.Relationships is null)
continue;
serie.PurchaseDate = new DateTimeOffset(lb.DateAdded);
serie.Series = new AudibleApi.Common.Series[]
{
new AudibleApi.Common.Series{ Asin = serie.Asin, Title = serie.TitleWithSubtitle, Sequence = "-1"}
};
newParentsImportItems.Add(new ImportItem { DtoItem = serie, AccountId = lb.Account, LocaleName = lb.Book.Locale });
}
var newCoutn = new LibraryBookImporter(context)
.Import(newParentsImportItems);
await context.SaveChangesAsync();
return newCoutn;
}
catch (Exception ex)
{
Serilog.Log.Logger.Error(ex, "An error occured while trying to scan for orphaned episode parents.");
return -1;
}
}
public static int SaveContext(LibationContext context)
{
try

4
Source/DataLayer/QueryObjects/BookQueries.cs
View File

@ -44,6 +44,8 @@ namespace DataLayer
public static bool IsEpisodeParent(this Book book)
=> book.ContentType is ContentType.Parent;
public static bool HasLiberated(this Book book)
=> book.UserDefinedItem.BookStatus is LiberatedStatus.Liberated ||
book.UserDefinedItem.PdfStatus is not null and LiberatedStatus.Liberated;
}
}

23
Source/DataLayer/QueryObjects/LibraryBookQueries.cs
View File

@ -43,11 +43,28 @@ namespace DataLayer
.Include(le => le.Book).ThenInclude(b => b.ContributorsLink).ThenInclude(c => c.Contributor)
.Include(le => le.Book).ThenInclude(b => b.Category).ThenInclude(c => c.ParentCategory);
public static IEnumerable<LibraryBook> FindOrphanedEpisodes(this IEnumerable<LibraryBook> libraryBooks)
{
var parentedEpisodes =
libraryBooks
.Where(lb => lb.Book.IsEpisodeParent())
.SelectMany(s => libraryBooks.FindChildren(s));
return
libraryBooks
.Where(lb => lb.Book.IsEpisodeChild())
.ExceptBy(
parentedEpisodes
.Select(ge => ge.Book.AudibleProductId), ge => ge.Book.AudibleProductId);
}
#nullable enable
public static LibraryBook? FindSeriesParent(this IEnumerable<LibraryBook> libraryBooks, LibraryBook seriesEpisode)
{
if (seriesEpisode.Book.SeriesLink is null) return null;
try
{
//Parent books will always have exactly 1 SeriesBook due to how
//they are imported in ApiExtended.getChildEpisodesAsync()
return libraryBooks.FirstOrDefault(
@ -56,6 +73,12 @@ namespace DataLayer
seriesEpisode.Book.SeriesLink.Any(
s => s.Series.AudibleSeriesId == lb.Book.SeriesLink.Single().Series.AudibleSeriesId));
}
catch (System.Exception ex)
{
Serilog.Log.Error(ex, "Query error in {0}", nameof(FindSeriesParent));
return null;
}
}
#nullable disable
public static IEnumerable<LibraryBook> FindChildren(this IEnumerable<LibraryBook> bookList, LibraryBook parent)

22
Source/LibationWinForms/GridView/ProductsGrid.cs
View File

@ -114,11 +114,17 @@ namespace LibationWinForms.GridView
internal void BindToGrid(List<LibraryBook> dbBooks)
{
var geList = dbBooks.Where(lb => lb.Book.IsProduct()).Select(b => new LibraryBookEntry(b)).Cast<GridEntry>().ToList();
var geList = dbBooks
.Where(lb => lb.Book.IsProduct())
.Select(b => new LibraryBookEntry(b))
.Cast<GridEntry>()
.ToList();
var episodes = dbBooks.Where(lb => lb.Book.IsEpisodeChild());
foreach (var parent in dbBooks.Where(lb => lb.Book.IsEpisodeParent()))
var seriesBooks = dbBooks.Where(lb => lb.Book.IsEpisodeParent()).ToList();
foreach (var parent in seriesBooks)
{
var seriesEpisodes = episodes.FindChildren(parent);
@ -216,6 +222,7 @@ namespace LibationWinForms.GridView
if (existingEpisodeEntry is null)
{
LibraryBookEntry episodeEntry;
var seriesEntry = seriesEntries.FindSeriesParent(episodeBook);
if (seriesEntry is null)
@ -225,13 +232,14 @@ namespace LibationWinForms.GridView
if (seriesBook is null)
{
//This should be impossible because the importer ensures every episode has a parent.
var ex = new ApplicationException($"Episode's series parent not found in database.");
var seriesLinks = string.Join("\r\n", episodeBook.Book.SeriesLink?.Select(sb => $"{nameof(sb.Series.Name)}={sb.Series.Name}, {nameof(sb.Series.AudibleSeriesId)}={sb.Series.AudibleSeriesId}"));
Serilog.Log.Logger.Error(ex, "Episode={episodeBook}, Series: {seriesLinks}", episodeBook, seriesLinks);
throw ex;
//This is only possible if the user's db has some malformed
//entries from earlier Libation releases that could not be
//automatically fixed. Log, but don't throw.
Serilog.Log.Logger.Error("Episode={0}, Episode Series: {1}", episodeBook, episodeBook.Book.SeriesNames());
return;
}
seriesEntry = new SeriesEntry(seriesBook, episodeBook);
seriesEntries.Add(seriesEntry);

8
Source/LibationWinForms/GridView/QueryExtensions.cs
View File

@ -24,6 +24,8 @@ namespace LibationWinForms.GridView
{
if (seriesEpisode.Book.SeriesLink is null) return null;
try
{
//Parent books will always have exactly 1 SeriesBook due to how
//they are imported in ApiExtended.getChildEpisodesAsync()
return gridEntries.SeriesEntries().FirstOrDefault(
@ -31,6 +33,12 @@ namespace LibationWinForms.GridView
seriesEpisode.Book.SeriesLink.Any(
s => s.Series.AudibleSeriesId == lb.LibraryBook.Book.SeriesLink.Single().Series.AudibleSeriesId));
}
catch (Exception ex)
{
Serilog.Log.Error(ex, "Query error in {0}", nameof(FindSeriesParent));
return null;
}
}
}
#nullable disable
}