sqlite db file should live in LibationFiles dir, not in app dir

This commit is contained in:
Robert McRackan 2019-12-09 14:44:46 -05:00
parent 65dc273e12
commit bcc237c693
17 changed files with 150 additions and 120 deletions

View File

@ -0,0 +1,16 @@
using System;
using DataLayer;
using FileManager;
namespace ApplicationServices
{
public static class DbContexts
{
//// idea for future command/query separation
// public static LibationContext GetCommandContext() { }
// public static LibationContext GetQueryContext() { }
public static LibationContext GetContext()
=> LibationContext.Create(SqliteStorage.ConnectionString);
}
}

View File

@ -18,7 +18,8 @@ namespace ApplicationServices
var totalCount = items.Count;
Serilog.Log.Logger.Debug($"GetAllLibraryItems: Total count {totalCount}");
var libImporter = new LibraryImporter();
using var context = DbContexts.GetContext();
var libImporter = new LibraryImporter(context);
var newCount = await Task.Run(() => libImporter.Import(items));
Serilog.Log.Logger.Debug($"Import: New count {newCount}");

View File

@ -1,4 +1,5 @@
using DataLayer;
using System.IO;
using DataLayer;
using LibationSearchEngine;
namespace ApplicationServices
@ -7,18 +8,18 @@ namespace ApplicationServices
{
public static void FullReIndex()
{
var engine = new SearchEngine();
var engine = new SearchEngine(DbContexts.GetContext());
engine.CreateNewIndex();
}
public static SearchResultSet Search(string searchString)
{
var engine = new SearchEngine();
var engine = new SearchEngine(DbContexts.GetContext());
try
{
return engine.Search(searchString);
}
catch (System.IO.FileNotFoundException)
catch (FileNotFoundException)
{
FullReIndex();
return engine.Search(searchString);
@ -27,12 +28,12 @@ namespace ApplicationServices
public static void UpdateBookTags(Book book)
{
var engine = new SearchEngine();
var engine = new SearchEngine(DbContexts.GetContext());
try
{
engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
}
catch (System.IO.FileNotFoundException)
catch (FileNotFoundException)
{
FullReIndex();
engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);

View File

@ -25,10 +25,10 @@ namespace DataLayer
public DbSet<Series> Series { get; private set; }
public DbSet<Category> Categories { get; private set; }
public static LibationContext Create()
public static LibationContext Create(string connectionString)
{
var factory = new LibationContextFactory();
var context = factory.Create();
var context = factory.Create(connectionString);
return context;
}

View File

@ -8,14 +8,11 @@ namespace DataLayer
{
public static class BookQueries
{
public static Book GetBook_Flat_NoTracking(string productId)
{
using var context = LibationContext.Create();
return context
public static Book GetBook_Flat_NoTracking(this LibationContext context, string productId)
=> context
.Books
.AsNoTracking()
.GetBook(productId);
}
public static Book GetBook(this IQueryable<Book> books, string productId)
=> books

View File

@ -12,24 +12,18 @@ namespace DataLayer
.GetLibrary()
.ToList();
public static List<LibraryBook> GetLibrary_Flat_NoTracking()
{
using var context = LibationContext.Create();
return context
public static List<LibraryBook> GetLibrary_Flat_NoTracking(this LibationContext context)
=> context
.Library
.AsNoTracking()
.GetLibrary()
.ToList();
}
public static LibraryBook GetLibraryBook_Flat_NoTracking(string productId)
{
using var context = LibationContext.Create();
return context
public static LibraryBook GetLibraryBook_Flat_NoTracking(this LibationContext context, string productId)
=> context
.Library
.AsNoTracking()
.GetLibraryBook(productId);
}
/// <summary>This is still IQueryable. YOU MUST CALL ToList() YOURSELF</summary>
public static IQueryable<LibraryBook> GetLibrary(this IQueryable<LibraryBook> library)

View File

@ -9,29 +9,31 @@ namespace DtoImporterService
{
public class BookImporter : ItemsImporterBase
{
public BookImporter(LibationContext context) : base(context) { }
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new BookValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
protected override int DoImport(IEnumerable<Item> items)
{
// pre-req.s
new ContributorImporter().Import(items, context);
new SeriesImporter().Import(items, context);
new CategoryImporter().Import(items, context);
new ContributorImporter(DbContext).Import(items);
new SeriesImporter(DbContext).Import(items);
new CategoryImporter(DbContext).Import(items);
// get distinct
var productIds = items.Select(i => i.ProductId).ToList();
// load db existing => .Local
loadLocal_books(productIds, context);
loadLocal_books(productIds);
// upsert
var qtyNew = upsertBooks(items, context);
var qtyNew = upsertBooks(items);
return qtyNew;
}
private void loadLocal_books(List<string> productIds, LibationContext context)
private void loadLocal_books(List<string> productIds)
{
var localProductIds = context.Books.Local.Select(b => b.AudibleProductId);
var localProductIds = DbContext.Books.Local.Select(b => b.AudibleProductId);
var remainingProductIds = productIds
.Distinct()
.Except(localProductIds)
@ -39,29 +41,29 @@ namespace DtoImporterService
// GetBooks() eager loads Series, category, et al
if (remainingProductIds.Any())
context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
DbContext.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
}
private int upsertBooks(IEnumerable<Item> items, LibationContext context)
private int upsertBooks(IEnumerable<Item> items)
{
var qtyNew = 0;
foreach (var item in items)
{
var book = context.Books.Local.SingleOrDefault(p => p.AudibleProductId == item.ProductId);
var book = DbContext.Books.Local.SingleOrDefault(p => p.AudibleProductId == item.ProductId);
if (book is null)
{
book = createNewBook(item, context);
book = createNewBook(item);
qtyNew++;
}
updateBook(item, book, context);
updateBook(item, book);
}
return qtyNew;
}
private static Book createNewBook(Item item, LibationContext context)
private Book createNewBook(Item item)
{
// absence of authors is very rare, but possible
if (!item.Authors?.Any() ?? true)
@ -70,7 +72,7 @@ namespace DtoImporterService
// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
var authors = item
.Authors
.Select(a => context.Contributors.Local.Single(c => a.Name == c.Name))
.Select(a => DbContext.Contributors.Local.Single(c => a.Name == c.Name))
.ToList();
var narrators
@ -80,15 +82,15 @@ namespace DtoImporterService
// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
: item
.Narrators
.Select(n => context.Contributors.Local.Single(c => n.Name == c.Name))
.Select(n => DbContext.Contributors.Local.Single(c => n.Name == c.Name))
.ToList();
// categories are laid out for a breadcrumb. category is 1st, subcategory is 2nd
// absence of categories is very rare, but possible
var lastCategory = item.Categories.LastOrDefault()?.CategoryId ?? "";
var category = context.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == lastCategory);
var category = DbContext.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == lastCategory);
var book = context.Books.Add(new Book(
var book = DbContext.Books.Add(new Book(
new AudibleProductId(item.ProductId),
item.Title,
item.Description,
@ -101,7 +103,7 @@ namespace DtoImporterService
var publisherName = item.Publisher;
if (!string.IsNullOrWhiteSpace(publisherName))
{
var publisher = context.Contributors.Local.Single(c => publisherName == c.Name);
var publisher = DbContext.Contributors.Local.Single(c => publisherName == c.Name);
book.ReplacePublisher(publisher);
}
@ -113,7 +115,7 @@ namespace DtoImporterService
return book;
}
private static void updateBook(Item item, Book book, LibationContext context)
private void updateBook(Item item, Book book)
{
// set/update book-specific info which may have changed
book.PictureId = item.PictureId;
@ -128,7 +130,7 @@ namespace DtoImporterService
{
foreach (var seriesEntry in item.Series)
{
var series = context.Series.Local.Single(s => seriesEntry.SeriesId == s.AudibleSeriesId);
var series = DbContext.Series.Local.Single(s => seriesEntry.SeriesId == s.AudibleSeriesId);
book.UpsertSeries(series, seriesEntry.Index);
}
}

View File

@ -9,25 +9,27 @@ namespace DtoImporterService
{
public class CategoryImporter : ItemsImporterBase
{
public CategoryImporter(LibationContext context) : base(context) { }
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new CategoryValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
protected override int DoImport(IEnumerable<Item> items)
{
// get distinct
var categoryIds = items.GetCategoriesDistinct().Select(c => c.CategoryId).ToList();
// load db existing => .Local
loadLocal_categories(categoryIds, context);
loadLocal_categories(categoryIds);
// upsert
var categoryPairs = items.GetCategoryPairsDistinct().ToList();
var qtyNew = upsertCategories(categoryPairs, context);
var qtyNew = upsertCategories(categoryPairs);
return qtyNew;
}
private void loadLocal_categories(List<string> categoryIds, LibationContext context)
private void loadLocal_categories(List<string> categoryIds)
{
var localIds = context.Categories.Local.Select(c => c.AudibleCategoryId);
var localIds = DbContext.Categories.Local.Select(c => c.AudibleCategoryId);
var remainingCategoryIds = categoryIds
.Distinct()
.Except(localIds)
@ -37,11 +39,11 @@ namespace DtoImporterService
// remember to include default/empty/missing
var emptyName = Contributor.GetEmpty().Name;
if (remainingCategoryIds.Any())
context.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId) || c.Name == emptyName).ToList();
DbContext.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId) || c.Name == emptyName).ToList();
}
// only use after loading contributors => local
private int upsertCategories(List<Ladder[]> categoryPairs, LibationContext context)
private int upsertCategories(List<Ladder[]> categoryPairs)
{
var qtyNew = 0;
@ -54,12 +56,12 @@ namespace DtoImporterService
Category parentCategory = null;
if (i == 1)
parentCategory = context.Categories.Local.Single(c => c.AudibleCategoryId == pair[0].CategoryId);
parentCategory = DbContext.Categories.Local.Single(c => c.AudibleCategoryId == pair[0].CategoryId);
var category = context.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == id);
var category = DbContext.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == id);
if (category is null)
{
category = context.Categories.Add(new Category(new AudibleCategoryId(id), name)).Entity;
category = DbContext.Categories.Add(new Category(new AudibleCategoryId(id), name)).Entity;
qtyNew++;
}

View File

@ -9,9 +9,11 @@ namespace DtoImporterService
{
public class ContributorImporter : ItemsImporterBase
{
public ContributorImporter(LibationContext context) : base(context) { }
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new ContributorValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
protected override int DoImport(IEnumerable<Item> items)
{
// get distinct
var authors = items.GetAuthorsDistinct().ToList();
@ -24,23 +26,23 @@ namespace DtoImporterService
.Union(narrators.Select(n => n.Name))
.Where(name => !string.IsNullOrWhiteSpace(name))
.ToList();
loadLocal_contributors(allNames, context);
loadLocal_contributors(allNames);
// upsert
var qtyNew = 0;
qtyNew += upsertPeople(authors, context);
qtyNew += upsertPeople(narrators, context);
qtyNew += upsertPublishers(publishers, context);
qtyNew += upsertPeople(authors);
qtyNew += upsertPeople(narrators);
qtyNew += upsertPublishers(publishers);
return qtyNew;
}
private void loadLocal_contributors(List<string> contributorNames, LibationContext context)
private void loadLocal_contributors(List<string> contributorNames)
{
//// BAD: very inefficient
// var x = context.Contributors.Local.Where(c => !contribNames.Contains(c.Name));
// GOOD: Except() is efficient. Due to hashing, it's close to O(n)
var localNames = context.Contributors.Local.Select(c => c.Name);
var localNames = DbContext.Contributors.Local.Select(c => c.Name);
var remainingContribNames = contributorNames
.Distinct()
.Except(localNames)
@ -50,20 +52,20 @@ namespace DtoImporterService
// remember to include default/empty/missing
var emptyName = Contributor.GetEmpty().Name;
if (remainingContribNames.Any())
context.Contributors.Where(c => remainingContribNames.Contains(c.Name) || c.Name == emptyName).ToList();
DbContext.Contributors.Where(c => remainingContribNames.Contains(c.Name) || c.Name == emptyName).ToList();
}
// only use after loading contributors => local
private int upsertPeople(List<Person> people, LibationContext context)
private int upsertPeople(List<Person> people)
{
var qtyNew = 0;
foreach (var p in people)
{
var person = context.Contributors.Local.SingleOrDefault(c => c.Name == p.Name);
var person = DbContext.Contributors.Local.SingleOrDefault(c => c.Name == p.Name);
if (person == null)
{
person = context.Contributors.Add(new Contributor(p.Name, p.Asin)).Entity;
person = DbContext.Contributors.Add(new Contributor(p.Name, p.Asin)).Entity;
qtyNew++;
}
}
@ -72,15 +74,15 @@ namespace DtoImporterService
}
// only use after loading contributors => local
private int upsertPublishers(List<string> publishers, LibationContext context)
private int upsertPublishers(List<string> publishers)
{
var qtyNew = 0;
foreach (var publisherName in publishers)
{
if (context.Contributors.Local.SingleOrDefault(c => c.Name == publisherName) == null)
if (DbContext.Contributors.Local.SingleOrDefault(c => c.Name == publisherName) == null)
{
context.Contributors.Add(new Contributor(publisherName));
DbContext.Contributors.Add(new Contributor(publisherName));
qtyNew++;
}
}

View File

@ -3,23 +3,25 @@ using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
using Dinah.Core;
namespace DtoImporterService
{
public interface IContextRunner<T>
public abstract class ImporterBase<T>
{
public TResult Run<TResult>(Func<T, LibationContext, TResult> func, T param, LibationContext context = null)
protected LibationContext DbContext { get; }
public ImporterBase(LibationContext context)
{
if (context is null)
{
using (context = LibationContext.Create())
{
var r = Run(func, param, context);
context.SaveChanges();
return r;
}
ArgumentValidator.EnsureNotNull(DbContext, nameof(context));
DbContext = context;
}
/// <summary>LONG RUNNING. call with await Task.Run</summary>
public int Import(T param) => Run(DoImport, param);
public TResult Run<TResult>(Func<T, TResult> func, T param)
{
try
{
var exceptions = Validate(param);
@ -34,7 +36,7 @@ namespace DtoImporterService
try
{
var result = func(param, context);
var result = func(param);
return result;
}
catch (Exception ex)
@ -43,18 +45,13 @@ namespace DtoImporterService
throw;
}
}
IEnumerable<Exception> Validate(T param);
}
public abstract class ImporterBase<T> : IContextRunner<T>
{
/// <summary>LONG RUNNING. call with await Task.Run</summary>
public int Import(T param, LibationContext context = null)
=> ((IContextRunner<T>)this).Run(DoImport, param, context);
protected abstract int DoImport(T elements, LibationContext context);
protected abstract int DoImport(T elements);
public abstract IEnumerable<Exception> Validate(T param);
}
public abstract class ItemsImporterBase : ImporterBase<IEnumerable<Item>> { }
public abstract class ItemsImporterBase : ImporterBase<IEnumerable<Item>>
{
public ItemsImporterBase(LibationContext context) : base(context) { }
}
}

View File

@ -9,27 +9,29 @@ namespace DtoImporterService
{
public class LibraryImporter : ItemsImporterBase
{
public LibraryImporter(LibationContext context) : base(context) { }
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new LibraryValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
protected override int DoImport(IEnumerable<Item> items)
{
new BookImporter().Import(items, context);
new BookImporter(DbContext).Import(items);
var qtyNew = upsertLibraryBooks(items, context);
var qtyNew = upsertLibraryBooks(items);
return qtyNew;
}
private int upsertLibraryBooks(IEnumerable<Item> items, LibationContext context)
private int upsertLibraryBooks(IEnumerable<Item> items)
{
var currentLibraryProductIds = context.Library.Select(l => l.Book.AudibleProductId).ToList();
var currentLibraryProductIds = DbContext.Library.Select(l => l.Book.AudibleProductId).ToList();
var newItems = items.Where(dto => !currentLibraryProductIds.Contains(dto.ProductId)).ToList();
foreach (var newItem in newItems)
{
var libraryBook = new LibraryBook(
context.Books.Local.Single(b => b.AudibleProductId == newItem.ProductId),
DbContext.Books.Local.Single(b => b.AudibleProductId == newItem.ProductId),
newItem.DateAdded);
context.Library.Add(libraryBook);
DbContext.Library.Add(libraryBook);
}
var qtyNew = newItems.Count;

View File

@ -9,44 +9,46 @@ namespace DtoImporterService
{
public class SeriesImporter : ItemsImporterBase
{
public SeriesImporter(LibationContext context) : base(context) { }
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new SeriesValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
protected override int DoImport(IEnumerable<Item> items)
{
// get distinct
var series = items.GetSeriesDistinct().ToList();
// load db existing => .Local
loadLocal_series(series, context);
loadLocal_series(series);
// upsert
var qtyNew = upsertSeries(series, context);
var qtyNew = upsertSeries(series);
return qtyNew;
}
private void loadLocal_series(List<AudibleApiDTOs.Series> series, LibationContext context)
private void loadLocal_series(List<AudibleApiDTOs.Series> series)
{
var seriesIds = series.Select(s => s.SeriesId).ToList();
var localIds = context.Series.Local.Select(s => s.AudibleSeriesId).ToList();
var localIds = DbContext.Series.Local.Select(s => s.AudibleSeriesId).ToList();
var remainingSeriesIds = seriesIds
.Distinct()
.Except(localIds)
.ToList();
if (remainingSeriesIds.Any())
context.Series.Where(s => remainingSeriesIds.Contains(s.AudibleSeriesId)).ToList();
DbContext.Series.Where(s => remainingSeriesIds.Contains(s.AudibleSeriesId)).ToList();
}
private int upsertSeries(List<AudibleApiDTOs.Series> requestedSeries, LibationContext context)
private int upsertSeries(List<AudibleApiDTOs.Series> requestedSeries)
{
var qtyNew = 0;
foreach (var s in requestedSeries)
{
var series = context.Series.Local.SingleOrDefault(c => c.AudibleSeriesId == s.SeriesId);
var series = DbContext.Series.Local.SingleOrDefault(c => c.AudibleSeriesId == s.SeriesId);
if (series is null)
{
series = context.Series.Add(new DataLayer.Series(new AudibleSeriesId(s.SeriesId))).Entity;
series = DbContext.Series.Add(new DataLayer.Series(new AudibleSeriesId(s.SeriesId))).Entity;
qtyNew++;
}
series.UpdateName(s.SeriesName);

View File

@ -1,6 +1,7 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using ApplicationServices;
using DataLayer;
using Dinah.Core.ErrorHandling;
@ -29,7 +30,7 @@ namespace FileLiberator
/// <returns>Returns either the status handler from the process, or null if all books have been processed</returns>
public static async Task<StatusHandler> ProcessSingleAsync(this IProcessable processable, string productId)
{
using var context = LibationContext.Create();
using var context = DbContexts.GetContext();
var libraryBook = context
.Library
.GetLibrary()
@ -55,7 +56,7 @@ namespace FileLiberator
private static LibraryBook getNextValidBook(this IProcessable processable)
{
var libraryBooks = LibraryQueries.GetLibrary_Flat_NoTracking();
var libraryBooks = DbContexts.GetContext().GetLibrary_Flat_NoTracking();
foreach (var libraryBook in libraryBooks)
if (processable.Validate(libraryBook))

View File

@ -0,0 +1,11 @@
using System.IO;
namespace FileManager
{
public static class SqliteStorage
{
// not customizable. don't move to config
private static string databasePath => Path.Combine(Configuration.Instance.LibationFiles, "LibationContext.db");
public static string ConnectionString => $"Data Source={databasePath};Foreign Keys=False;";
}
}

View File

@ -3,7 +3,6 @@ using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DataLayer;
using Dinah.Core;
using FileManager;
@ -19,6 +18,8 @@ namespace LibationSearchEngine
{
public const Lucene.Net.Util.Version Version = Lucene.Net.Util.Version.LUCENE_30;
private LibationContext context { get; }
// not customizable. don't move to config
private static string SearchEngineDirectory { get; }
= new System.IO.DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("SearchEngine").FullName;
@ -160,6 +161,8 @@ namespace LibationSearchEngine
private Directory getIndex() => FSDirectory.Open(SearchEngineDirectory);
public SearchEngine(LibationContext context) => this.context = context;
public void CreateNewIndex(bool overwrite = true)
{
// 300 products
@ -172,7 +175,7 @@ namespace LibationSearchEngine
log();
var library = LibraryQueries.GetLibrary_Flat_NoTracking();
var library = context.GetLibrary_Flat_NoTracking();
log();
@ -233,7 +236,7 @@ namespace LibationSearchEngine
/// <summary>Long running. Use await Task.Run(() => UpdateBook(productId))</summary>
public void UpdateBook(string productId)
{
var libraryBook = LibraryQueries.GetLibraryBook_Flat_NoTracking(productId);
var libraryBook = context.GetLibraryBook_Flat_NoTracking(productId);
var term = new Term(_ID_, productId);
var document = createBookIndexDocument(libraryBook);

View File

@ -1,11 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using System.Windows.Forms;
using ApplicationServices;
using DataLayer;
using Dinah.Core;
using Dinah.Core.Collections.Generic;
using Dinah.Core.Drawing;
using Dinah.Core.Windows.Forms;
using FileManager;
@ -107,7 +106,8 @@ namespace LibationWinForm
#region bottom: backup counts
private void setBackupCounts(object _, object __)
{
var books = LibraryQueries.GetLibrary_Flat_NoTracking()
var books = DbContexts.GetContext()
.GetLibrary_Flat_NoTracking()
.Select(sp => sp.Book)
.ToList();

View File

@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Windows.Forms;
@ -279,7 +278,7 @@ namespace LibationWinForm
//
// transform into sorted GridEntry.s BEFORE binding
//
context = LibationContext.Create();
context = DbContexts.GetContext();
var lib = context.GetLibrary_Flat_WithTracking();
// if no data. hide all columns. return