scraping => api transition almost complete

AudibleApiDomainService/AudibleApiDomainService.csproj

@@ -1,13 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
-    <ProjectReference Include="..\FileManager\FileManager.csproj" />
-    <ProjectReference Include="..\ScrapingDomainServices\ScrapingDomainServices.csproj" />
-  </ItemGroup>
-
-</Project>

AudibleApiDomainService/AudibleApiLibationClient.cs

@@ -1,151 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Net.Http;
-using System.Threading.Tasks;
-using AudibleApi;
-using AudibleApi.Authentication;
-using AudibleApi.Authorization;
-using DTOs;
-
-namespace AudibleApiDomainService
-{
-	public class AudibleApiLibationClient
-	{
-		private Api _api;
-
-		#region initialize api
-		private AudibleApiLibationClient() { }
-		public async static Task<AudibleApiLibationClient> CreateClientAsync(Settings settings, IAudibleApiResponder responder)
-		{
-			Localization.SetLocale(settings.LocaleCountryCode);
-
-			Api api;
-			try
-			{
-				api = await EzApiCreator.GetApiAsync(settings.IdentityFilePath);
-			}
-			catch
-			{
-				var inMemoryIdentity = await loginAsync(responder);
-				api = await EzApiCreator.GetApiAsync(settings.IdentityFilePath, inMemoryIdentity);
-			}
-
-			return new AudibleApiLibationClient { _api = api };
-		}
-
-		// LOGIN PATTERN
-		// - Start with Authenticate. Submit email + pw
-		// - Each step in the login process will return a LoginResult
-		// - Each result which has required user input has a SubmitAsync method
-		// - The final LoginComplete result returns "Identity" -- in-memory authorization items
-		private static async Task<IIdentity> loginAsync(IAudibleApiResponder responder)
-		{
-			var login = new Authenticate();
-
-			var (email, password) = responder.GetLogin();
-
-			var loginResult = await login.SubmitCredentialsAsync(email, password);
-
-			while (true)
-			{
-				switch (loginResult)
-				{
-					case CredentialsPage credentialsPage:
-						var (emailInput, pwInput) = responder.GetLogin();
-						loginResult = await credentialsPage.SubmitAsync(emailInput, pwInput);
-						break;
-
-					case CaptchaPage captchaResult:
-						var imageBytes = await downloadImageAsync(captchaResult.CaptchaImage);
-						var guess = responder.GetCaptchaAnswer(imageBytes);
-						loginResult = await captchaResult.SubmitAsync(guess);
-						break;
-
-					case TwoFactorAuthenticationPage _2fa:
-						var _2faCode = responder.Get2faCode();
-						loginResult = await _2fa.SubmitAsync(_2faCode);
-						break;
-
-					case LoginComplete final:
-						return final.Identity;
-
-					default:
-						throw new Exception("Unknown LoginResult");
-				}
-			}
-		}
-
-		private static async Task<byte[]> downloadImageAsync(Uri imageUri)
-		{
-			using var client = new HttpClient();
-			using var contentStream = await client.GetStreamAsync(imageUri);
-			using var localStream = new MemoryStream();
-			await contentStream.CopyToAsync(localStream);
-			return localStream.ToArray();
-		}
-		#endregion
-
-		public async Task ImportLibraryAsync()
-		{
-			try
-			{
-				var items = await GetLibraryItemsAsync();
-//var (total, newEntries) = await ScrapingDomainServices.Indexer.IndexLibraryAsync(items);
-			}
-			catch (Exception ex)
-			{
-				// catch here for easier debugging
-				throw;
-			}
-		}
-
-		private async Task<List<Item>> GetLibraryItemsAsync()
-		{
-			var allItems = new List<Item>();
-
-			for (var i = 1; ; i++)
-			{
-				var page = await _api.GetLibraryAsync(new LibraryOptions
-				{
-					NumberOfResultPerPage = 1000,
-					PageNumber = i,
-					PurchasedAfter = new DateTime(2000, 1, 1),
-					ResponseGroups = LibraryOptions.ResponseGroupOptions.ALL_OPTIONS
-				});
-
-				// important! use this convert method
-				var libResult = LibraryApiV10.FromJson(page.ToString());
-
-				if (!libResult.Items.Any())
-					break;
-
-				allItems.AddRange(libResult.Items);
-			}
-
-			return allItems;
-		}
-
-		//public async Task DownloadBookAsync(string asinToDownload)
-		//{
-		//	// console example
-		//	using var progressBar = new Dinah.Core.ConsoleLib.ProgressBar();
-		//	var progress = new Progress<Dinah.Core.Net.Http.DownloadProgress>();
-		//	progress.ProgressChanged += (_, e) => progressBar.Report(Math.Round((double)(100 * e.BytesReceived) / e.TotalFileSize.Value) / 100);
-
-		//	logger.WriteLine("Download book");
-		//	var finalFile = await _api.DownloadAaxWorkaroundAsync(asinToDownload, "downloadExample.xyz", progress);
-
-		//	logger.WriteLine(" Done!");
-		//	logger.WriteLine("final file: " + Path.GetFullPath(finalFile));
-
-		//	// benefit of this small delay:
-		//	// - if you try to delete a file too soon after it's created, the OS isn't done with the creation and you can get an unexpected error
-		//	// - give progressBar's internal timer time to finish. if timer is disposed before the final message is processed, "100%" will never get a chance to be displayed
-		//	await Task.Delay(100);
-
-		//	File.Delete(finalFile);
-		//}
-	}
-}

AudibleApiDomainService/IAudibleApiResponder.cs

@@ -1,9 +0,0 @@
-namespace AudibleApiDomainService
-{
-	public interface IAudibleApiResponder
-	{
-		(string email, string password) GetLogin();
-		string GetCaptchaAnswer(byte[] captchaImage);
-		string Get2faCode();
-	}
-}

AudibleApiDomainService/Settings.cs

@@ -1,17 +0,0 @@
-using System.IO;
-using FileManager;
-
-namespace AudibleApiDomainService
-{
-	public class Settings
-	{
-		public string IdentityFilePath { get; }
-		public string LocaleCountryCode { get; }
-
-		public Settings(Configuration config)
-		{
-			IdentityFilePath = Path.Combine(config.LibationFiles, "IdentityTokens.json");
-			LocaleCountryCode = config.LocaleCountryCode;
-		}
-	}
-}

CookieMonster/CookieMonster.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Data.SQLite.Core" Version="1.0.111" />
+    <PackageReference Include="System.Data.SQLite.Core" Version="1.0.112" />
     <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.6.0" />
   </ItemGroup>
 

DTOs/DTOs.csproj

@@ -1,11 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>
-    <TargetFramework>netstandard2.1</TargetFramework>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\..\Dinah.Core\Dinah.Core\Dinah.Core.csproj" />
-  </ItemGroup>
-
-</Project>

DTOs/LibraryApiV10.custom.cs

@@ -1,148 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using Dinah.Core.Collections.Generic;
-
-namespace DTOs
-{
-	public partial class LibraryApiV10
-	{
-		public IEnumerable<Person> AuthorsDistinct => Items.GetAuthorsDistinct();
-		public IEnumerable<string> NarratorsDistinct => Items.GetNarratorsDistinct();
-		public IEnumerable<string> PublishersDistinct => Items.GetPublishersDistinct();
-		public IEnumerable<Series> SeriesDistinct => Items.GetSeriesDistinct();
-		public IEnumerable<Ladder> ParentCategoriesDistinct => Items.GetParentCategoriesDistinct();
-		public IEnumerable<Ladder> ChildCategoriesDistinct => Items.GetChildCategoriesDistinct();
-
-		public override string ToString() => $"{Items.Length} {nameof(Items)}, {ResponseGroups.Length} {nameof(ResponseGroups)}";
-	}
-	public partial class Item
-	{
-		public string ProductId => Asin;
-		public int LengthInMinutes => RuntimeLengthMin ?? 0;
-		public string Description => PublisherSummary;
-		public bool Episodes
-			=> Relationships
-				?.Where(r => r.RelationshipToProduct == RelationshipToProduct.Child && r.RelationshipType == RelationshipType.Episode)
-				.Any()
-			?? false;
-		public string PictureId => ProductImages?.PictureId;
-		public string SupplementUrls => PdfUrl.AbsoluteUri; // item.PdfUrl == item.PdfLink
-		public DateTime DateAdded => PurchaseDate.UtcDateTime;
-
-		public float Product_OverallStars => Convert.ToSingle(Rating?.OverallDistribution.DisplayStars ?? 0);
-		public float Product_PerformanceStars => Convert.ToSingle(Rating?.PerformanceDistribution.DisplayStars ?? 0);
-		public float Product_StoryStars => Convert.ToSingle(Rating?.StoryDistribution.DisplayStars ?? 0);
-
-		public int MyUserRating_Overall => Convert.ToInt32(ProvidedReview?.Ratings.OverallRating ?? 0L);
-		public int MyUserRating_Performance => Convert.ToInt32(ProvidedReview?.Ratings.PerformanceRating ?? 0L);
-		public int MyUserRating_Story => Convert.ToInt32(ProvidedReview?.Ratings.StoryRating ?? 0L);
-
-		public bool IsAbridged
-			=> FormatType.HasValue
-			? FormatType == DTOs.FormatType.Abridged
-			: false;
-		public DateTime? DatePublished => IssueDate?.UtcDateTime; // item.IssueDate == item.ReleaseDate
-		public string Publisher => PublisherName;
-
-		// these category properties assume:
-		// - we're only exposing 1 category, irrespective of how many the Item actually has
-		// - each ladder will have either 1 or 2 levels: parent and optional child
-		public Ladder[] Categories => CategoryLadders?.FirstOrDefault()?.Ladder ?? new Ladder[0];
-		public Ladder ParentCategory => Categories[0];
-		public Ladder ChildCategory => Categories.Length > 1 ? Categories[1] : null;
-
-		// LibraryDTO.DownloadBookLink will be handled differently. see api.DownloadAaxWorkaroundAsync(asin)
-
-		public IEnumerable<Person> AuthorsDistinct => Authors.DistinctBy(a => new { a.Name, a.Asin });
-		public IEnumerable<Person> NarratorsDistinct => Narrators.DistinctBy(a => new { a.Name, a.Asin });
-
-		public override string ToString() => $"[{ProductId}] {Title}";
-	}
-	public partial class Person
-	{
-		public override string ToString() => $"{Name}";
-	}
-	public partial class AvailableCodec
-	{
-		public override string ToString() => $"{Name} {Format} {EnhancedCodec}";
-	}
-	public partial class CategoryLadder
-	{
-		public override string ToString() => Ladder.Select(l => l.CategoryName).Aggregate((a, b) => $"{a} | {b}");
-	}
-	public partial class Ladder
-	{
-		public string CategoryId => Id;
-		public string CategoryName => Name;
-
-		public override string ToString() => $"[{CategoryId}] {CategoryName}";
-	}
-	public partial class ContentRating
-	{
-		public override string ToString() => $"{Steaminess}";
-	}
-	public partial class Review
-	{
-		public override string ToString() => $"{this.Title}";
-	}
-	public partial class GuidedResponse
-	{
-		//public override string ToString() => 
-	}
-	public partial class Ratings
-	{
-		public override string ToString() => $"{OverallRating:0.0}|{PerformanceRating:0.0}|{StoryRating:0.0}";
-	}
-	public partial class ReviewContentScores
-	{
-		public override string ToString() => $"Helpful={NumHelpfulVotes}, Unhelpful={NumUnhelpfulVotes}";
-	}
-	public partial class Plan
-	{
-		public override string ToString() => $"{PlanName}";
-	}
-	public partial class Price
-	{
-		public override string ToString() => $"List={ListPrice}, Lowest={LowestPrice}";
-	}
-	public partial class ListPriceClass
-	{
-		public override string ToString() => $"{Base}";
-	}
-	public partial class ProductImages
-	{
-		public string PictureId
-			=> The500
-				.AbsoluteUri // https://m.media-amazon.com/images/I/51T1NWIkR4L._SL500_.jpg?foo=bar
-				?.Split('/').Last() // 51T1NWIkR4L._SL500_.jpg?foo=bar
-				?.Split('.').First() // 51T1NWIkR4L
-			;
-
-		public override string ToString() => $"{The500}";
-	}
-	public partial class Rating
-	{
-		public override string ToString() => $"{OverallDistribution}|{PerformanceDistribution}|{StoryDistribution}";
-	}
-	public partial class Distribution
-	{
-		public override string ToString() => $"{DisplayStars:0.0}";
-	}
-	public partial class Relationship
-	{
-		public override string ToString() => $"{RelationshipToProduct} {RelationshipType}";
-	}
-	public partial class Series
-	{
-		public string SeriesName => Title;
-		public string SeriesId => Asin;
-		public float Index
-			=> string.IsNullOrEmpty(Sequence)
-			? 0
-			// eg: a book containing volumes 5,6,7,8 has sequence "5-8"
-			: float.Parse(Sequence.Split('-').First());
-
-		public override string ToString() => $"[{SeriesId}] {SeriesName}";
-	}
-}

DTOs/LibraryApiV10Extensions.cs

@@ -1,31 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using Dinah.Core.Collections.Generic;
-
-namespace DTOs
-{
-	public static class LibraryApiV10Extensions
-	{
-		public static IEnumerable<Person> GetAuthorsDistinct(this IEnumerable<Item> items)
-			=> items.SelectMany(i => i.Authors).DistinctBy(a => new { a.Name, a.Asin });
-
-		public static IEnumerable<string> GetNarratorsDistinct(this IEnumerable<Item> items)
-			=> items.SelectMany(i => i.Narrators, (i, n) => n.Name).Distinct();
-
-		public static IEnumerable<string> GetPublishersDistinct(this IEnumerable<Item> items)
-			=> items.Select(i => i.Publisher).Distinct();
-
-		public static IEnumerable<Series> GetSeriesDistinct(this IEnumerable<Item> items)
-			=> items.SelectMany(i => i.Series).DistinctBy(s => new { s.SeriesName, s.SeriesId });
-
-		public static IEnumerable<Ladder> GetParentCategoriesDistinct(this IEnumerable<Item> items)
-			=> items.Select(l => l.ParentCategory).DistinctBy(l => new { l.CategoryName, l.CategoryId });
-
-		public static IEnumerable<Ladder> GetChildCategoriesDistinct(this IEnumerable<Item> items)
-			=> items
-				.Select(l => l.ChildCategory)
-				.Where(l => l != null)
-				.DistinctBy(l => new { l.CategoryName, l.CategoryId });
-	}
-}

DataLayer/UNTESTED/EfClasses/Book.cs

@@ -62,8 +62,7 @@ namespace DataLayer
             string title,
             string description,
             int lengthInMinutes,
-            IEnumerable<Contributor> authors,
+            IEnumerable<Contributor> authors)
-            IEnumerable<Contributor> narrators)
         {
             // validate
             ArgumentValidator.EnsureNotNull(audibleProductId, nameof(audibleProductId));
@@ -88,7 +87,7 @@ namespace DataLayer
 
             // assigns with biz logic
             ReplaceAuthors(authors);
-            ReplaceNarrators(narrators);
+            //ReplaceNarrators(narrators);
 
             // import previously saved tags
             // do this immediately. any save occurs before reloading tags will overwrite persistent tags with new blank entries; all old persisted tags will be lost
@@ -218,7 +217,6 @@ namespace DataLayer
             //  Are automatically loaded, and can only be tracked by a DbContext alongside their owner."
 
             ArgumentValidator.EnsureNotNullOrWhiteSpace(url, nameof(url));
-            url = FileManager.FileUtility.RestoreDeclawed(url);
 
             if (!_supplements.Any(s => url.EqualsInsensitive(url)))
                 _supplements.Add(new Supplement(this, url));

DataLayer/UNTESTED/EfClasses/LibraryBook.cs

@@ -14,7 +14,9 @@ namespace DataLayer
 public string DownloadBookLink { get; private set; }
 
         private LibraryBook() { }
-        public LibraryBook(Book book, DateTime dateAdded, string downloadBookLink)
+        public LibraryBook(Book book, DateTime dateAdded
+, string downloadBookLink = null
+)
         {
             ArgumentValidator.EnsureNotNull(book, nameof(book));
             Book = book;

DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs

@@ -11,7 +11,7 @@ namespace DataLayer
 			using var context = LibationContext.Create();
 			return context
                 .Library
-                .AsNoTracking()
+//.AsNoTracking()
                 .GetLibrary()
                 .ToList();
 		}
@@ -21,7 +21,7 @@ namespace DataLayer
 			using var context = LibationContext.Create();
 			return context
                 .Library
-                .AsNoTracking()
+//.AsNoTracking()
 				.GetLibraryBook(productId);
 		}
 

DtoImporterService/BookImporter.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApiDTOs;
+using DataLayer;
+
+namespace DtoImporterService
+{
+	public class BookImporter : ItemsImporterBase
+	{
+		public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
+		{
+			var exceptions = new List<Exception>();
+
+			if (items.Any(i => string.IsNullOrWhiteSpace(i.ProductId)))
+				exceptions.Add(new ArgumentException($"Collection contains item(s) with blank {nameof(Item.ProductId)}", nameof(items)));
+			if (items.Any(i => string.IsNullOrWhiteSpace(i.Title)))
+				exceptions.Add(new ArgumentException($"Collection contains item(s) with blank {nameof(Item.Title)}", nameof(items)));
+			if (items.Any(i => i.Authors is null))
+				exceptions.Add(new ArgumentException($"Collection contains item(s) with null {nameof(Item.Authors)}", nameof(items)));
+
+			return exceptions;
+		}
+
+		protected override int DoImport(IEnumerable<Item> items, LibationContext context)
+		{
+			// pre-req.s
+			new ContributorImporter().Import(items, context);
+			new SeriesImporter().Import(items, context);
+			new CategoryImporter().Import(items, context);
+
+			// get distinct
+			var productIds = items.Select(i => i.ProductId).ToList();
+
+			// load db existing => .Local
+			loadLocal_books(productIds, context);
+
+			// upsert
+			var qtyNew = upsertBooks(items, context);
+			return qtyNew;
+		}
+
+		private void loadLocal_books(List<string> productIds, LibationContext context)
+		{
+			var localProductIds = context.Books.Local.Select(b => b.AudibleProductId);
+			var remainingProductIds = productIds
+				.Distinct()
+				.Except(localProductIds)
+				.ToList();
+
+			// GetBooks() eager loads Series, category, et al
+			if (remainingProductIds.Any())
+				context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
+		}
+
+		private int upsertBooks(IEnumerable<Item> items, LibationContext context)
+		{
+			var qtyNew = 0;
+
+			foreach (var item in items)
+			{
+				var book = context.Books.Local.SingleOrDefault(p => p.AudibleProductId == item.ProductId);
+				if (book is null)
+				{
+					// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
+					var authors = item
+						.Authors
+						.Select(a => context.Contributors.Local.Single(c => a.Name == c.Name))
+						.ToList();
+
+					book = context.Books.Add(new Book(
+						new AudibleProductId(item.ProductId), item.Title, item.Description, item.LengthInMinutes, authors))
+						.Entity;
+
+					qtyNew++;
+				}
+
+				// if no narrators listed, author is the narrator
+				if (item.Narrators is null || !item.Narrators.Any())
+					item.Narrators = item.Authors;
+				// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
+				var narrators = item
+					.Narrators
+					.Select(n => context.Contributors.Local.Single(c => n.Name == c.Name))
+					.ToList();
+				// not all books have narrators. these will already be using author as narrator. don't undo this
+				if (narrators.Any())
+					book.ReplaceNarrators(narrators);
+
+				// set/update book-specific info which may have changed
+				book.PictureId = item.PictureId;
+				book.UpdateProductRating(item.Product_OverallStars, item.Product_PerformanceStars, item.Product_StoryStars);
+				if (!string.IsNullOrWhiteSpace(item.SupplementUrl))
+					book.AddSupplementDownloadUrl(item.SupplementUrl);
+
+				var publisherName = item.Publisher;
+				if (!string.IsNullOrWhiteSpace(publisherName))
+				{
+					var publisher = context.Contributors.Local.Single(c => publisherName == c.Name);
+					book.ReplacePublisher(publisher);
+				}
+
+				// important to update user-specific info. this will have changed if user has rated/reviewed the book since last library import
+				book.UserDefinedItem.UpdateRating(item.MyUserRating_Overall, item.MyUserRating_Performance, item.MyUserRating_Story);
+
+				//
+				// this was round 1 when it was a 2 step process
+				//
+				//// update series even for existing books. these are occasionally updated
+				//var seriesIds = item.Series.Select(kvp => kvp.SeriesId).ToList();
+				//var allSeries = context.Series.Local.Where(c => seriesIds.Contains(c.AudibleSeriesId)).ToList();
+				//foreach (var series in allSeries)
+				//	book.UpsertSeries(series);
+
+				// these will upsert over library-scraped series, but will not leave orphans
+				if (item.Series != null)
+				{
+					foreach (var seriesEntry in item.Series)
+					{
+						var series = context.Series.Local.Single(s => seriesEntry.SeriesId == s.AudibleSeriesId);
+						book.UpsertSeries(series, seriesEntry.Index);
+					}
+				}
+
+				// categories are laid out for a breadcrumb. category is 1st, subcategory is 2nd
+				var category = context.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == item.Categories.LastOrDefault().CategoryId);
+				if (category != null)
+					book.UpdateCategory(category, context);
+
+				book.UpdateBookDetails(item.IsAbridged, item.DatePublished);
+			}
+
+			return qtyNew;
+		}
+	}
+}

DtoImporterService/CategoryImporter.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApiDTOs;
+using DataLayer;
+
+namespace DtoImporterService
+{
+	public class CategoryImporter : ItemsImporterBase
+	{
+		public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
+		{
+			var exceptions = new List<Exception>();
+
+			var distinct = items.GetCategoriesDistinct();
+			if (distinct.Any(s => s.CategoryId is null))
+				exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with null {nameof(Ladder.CategoryId)}", nameof(items)));
+			if (distinct.Any(s => s.CategoryName is null))
+				exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with null {nameof(Ladder.CategoryName)}", nameof(items)));
+
+			if (items.GetCategoryPairsDistinct().Any(p => p.Length > 2))
+				exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with wrong number of categories. Expecting 0, 1, or 2 categories per title", nameof(items)));
+
+			return exceptions;
+		}
+
+		protected override int DoImport(IEnumerable<Item> items, LibationContext context)
+		{
+			// get distinct
+			var categoryIds = items.GetCategoriesDistinct().Select(c => c.CategoryId).ToList();
+
+			// load db existing => .Local
+			loadLocal_categories(categoryIds, context);
+
+			// upsert
+			var categoryPairs = items.GetCategoryPairsDistinct().ToList();
+			var qtyNew = upsertCategories(categoryPairs, context);
+			return qtyNew;
+		}
+
+		private void loadLocal_categories(List<string> categoryIds, LibationContext context)
+		{
+			var localIds = context.Categories.Local.Select(c => c.AudibleCategoryId);
+			var remainingCategoryIds = categoryIds
+				.Distinct()
+				.Except(localIds)
+				.ToList();
+
+			if (remainingCategoryIds.Any())
+				context.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId)).ToList();
+		}
+
+		// only use after loading contributors => local
+		private int upsertCategories(List<Ladder[]> categoryPairs, LibationContext context)
+		{
+			var qtyNew = 0;
+
+			foreach (var pair in categoryPairs)
+			{
+				for (var i = 0; i < pair.Length; i++)
+				{
+					var id = pair[i].CategoryId;
+					var name = pair[i].CategoryName;
+
+					Category parentCategory = null;
+					if (i == 1)
+						parentCategory = context.Categories.Local.Single(c => c.AudibleCategoryId == pair[0].CategoryId);
+
+					var category = context.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == id);
+					if (category is null)
+					{
+						category = context.Categories.Add(new Category(new AudibleCategoryId(id), name)).Entity;
+						qtyNew++;
+					}
+
+					category.UpdateParentCategory(parentCategory);
+				}
+			}
+
+			return qtyNew;
+		}
+	}
+}

DtoImporterService/ContributorImporter.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApiDTOs;
+using DataLayer;
+
+namespace DtoImporterService
+{
+	public class ContributorImporter : ItemsImporterBase
+	{
+		public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
+		{
+			var exceptions = new List<Exception>();
+
+			if (items.GetAuthorsDistinct().Any(a => string.IsNullOrWhiteSpace(a.Name)))
+				exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Authors)} with null {nameof(Person.Name)}", nameof(items)));
+			if (items.GetNarratorsDistinct().Any(a => string.IsNullOrWhiteSpace(a.Name)))
+				exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Narrators)} with null {nameof(Person.Name)}", nameof(items)));
+
+			return exceptions;
+		}
+
+		protected override int DoImport(IEnumerable<Item> items, LibationContext context)
+		{
+			// get distinct
+			var authors = items.GetAuthorsDistinct().ToList();
+			var narrators = items.GetNarratorsDistinct().ToList();
+			var publishers = items.GetPublishersDistinct().ToList();
+
+			// load db existing => .Local
+			var allNames = authors
+				.Select(a => a.Name)
+				.Union(narrators.Select(n => n.Name))
+				.Union(publishers)
+				.ToList();
+			loadLocal_contributors(allNames, context);
+
+			// upsert
+			var qtyNew = 0;
+			qtyNew += upsertPeople(authors, context);
+			qtyNew += upsertPeople(narrators, context);
+			qtyNew += upsertPublishers(publishers, context);
+			return qtyNew;
+		}
+
+		private void loadLocal_contributors(List<string> contributorNames, LibationContext context)
+		{
+			contributorNames.Remove(null);
+			contributorNames.Remove("");
+
+			//// BAD: very inefficient
+			// var x = context.Contributors.Local.Where(c => !contribNames.Contains(c.Name));
+
+			// GOOD: Except() is efficient. Due to hashing, it's close to O(n)
+			var localNames = context.Contributors.Local.Select(c => c.Name);
+			var remainingContribNames = contributorNames
+				.Distinct()
+				.Except(localNames)
+				.ToList();
+
+			// load existing => local
+			if (remainingContribNames.Any())
+				context.Contributors.Where(c => remainingContribNames.Contains(c.Name)).ToList();
+			// _________________________________^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+			// i tried to extract this pattern, but this part prohibits doing so
+			// wouldn't work anyway for Books.GetBooks()
+		}
+
+		// only use after loading contributors => local
+		private int upsertPeople(List<Person> people, LibationContext context)
+		{
+			var qtyNew = 0;
+
+			foreach (var p in people)
+			{
+				var person = context.Contributors.Local.SingleOrDefault(c => c.Name == p.Name);
+				if (person == null)
+				{
+					person = context.Contributors.Add(new Contributor(p.Name)).Entity;
+					qtyNew++;
+				}
+
+				person.UpdateAudibleAuthorId(p.Asin);
+			}
+
+			return qtyNew;
+		}
+
+		// only use after loading contributors => local
+		private int upsertPublishers(List<string> publishers, LibationContext context)
+		{
+			var qtyNew = 0;
+
+			foreach (var publisherName in publishers)
+			{
+				if (context.Contributors.Local.SingleOrDefault(c => c.Name == publisherName) == null)
+				{
+					context.Contributors.Add(new Contributor(publisherName));
+					qtyNew++;
+				}
+			}
+
+			return qtyNew;
+		}
+	}
+}

DtoImporterService/DtoImporterService.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\audible api\AudibleApi\AudibleApiDTOs\AudibleApiDTOs.csproj" />
+    <ProjectReference Include="..\DataLayer\DataLayer.csproj" />
+  </ItemGroup>
+
+</Project>

DtoImporterService/ImporterBase.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApiDTOs;
+using DataLayer;
+
+namespace DtoImporterService
+{
+	public interface IContextRunner<T>
+	{
+		public TResult Run<TResult>(Func<T, LibationContext, TResult> func, T param, LibationContext context = null)
+		{
+			if (context is null)
+			{
+				using (context = LibationContext.Create())
+				{
+					var r = Run(func, param, context);
+					context.SaveChanges();
+					return r;
+				}
+			}
+
+			var exceptions = Validate(param);
+			if (exceptions != null && exceptions.Any())
+				throw new AggregateException($"Device Jobs Service configuration validation failed", exceptions);
+
+			var result = func(param, context);
+			return result;
+		}
+		IEnumerable<Exception> Validate(T param);
+	}
+
+	public abstract class ImporterBase<T> : IContextRunner<T>
+	{
+		/// <summary>LONG RUNNING. call with await Task.Run</summary>
+		public int Import(T param, LibationContext context = null)
+			=> ((IContextRunner<T>)this).Run(DoImport, param, context);
+
+		protected abstract int DoImport(T elements, LibationContext context);
+		public abstract IEnumerable<Exception> Validate(T param);
+	}
+
+	public abstract class ItemsImporterBase : ImporterBase<IEnumerable<Item>> { }
+}

DtoImporterService/LibraryImporter.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApiDTOs;
+using DataLayer;
+
+namespace DtoImporterService
+{
+	public class LibraryImporter : ItemsImporterBase
+	{
+		public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
+		{
+			var exceptions = new List<Exception>();
+
+			if (items.Any(i => string.IsNullOrWhiteSpace(i.ProductId)))
+				exceptions.Add(new ArgumentException($"Collection contains item(s) with null or blank {nameof(Item.ProductId)}", nameof(items)));
+			if (items.Any(i => i.DateAdded < new DateTime(1980, 1, 1)))
+				exceptions.Add(new ArgumentException($"Collection contains item(s) with invalid {nameof(Item.DateAdded)}", nameof(items)));
+
+			return exceptions;
+		}
+
+		protected override int DoImport(IEnumerable<Item> items, LibationContext context)
+		{
+			new BookImporter().Import(items, context);
+
+			var qtyNew = upsertLibraryBooks(items, context);
+			return qtyNew;
+		}
+
+		private int upsertLibraryBooks(IEnumerable<Item> items, LibationContext context)
+		{
+			var currentLibraryProductIds = context.Library.Select(l => l.Book.AudibleProductId).ToList();
+			var newItems = items.Where(dto => !currentLibraryProductIds.Contains(dto.ProductId)).ToList();
+
+			foreach (var newItem in newItems)
+			{
+				var libraryBook = new LibraryBook(
+					context.Books.Local.Single(b => b.AudibleProductId == newItem.ProductId),
+					newItem.DateAdded
+//,FileManager.FileUtility.RestoreDeclawed(newLibraryDTO.DownloadBookLink)
+					);
+				context.Library.Add(libraryBook);
+			}
+
+			var qtyNew = newItems.Count;
+			return qtyNew;
+		}
+	}
+}

DtoImporterService/SeriesImporter.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using AudibleApiDTOs;
+using DataLayer;
+
+namespace DtoImporterService
+{
+	public class SeriesImporter : ItemsImporterBase
+	{
+		public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
+		{
+			var exceptions = new List<Exception>();
+
+			var distinct = items .GetSeriesDistinct();
+			if (distinct.Any(s => s.SeriesId is null))
+				exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Series)} with null {nameof(AudibleApiDTOs.Series.SeriesId)}", nameof(items)));
+			if (distinct.Any(s => s.SeriesName is null))
+				exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Series)} with null {nameof(AudibleApiDTOs.Series.SeriesName)}", nameof(items)));
+
+			return exceptions;
+		}
+
+		protected override int DoImport(IEnumerable<Item> items, LibationContext context)
+		{
+			// get distinct
+			var series = items.GetSeriesDistinct().ToList();
+
+			// load db existing => .Local
+			var seriesIds = series.Select(s => s.SeriesId).ToList();
+			loadLocal_series(seriesIds, context);
+
+			// upsert
+			var qtyNew = upsertSeries(series, context);
+			return qtyNew;
+		}
+
+		private void loadLocal_series(List<string> seriesIds, LibationContext context)
+		{
+			var localIds = context.Series.Local.Select(s => s.AudibleSeriesId);
+			var remainingSeriesIds = seriesIds
+				.Distinct()
+				.Except(localIds)
+				.ToList();
+
+			if (remainingSeriesIds.Any())
+				context.Series.Where(s => remainingSeriesIds.Contains(s.AudibleSeriesId)).ToList();
+		}
+
+		private int upsertSeries(List<AudibleApiDTOs.Series> requestedSeries, LibationContext context)
+		{
+			var qtyNew = 0;
+
+			foreach (var s in requestedSeries)
+			{
+				var series = context.Series.Local.SingleOrDefault(c => c.AudibleSeriesId == s.SeriesId);
+				if (series is null)
+				{
+					series = context.Series.Add(new DataLayer.Series(new AudibleSeriesId(s.SeriesId))).Entity;
+					qtyNew++;
+				}
+				series.UpdateName(s.SeriesName);
+			}
+
+			return qtyNew;
+		}
+	}
+}

InternalUtilities/InternalUtilities.csproj

@@ -5,6 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
     <ProjectReference Include="..\LibationSearchEngine\LibationSearchEngine.csproj" />
     <ProjectReference Include="..\Scraping\Scraping.csproj" />
   </ItemGroup>

InternalUtilities/UNTESTED/AudibleApiExtensions.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using AudibleApi;
+using AudibleApiDTOs;
+
+//
+// probably not the best place for this
+// but good enough for now
+//
+namespace InternalUtilities
+{
+	public static class AudibleApiExtensions
+	{
+		public static async Task<List<Item>> GetAllLibraryItemsAsync(this Api api)
+		{
+			var allItems = new List<Item>();
+
+			for (var i = 1; ; i++)
+			{
+				var page = await api.GetLibraryAsync(new LibraryOptions
+				{
+					NumberOfResultPerPage = 1000,
+					PageNumber = i,
+					PurchasedAfter = new DateTime(2000, 1, 1),
+					ResponseGroups = LibraryOptions.ResponseGroupOptions.ALL_OPTIONS
+				});
+
+				// important! use this convert method
+				var libResult = LibraryApiV10.FromJson(page.ToString());
+
+				if (!libResult.Items.Any())
+					break;
+
+				allItems.AddRange(libResult.Items);
+			}
+
+			return allItems;
+		}
+	}
+}

Libation.sln

@@ -57,8 +57,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GitAllRepos", "..\GitAllRep
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApi", "..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj", "{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiClientExample", "..\audible api\AudibleApi\AudibleApiClientExample\AudibleApiClientExample.csproj", "{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApi.Tests", "..\audible api\AudibleApi\_Tests\AudibleApi.Tests\AudibleApi.Tests.csproj", "{111420E2-D4F0-4068-B46A-C4B6DCC823DC}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LibationWinForm", "LibationWinForm\LibationWinForm.csproj", "{635F00E1-AAD1-45F7-BEB7-D909AD33B9F6}"
@@ -81,9 +79,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2", "..\Lucene
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2.Tests", "..\LuceneNet303r2\LuceneNet303r2.Tests\LuceneNet303r2.Tests.csproj", "{5A7681A5-60D9-480B-9AC7-63E0812A2548}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DTOs", "DTOs\DTOs.csproj", "{5FDA62B1-55FD-407A-BECA-38A969235541}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DtoImporterService", "DtoImporterService\DtoImporterService.csproj", "{401865F5-1942-4713-B230-04544C0A97B0}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiDomainService", "AudibleApiDomainService\AudibleApiDomainService.csproj", "{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiDTOs", "..\audible api\AudibleApi\AudibleApiDTOs\AudibleApiDTOs.csproj", "{C03C5D65-3B7F-453B-972F-23950B7E0604}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiDTOs.Tests", "..\audible api\AudibleApi\_Tests\AudibleApiDTOs.Tests\AudibleApiDTOs.Tests.csproj", "{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiClientExample", "..\audible api\AudibleApi\_Demos\AudibleApiClientExample\AudibleApiClientExample.csproj", "{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -151,10 +153,6 @@ Global
 		{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Release|Any CPU.Build.0 = Release|Any CPU
-		{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}.Release|Any CPU.Build.0 = Release|Any CPU
 		{111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -199,14 +197,22 @@ Global
 		{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.Build.0 = Release|Any CPU
-		{5FDA62B1-55FD-407A-BECA-38A969235541}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{401865F5-1942-4713-B230-04544C0A97B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{5FDA62B1-55FD-407A-BECA-38A969235541}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{401865F5-1942-4713-B230-04544C0A97B0}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{5FDA62B1-55FD-407A-BECA-38A969235541}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{401865F5-1942-4713-B230-04544C0A97B0}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{5FDA62B1-55FD-407A-BECA-38A969235541}.Release|Any CPU.Build.0 = Release|Any CPU
+		{401865F5-1942-4713-B230-04544C0A97B0}.Release|Any CPU.Build.0 = Release|Any CPU
-		{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C03C5D65-3B7F-453B-972F-23950B7E0604}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C03C5D65-3B7F-453B-972F-23950B7E0604}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C03C5D65-3B7F-453B-972F-23950B7E0604}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C03C5D65-3B7F-453B-972F-23950B7E0604}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -227,7 +233,6 @@ Global
 		{FF12ADA0-8975-4E67-B6EA-4AC82E0C8994} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
 		{AD1FDDC9-8D2A-436A-8EED-91FD74E7C7B4} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
 		{7EA01F9C-E579-4B01-A3B9-733B49DD0B60} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
-		{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
 		{111420E2-D4F0-4068-B46A-C4B6DCC823DC} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
 		{635F00E1-AAD1-45F7-BEB7-D909AD33B9F6} = {8679CAC8-9164-4007-BDD2-F004810EDA14}
 		{0807616A-A77A-4B08-A65A-1582B09E114B} = {8679CAC8-9164-4007-BDD2-F004810EDA14}
@@ -239,8 +244,10 @@ Global
 		{1255D9BA-CE6E-42E4-A253-6376540B9661} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
 		{35803735-B669-4090-9681-CC7F7FABDC71} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
 		{5A7681A5-60D9-480B-9AC7-63E0812A2548} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
-		{5FDA62B1-55FD-407A-BECA-38A969235541} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
+		{401865F5-1942-4713-B230-04544C0A97B0} = {41CDCC73-9B81-49DD-9570-C54406E852AF}
-		{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050} = {41CDCC73-9B81-49DD-9570-C54406E852AF}
+		{C03C5D65-3B7F-453B-972F-23950B7E0604} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
+		{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
+		{282EEE16-F569-47E1-992F-C6DB8AEC7AA6} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9}

LibationWinForm/LibationWinForm.csproj

@@ -11,7 +11,7 @@
     <ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
     <ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj" />
     <ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj" />
-    <ProjectReference Include="..\AudibleApiDomainService\AudibleApiDomainService.csproj" />
+    <ProjectReference Include="..\DtoImporterService\DtoImporterService.csproj" />
     <ProjectReference Include="..\ScrapingDomainServices\ScrapingDomainServices.csproj" />
   </ItemGroup>
 

LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.cs

@@ -83,8 +83,8 @@ namespace LibationWinForm.BookLiberation
             // close form on DOWNLOAD completed, not final Completed. Else for BackupBook this form won't close until DECRYPT is also complete
             void fileDownloadCompleted(object _, string __) => downloadDialog.Close();
 
-            void downloadProgressChanged(object _, System.Net.DownloadProgressChangedEventArgs arg)
+            void downloadProgressChanged(object _, Dinah.Core.Net.Http.DownloadProgress progress)
-                => downloadDialog.DownloadProgressChanged(arg.BytesReceived, arg.TotalBytesToReceive);
+                => downloadDialog.DownloadProgressChanged(progress.BytesReceived, progress.TotalBytesToReceive.Value);
 
             void unsubscribe(object _ = null, EventArgs __ = null)
             {

LibationWinForm/UNTESTED/Dialogs/Login/WinformResponder.cs

@@ -3,7 +3,7 @@ using LibationWinForm.Dialogs.Login;
 
 namespace LibationWinForm.Login
 {
-	public class WinformResponder : AudibleApiDomainService.IAudibleApiResponder
+	public class WinformResponder : AudibleApi.ILoginCallback
 	{
 		public string Get2faCode()
 		{

LibationWinForm/UNTESTED/Form1.cs

@@ -357,15 +357,45 @@ namespace LibationWinForm
 
 		private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e)
 		{
-			// audible api
+// legacy/scraping method
-			var settings = new AudibleApiDomainService.Settings(config);
+			//await indexDialog(new ScanLibraryDialog());
-			var responder = new Login.WinformResponder();
+// new/api method
-			var client = await AudibleApiDomainService.AudibleApiLibationClient.CreateClientAsync(settings, responder);
+await audibleApi();
+		}
 
-await client.ImportLibraryAsync();
+		private async Task audibleApi()
+		{
+			var identityFilePath = System.IO.Path.Combine(config.LibationFiles, "IdentityTokens.json");
+			var callback = new Login.WinformResponder();
+			var api = await AudibleApi.EzApiCreator.GetApiAsync(identityFilePath, callback, config.LocaleCountryCode);
 
-			// scrape
+			int totalCount;
-			await indexDialog(new ScanLibraryDialog());
+			int newCount;
+
+			// seems to be very common the 1st time after long absence. either figure out why, or run 2x before declaring error
+			try
+			{
+				var items = await InternalUtilities.AudibleApiExtensions.GetAllLibraryItemsAsync(api);
+				totalCount = items.Count;
+				newCount = await Task.Run(() => new DtoImporterService.LibraryImporter().Import(items));
+			}
+			catch
+			{
+				try
+				{
+					var items = await InternalUtilities.AudibleApiExtensions.GetAllLibraryItemsAsync(api);
+					totalCount = items.Count;
+					newCount = await Task.Run(() => new DtoImporterService.LibraryImporter().Import(items));
+				}
+				catch (Exception ex)
+				{
+					MessageBox.Show("Error importing library.\r\n" + ex.Message, "Error importing library", MessageBoxButtons.OK, MessageBoxIcon.Error);
+					return;
+				}
+			}
+
+			await InternalUtilities.SearchEngineActions.FullReIndexAsync();
+			await indexComplete(totalCount, newCount);
 		}
 
 		private async void reimportMostRecentLibraryScanToolStripMenuItem_Click(object sender, EventArgs e)
@@ -377,15 +407,15 @@ await client.ImportLibraryAsync();
 
             MessageBox.Show($"Total processed: {TotalBooksProcessed}\r\nNew: {NewBooksAdded}");
 
-            await index(NewBooksAdded, TotalBooksProcessed);
+            await indexComplete(TotalBooksProcessed, NewBooksAdded);
         }
 
         private async Task indexDialog(IIndexLibraryDialog dialog)
         {
             if (!dialog.RunDialog().In(DialogResult.Abort, DialogResult.Cancel, DialogResult.None))
-                await index(dialog.NewBooksAdded, dialog.TotalBooksProcessed);
+                await indexComplete(dialog.TotalBooksProcessed, dialog.NewBooksAdded);
         }
-        private async Task index(int newBooksAdded, int totalBooksProcessed)
+        private async Task indexComplete(int totalBooksProcessed, int newBooksAdded)
         {
             // update backup counts if we have new library items
             if (newBooksAdded > 0)

LibationWinForm/UNTESTED/GridEntry.cs

@@ -26,6 +26,13 @@ namespace LibationWinForm
 		[Browsable(false)]
 		public IEnumerable<string> TagsEnumerated => book.UserDefinedItem.TagsEnumerated;
 
+		// formatReplacements is what gets displayed
+		// the value that gets returned from the property is the cell's value
+		// this allows for the value to be sorted one way and displayed another
+		// eg:
+		//   orig title: The Computer
+		//   formatReplacement: The Computer
+		//   value for sorting: Computer
 		private Dictionary<string, string> formatReplacements { get; } = new Dictionary<string, string>();
 		public bool TryGetFormatted(string key, out string value) => formatReplacements.TryGetValue(key, out value);
 
@@ -60,7 +67,7 @@ namespace LibationWinForm
 			{
 				formatReplacements[nameof(Length)]
 					= book.LengthInMinutes == 0
-                    ? "[pre-release]"
+					? ""
 					: $"{book.LengthInMinutes / 60} hr {book.LengthInMinutes % 60} min";
 
 				return book.LengthInMinutes;
@@ -69,10 +76,31 @@ namespace LibationWinForm
 
 		public string Series => book.SeriesNames;
 
+		private string descriptionCache = null;
 		public string Description
-            => book.Description == null ? ""
+		{
-            : book.Description.Length < 63 ? book.Description
+			get
-            : book.Description.Substring(0, 60) + "...";
+			{
+				// HtmlAgilityPack is expensive. cache results
+				if (descriptionCache is null)
+				{
+					if (book.Description is null)
+						descriptionCache = "";
+					else
+					{
+						var doc = new HtmlAgilityPack.HtmlDocument();
+						doc.LoadHtml(book.Description);
+						var noHtml = doc.DocumentNode.InnerText;
+						descriptionCache
+							= noHtml.Length < 63
+							? noHtml
+							: noHtml.Substring(0, 60) + "...";
+					}
+				}
+
+				return descriptionCache;
+			}
+		}
 
 		public string Category => string.Join(" > ", book.CategoriesNames);
 
@@ -85,25 +113,26 @@ namespace LibationWinForm
 		{
 			get
 			{
-                Rating rating = book.Rating;
+				formatReplacements[nameof(Product_Rating)] = starString(book.Rating);
-
+				return firstScore(book.Rating);
-                formatReplacements[nameof(Product_Rating)] = starString(rating);
-
-                return firstScore(rating);
 			}
 		}
 
-        public DateTime? Purchase_Date => libraryBook.DateAdded;
+		public string Purchase_Date
+		{
+			get
+			{
+				formatReplacements[nameof(Purchase_Date)] = libraryBook.DateAdded.ToString("d");
+				return libraryBook.DateAdded.ToString("yyyy-MM-dd HH:mm:ss");
+			}
+		}
 
         public string My_Rating
         {
             get
             {
-                Rating rating = book.UserDefinedItem.Rating;
+                formatReplacements[nameof(My_Rating)] = starString(book.UserDefinedItem.Rating);
-
+                return firstScore(book.UserDefinedItem.Rating);
-                formatReplacements[nameof(My_Rating)] = starString(rating);
-
-                return firstScore(rating);
             }
         }
 

LibationWinForm/UNTESTED/ProductsGrid.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using System.Windows.Forms;
 using Dinah.Core.DataBinding;
 using DataLayer;
+using Dinah.Core.Collections.Generic;
 
 namespace LibationWinForm
 {
@@ -76,11 +77,14 @@ namespace LibationWinForm
 
                 col.HeaderText = col.HeaderText.Replace("_", " ");
 
-                if (col.Name == nameof(GridEntry.Title))
+				col.Width = col.Name switch
-                    col.Width *= 2;
+				{
-
+					nameof(GridEntry.Cover) => 80,
-                if (col.Name == nameof(GridEntry.Misc))
+					nameof(GridEntry.Title) => col.Width * 2,
-                    col.Width = (int)(col.Width * 1.35);
+					nameof(GridEntry.Misc) => (int)(col.Width * 1.35),
+					var n when n.In(nameof(GridEntry.My_Rating), nameof(GridEntry.Product_Rating)) => col.Width + 8,
+					_ => col.Width
+				};
             }
 
 
@@ -90,10 +94,12 @@ namespace LibationWinForm
             var lib = LibraryQueries.GetLibrary_Flat_NoTracking();
 			var orderedGridEntries = lib
                 .Select(lb => new GridEntry(lb)).ToList()
-                // default load order: sort by author, then series, then title
+				// default load order
-                .OrderBy(ge => ge.Authors)
+				.OrderByDescending(ge => ge.Purchase_Date)
-                    .ThenBy(ge => ge.Series)
+                //// more advanced example: sort by author, then series, then title
-                    .ThenBy(ge => ge.Title)
+                //.OrderBy(ge => ge.Authors)
+                //    .ThenBy(ge => ge.Series)
+                //    .ThenBy(ge => ge.Title)
                 .ToList();
 
             //

Scraping/Scraping.csproj

@@ -6,7 +6,6 @@
 
   <ItemGroup>
     <ProjectReference Include="..\AudibleDotCom\AudibleDotCom.csproj" />
-    <ProjectReference Include="..\DTOs\DTOs.csproj" />
   </ItemGroup>
 
 </Project>

ScrapingDomainServices/UNTESTED/DownloadBook.cs

@@ -38,16 +38,10 @@ namespace ScrapingDomainServices
 			// in cases where title includes '&', just use everything before the '&' and ignore the rest
 			//// var adhTitle = product.Title.Split('&')[0]
 
-            var aaxDownloadLink = libraryBook.DownloadBookLink
+// legacy/scraping method
-                .Replace("/admhelper", "")
+			//await performDownloadAsync(libraryBook, tempAaxFilename);
-                .Replace("&DownloadType=Now", "")
+// new/api method
-                + "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download";
+tempAaxFilename = await performApiDownloadAsync(libraryBook, tempAaxFilename);
-            var uri = new Uri(aaxDownloadLink);
-
-			using var webClient = await GetWebClient(tempAaxFilename);
-            // for book downloads only: pretend to be the audible download manager. from inAudible:
-            webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601";
-            await webClient.DownloadFileTaskAsync(uri, tempAaxFilename);
 
 			// move
 			var aaxFilename = FileUtility.GetValidFilename(
@@ -65,5 +59,41 @@ namespace ScrapingDomainServices
 				statusHandler.AddError("Downloaded AAX file cannot be found");
 			return statusHandler;
 		}
+
+		// GetWebClientAsync:
+		//   wires up webClient events
+		//     [DownloadProgressChanged, DownloadFileCompleted, DownloadDataCompleted, DownloadStringCompleted]
+		//   to DownloadableBase events
+		//     DownloadProgressChanged, DownloadCompleted
+		//   fires DownloadBegin event
+		// method begins async file download
+		private async Task performDownloadAsync(LibraryBook libraryBook, string tempAaxFilename)
+		{
+			var aaxDownloadLink = libraryBook.DownloadBookLink
+				.Replace("/admhelper", "")
+				.Replace("&DownloadType=Now", "")
+				+ "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download";
+			var uri = new Uri(aaxDownloadLink);
+
+			using var webClient = await GetWebClientAsync(tempAaxFilename);
+			// for book downloads only: pretend to be the audible download manager. from inAudible:
+			webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601";
+			await webClient.DownloadFileTaskAsync(uri, tempAaxFilename);
+		}
+
+		private async Task<string> performApiDownloadAsync(LibraryBook libraryBook, string tempAaxFilename)
+		{
+			var identityFilePath = Path.Combine(Configuration.Instance.LibationFiles, "IdentityTokens.json");
+			var api = await AudibleApi.EzApiCreator.GetApiAsync(identityFilePath);
+
+			var progress = new Progress<Dinah.Core.Net.Http.DownloadProgress>();
+			progress.ProgressChanged += (_, e) => Invoke_DownloadProgressChanged(this, e);
+
+			Invoke_DownloadBegin(tempAaxFilename);
+			var actualFilePath = await api.DownloadAaxWorkaroundAsync(libraryBook.Book.AudibleProductId, tempAaxFilename, progress);
+			Invoke_DownloadCompleted(this, $"Completed: {actualFilePath}");
+
+			return actualFilePath;
+		}
 	}
 }

ScrapingDomainServices/UNTESTED/DownloadPdf.cs

@@ -43,7 +43,7 @@ namespace ScrapingDomainServices
 
             var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url));
 
-			using var webClient = await GetWebClient(destinationFilename);
+			using var webClient = await GetWebClientAsync(destinationFilename);
             await webClient.DownloadFileTaskAsync(url, destinationFilename);
 
             var statusHandler = new StatusHandler();

ScrapingDomainServices/UNTESTED/DownloadableBase.cs

@@ -15,9 +15,14 @@ namespace ScrapingDomainServices
 		protected void DoStatusUpdate(string message) => StatusUpdate?.Invoke(this, message);
 
 		public event EventHandler<string> DownloadBegin;
-        public event DownloadProgressChangedEventHandler DownloadProgressChanged;
+		public event EventHandler<Dinah.Core.Net.Http.DownloadProgress> DownloadProgressChanged;
 		public event EventHandler<string> DownloadCompleted;
 
+		protected void Invoke_DownloadBegin(string downloadMessage) => DownloadBegin?.Invoke(this, downloadMessage);
+		protected void Invoke_DownloadProgressChanged(object sender, Dinah.Core.Net.Http.DownloadProgress progress) => DownloadProgressChanged?.Invoke(sender, progress);
+		protected void Invoke_DownloadCompleted(object sender, string str) => DownloadCompleted?.Invoke(sender, str);
+
+
 		public event EventHandler<string> Completed;
 
         static DownloadableBase()
@@ -55,12 +60,13 @@ namespace ScrapingDomainServices
             "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
             "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
 			"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36",
+			"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36",
 		};
         // we need a minimum delay between tries when hitting audible.com
         // in every case except decrypt (which is already long running), we hit audible.com
         static Humanizer humanizer { get; } = new Humanizer { Minimum = 5, Maximum = 20 };
-        static Random rnd = new Random();
+        static Random rnd { get; } = new Random();
-        protected async Task<WebClient> GetWebClient(string downloadMessage)
+        protected async Task<WebClient> GetWebClientAsync(string downloadMessage)
         {
             await humanizer.Wait();
 
@@ -79,12 +85,12 @@ namespace ScrapingDomainServices
 			// weirdly, it works for book download even though it uses https://cds.audible.com
 			//webClient.Headers["Host"] = "www.audible.com";
 
-            webClient.DownloadProgressChanged += (s, e) => DownloadProgressChanged?.Invoke(s, e);
+			webClient.DownloadProgressChanged += (s, e) => Invoke_DownloadProgressChanged(s, new Dinah.Core.Net.Http.DownloadProgress { BytesReceived = e.BytesReceived, ProgressPercentage = e.ProgressPercentage, TotalBytesToReceive = e.TotalBytesToReceive });
-            webClient.DownloadFileCompleted += (s, e) => DownloadCompleted?.Invoke(s, $"Completed: {downloadMessage}");
+            webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
-            webClient.DownloadDataCompleted += (s, e) => DownloadCompleted?.Invoke(s, $"Completed: {downloadMessage}");
+            webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
-            webClient.DownloadStringCompleted += (s, e) => DownloadCompleted?.Invoke(s, $"Completed: {downloadMessage}");
+            webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
 
-            DownloadBegin?.Invoke(this, downloadMessage);
+			Invoke_DownloadBegin(downloadMessage);
 
             return webClient;
         }

ScrapingDomainServices/UNTESTED/DtoImporter.cs

@@ -93,7 +93,7 @@ namespace ScrapingDomainServices
 
                 // if no narrators listed, author is the narrator
                 if (!libraryDTO.Narrators.Any())
-                    libraryDTO.Narrators = libraryDTO.Narrators = authors.Select(a => a.Name).ToArray();
+                    libraryDTO.Narrators = authors.Select(a => a.Name).ToArray();
 
                 // nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
                 var narrators = libraryDTO
@@ -102,15 +102,16 @@ namespace ScrapingDomainServices
                     .ToList();
 
                 book = context.Books.Add(new Book(
-                    new AudibleProductId(libraryDTO.ProductId), libraryDTO.Title, libraryDTO.Description, libraryDTO.LengthInMinutes, authors, narrators))
+                    new AudibleProductId(libraryDTO.ProductId), libraryDTO.Title, libraryDTO.Description, libraryDTO.LengthInMinutes, authors))
                     .Entity;
+				book.ReplaceNarrators(narrators);
             }
 
             // set/update book-specific info which may have changed
             book.PictureId = libraryDTO.PictureId;
             book.UpdateProductRating(libraryDTO.Product_OverallStars, libraryDTO.Product_PerformanceStars, libraryDTO.Product_StoryStars);
 			foreach (var url in libraryDTO.SupplementUrls)
-                book.AddSupplementDownloadUrl(url);
+				book.AddSupplementDownloadUrl(FileManager.FileUtility.RestoreDeclawed(url));
 
 			// important to update user-specific info. this will have changed if user has rated/reviewed the book since last library import
 			book.UserDefinedItem.UpdateRating(libraryDTO.MyUserRating_Overall, libraryDTO.MyUserRating_Performance, libraryDTO.MyUserRating_Story);

ScrapingDomainServices/UNTESTED/IDownloadable.cs

@@ -1,12 +1,11 @@
 using System;
-using System.Net;
 
 namespace ScrapingDomainServices
 {
     public interface IDownloadable : IProcessable
     {
         event EventHandler<string> DownloadBegin;
-        event DownloadProgressChangedEventHandler DownloadProgressChanged;
+        event EventHandler<Dinah.Core.Net.Http.DownloadProgress> DownloadProgressChanged;
         event EventHandler<string> DownloadCompleted;
     }
 }

ScrapingDomainServices/UNTESTED/IProcessableExt.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Threading.Tasks;
 using DataLayer;
+using Dinah.Core.Collections.Generic;
 using Dinah.Core.ErrorHandling;
 
 namespace ScrapingDomainServices
@@ -37,7 +38,11 @@ namespace ScrapingDomainServices
             var libraryBooks = LibraryQueries.GetLibrary_Flat_NoTracking();
 
             foreach (var libraryBook in libraryBooks)
-                if (await processable.ValidateAsync(libraryBook))
+                if (
+// hardcoded blacklist
+//episodes
+!libraryBook.Book.AudibleProductId.In("B079ZTTL4J", "B0779LK1TX", "B0779H7B38", "B0779M3KGC", "B076PQ6G9Z", "B07D4M18YC") &&
+					await processable.ValidateAsync(libraryBook))
 						return libraryBook;
 
             return null;

ScrapingDomainServices/UNTESTED/Indexer.cs

@@ -26,7 +26,8 @@ namespace ScrapingDomainServices
         public static async Task<(int total, int newEntries)> IndexLibraryAsync(List<FileInfo> jsonFileInfos)
         {
             var productItems = jsonFileInfos.SelectMany(fi => json2libraryDtos(fi)).ToList();
-            return await IndexLibraryAsync(productItems);
+            var newEntries = await IndexLibraryAsync(productItems);
+            return (productItems.Count, newEntries);
         }
         private static Regex jsonIsCollectionRegex = new Regex(@"^\s*\[\s*\{", RegexOptions.Compiled);
         private static IEnumerable<LibraryDTO> json2libraryDtos(FileInfo jsonFileInfo)
@@ -44,13 +45,11 @@ namespace ScrapingDomainServices
         }
 
         // new full index or library-file import: re-create search index
-        public static async Task<(int total, int newEntries)> IndexLibraryAsync(List<LibraryDTO> productItems)
+        /// <returns>qty new entries</returns>
-            => await IndexLibraryAsync(productItems, SearchEngineActions.FullReIndexAsync);
+		public static async Task<int> IndexLibraryAsync(List<LibraryDTO> productItems)
-
-        private static async Task<(int total, int newEntries)> IndexLibraryAsync(List<LibraryDTO> productItems, Func<Task> postIndexActionAsync)
         {
             if (productItems == null || !productItems.Any())
-                return (0, 0);
+                return 0;
 
 			productItems = filterAndValidate(productItems);
 
@@ -74,9 +73,9 @@ namespace ScrapingDomainServices
             await Task.Run(() => dtoImporter.ReloadBookDetails(productItems));
             await context.SaveChangesAsync();
 
-            await postIndexActionAsync?.Invoke();
+            await SearchEngineActions.FullReIndexAsync();
 
-            return (productItems.Count, newEntries);
+            return newEntries;
         }
         private static List<LibraryDTO> filterAndValidate(List<LibraryDTO> collection)
         {
@@ -121,19 +120,6 @@ namespace ScrapingDomainServices
         #endregion
 
         #region book details
-        public static async Task IndexBookDetailsAsync(FileInfo jsonFileInfo)
-        {
-            var bookDetailDTO = json2bookDetailDto(jsonFileInfo);
-            await IndexBookDetailsAsync(bookDetailDTO);
-        }
-        private static BookDetailDTO json2bookDetailDto(FileInfo jsonFileInfo)
-        {
-            validateJsonFile(jsonFileInfo);
-
-            var serialized = File.ReadAllText(jsonFileInfo.FullName);
-            return JsonConvert.DeserializeObject<BookDetailDTO>(serialized);
-        }
-
         public static async Task IndexBookDetailsAsync(BookDetailDTO bookDetailDTO)
             => await indexBookDetailsAsync(bookDetailDTO, () => SearchEngineActions.ProductReIndexAsync(bookDetailDTO.ProductId));
 

ScrapingDomainServices/UNTESTED/ScrapeBookDetails.cs

@@ -72,7 +72,7 @@ namespace ScrapingDomainServices
                     // download htm
                     string source;
                     var url = AudiblePage.Product.GetUrl(productId);
-					using var webClient = await GetWebClient($"Getting Book Details for {libraryBook.Book.Title}");
+					using var webClient = await GetWebClientAsync($"Getting Book Details for {libraryBook.Book.Title}");
 					try
 					{
 						source = await webClient.DownloadStringTaskAsync(url);

__TODO.txt

@@ -1,4 +1,121 @@
--- begin CONFIG FILES ---------------------------------------------------------------------------------------------------------------------
+-- begin REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
+integrate API into libation. replace all authentication, audible communication
+
+IN-PROGRESS
+-----------
+download via api with UI integration
+- functionality: episodes
+library import UI
+- disable main ui
+- updates on which stage and how long it's expected to take
+- error handling
+- dialog/pop up when done. show how many new and total
+move biz logic out of UI (Form1.scanLibraryToolStripMenuItem_Click)
+- non-db dependent: InternalUtilities. see: SearchEngineActions
+  InternalUtilities.AudibleApiExtensions may not belong here. not sure
+- db dependent: eg DtoImporterService
+
+extract IdentityTokens.json into FileManager
+replace all hardcoded occurances
+
+FIX
+// hardcoded blacklist
+
+datalayer stuff (eg: Book) need better ToString
+
+MOVE TO LEGACY
+--------------
+- scraping code
+- "legacy inAudible wire-up code"
+Book.IsEpisodes
+Book.HasBookDetails
+LibraryBook.DownloadBookLink
+humanizer
+replace all scraping with audible api
+    public partial class ScanLibraryDialog : Form, IIndexLibraryDialog
+        public async Task DoMainWorkAsync()
+            using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
+            jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
+scraping stuff to remove. order matters. go from top to bottom:
+REMOVED PROJECT REFERENCES
+3.2 Domain Utilities (post database)
+    DomainServices
+        AudibleDotComAutomation
+3.1 Domain Internal Utilities
+    InternalUtilities
+        Scraping
+REMOVED PROJECTS
+2 Utilities (domain ignorant)
+  Scraping
+  AudibleDotComAutomation
+  AudibleDotCom
+  CookieMonster
+REMOVED FILES
+\Libation\InternalUtilities\UNTESTED\DataConverter.cs
+\Libation\DomainServices\UNTESTED\DownloadLibrary.cs
+\Libation\DomainServices\UNTESTED\ScrapeBookDetails.cs
+ADDED PROJECT REFERENCES
+3.1 Domain Internal Utilities
+    InternalUtilities
+        DTOs
+-- end REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
+
+-- begin ENHANCEMENT, PERFORMANCE: IMPORT ---------------------------------------------------------------------------------------------------------------------
+imports are PAINFULLY slow for just a few hundred items. wtf is taking so long?
+-- end ENHANCEMENT, PERFORMANCE: IMPORT ---------------------------------------------------------------------------------------------------------------------
+
+-- begin ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
+when a book/pdf is NOT liberated, calculating the grid's [Liberated][NOT d/l'ed] label is very slow
+https://stackoverflow.com/a/12046333
+https://codereview.stackexchange.com/a/135074
+// do NOT use lock() or Monitor with async/await
+private static int _lockFlag = 0; // 0 - free
+if (Interlocked.CompareExchange(ref _lockFlag, 1, 0) != 0) return;
+// only 1 thread will enter here without locking the object/put the other threads to sleep
+try { await DoWorkAsync(); }
+// free the lock
+finally { Interlocked.Decrement(ref _lockFlag); }
+
+use stop light icons for liberated state: red=none, yellow=downloaded encrypted, green=liberated
+
+need a way to liberate ad hoc books and pdf.s
+
+use pdf icon with and without and X over it to indicate status
+-- end ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
+
+-- begin ENHANCEMENTS, GET LIBRARY ---------------------------------------------------------------------------------------------------------------------
+Audible API. GET /1.0/library , GET /1.0/library/{asin}
+TONS of expensive conversion: GetLibraryAsync > string > JObject > string > LibraryApiV10
+-- end ENHANCEMENTS, GET LIBRARY ---------------------------------------------------------------------------------------------------------------------
+
+-- begin ENHANCEMENTS, EPISODES ---------------------------------------------------------------------------------------------------------------------
+grid: episodes need a better Download_Status and Length/LengthInMinutes
+download: need a way to liberate episodes. show in grid 'x/y downloaded/liberated' etc
+-- end ENHANCEMENTS, EPISODES ---------------------------------------------------------------------------------------------------------------------
+
+-- begin BUG, MOVING FILES ---------------------------------------------------------------------------------------------------------------------
+with libation closed, move files
+start libation
+can get error below
+fixed on restart
+
+Form1_Load ... await setBackupCountsAsync();
+Collection was modified; enumeration operation may not execute.
+stack trace
+   at System.ThrowHelper.ThrowInvalidOperationException_InvalidOperation_EnumFailedVersion()
+   at System.Collections.Generic.List`1.Enumerator.MoveNextRare()
+   at System.Linq.Enumerable.SingleOrDefault[TSource](IEnumerable`1 source, Func`2 predicate)
+   at FileManager.FilePathCache.GetPath(String id, FileType type) in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\FileManager\UNTESTED\FilePathCache.cs:line 33
+   at FileManager.AudibleFileStorage.<getAsync>d__32.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\FileManager\UNTESTED\AudibleFileStorage.cs:line 112
+   at FileManager.AudibleFileStorage.<GetAsync>d__31.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\FileManager\UNTESTED\AudibleFileStorage.cs:line 107
+   at FileManager.AudibleFileStorage.<ExistsAsync>d__30.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\FileManager\UNTESTED\AudibleFileStorage.cs:line 104
+   at LibationWinForm.Form1.<<setBookBackupCountsAsync>g__getAudioFileStateAsync|15_1>d.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\LibationWinForm\UNTESTED\Form1.cs:line 110
+   at LibationWinForm.Form1.<setBookBackupCountsAsync>d__15.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\LibationWinForm\UNTESTED\Form1.cs:line 117
+   at LibationWinForm.Form1.<setBackupCountsAsync>d__13.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\LibationWinForm\UNTESTED\Form1.cs:line 81
+   at LibationWinForm.Form1.<Form1_Load>d__11.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\LibationWinForm\UNTESTED\Form1.cs:line 60
+-- end BUG, MOVING FILES ---------------------------------------------------------------------------------------------------------------------
+
+-- begin CONFIG FILES ---------------------------------------------------------------------------------------------------------------------
 .\appsettings.json should only be a pointer to the real settings file location: LibationSettings.json
 replace complex config saving throughout with new way in my ConsoleDependencyInjection solution
 all settings should be strongly typed
@@ -16,68 +133,21 @@ BasePath => recursively search directories upward-only until fild dir with .sln
 from here can set up a shared dir anywhere. use recursive upward search to find it. store shared files here. eg: identityTokens.json, ComputedTestValues
 -- end CONFIG FILES ---------------------------------------------------------------------------------------------------------------------
 
--- begin REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
+-- begin TAGS ---------------------------------------------------------------------------------------------------------------------
-integrate API into libation. eventually entirely replace all other authentication methods, audible communication. incl book download
+pulling previous tags into new Books. think: reloading db
-logged-in user's own ratings (aka rating-star(s)) -- is "provided_review"
+move out of Book and into DtoMapper?
-unlike traditional scraping, audible api calls do not need to be humanized. can likely delete humanizer altogether. still needed for downloading pdf.s?
 
-incl episodes. eg: "Bill Bryson's Appliance of Science"
+Extract file and tag stuff from domain objects. This should exist only in data layer. If domain objects are able to call EF context, it should go through data layer
-  refining episode retrieval might also get rid of the need for IsEpisodes property
+Why are tags in file AND database?
-replace all scraping with audible api
-    public partial class ScanLibraryDialog : Form, IIndexLibraryDialog
-        public async Task DoMainWorkAsync()
-            using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
-            jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
 
-move old DTOs back into scraping so it's easier to move them all to new legacy area
+extract FileManager dependency from data layer
-library to return strongly typed LibraryApiV10
+-- end TAGS ---------------------------------------------------------------------------------------------------------------------
-
-note: some json importing may still be relevant. might want to allow custom importing
-
-scraping stuff to remove:
-order matters. go from top to bottom:
-
-REMOVED PROJECT REFERENCES
-3.2 Domain Utilities (post database)
-    DomainServices
-        AudibleDotComAutomation
-3.1 Domain Internal Utilities
-    InternalUtilities
-        Scraping
-
-REMOVED PROJECTS
-2 Utilities (domain ignorant)
-  Scraping
-  AudibleDotComAutomation
-  AudibleDotCom
-  CookieMonster
-
-REMOVED FILES
-\Libation\InternalUtilities\UNTESTED\DataConverter.cs
-\Libation\DomainServices\UNTESTED\DownloadLibrary.cs
-\Libation\DomainServices\UNTESTED\ScrapeBookDetails.cs
-
-ADDED PROJECT REFERENCES
-3.1 Domain Internal Utilities
-    InternalUtilities
-        DTOs
-
-Audible API. GET /1.0/library , GET /1.0/library/{asin}
-TONS of expensive conversion: GetLibraryAsync > string > JObject > string > LibraryApiV10
--- end REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
 
 -- begin ENHANCEMENT, CATEGORIES ---------------------------------------------------------------------------------------------------------------------
 add support for multiple categories
 when i do this, learn about the different CategoryLadder.Root enums. probably only need Root.Genres
 -- end ENHANCEMENT, CATEGORIES ---------------------------------------------------------------------------------------------------------------------
 
--- begin LEGACY ---------------------------------------------------------------------------------------------------------------------
-retain legacy functionality in out-of-the-way area. turn on/off in settings?
-
-> scraping code
-> "legacy inAudible wire-up code"
--- end LEGACY ---------------------------------------------------------------------------------------------------------------------
-
 -- begin CLEAN UP ARCHITECTURE ---------------------------------------------------------------------------------------------------------------------
 my ui sucks. it's also tightly coupled with biz logic. can't replace ui until biz logic is extracted and loosely. remove all biz logic from presentation/winforms layer
 -- end CLEAN UP ARCHITECTURE ---------------------------------------------------------------------------------------------------------------------
@@ -98,6 +168,9 @@ Turn into unit tests or demos
   search 'example code' on: LibationWinForm\...\Form1.cs
   EnumerationFlagsExtensions.EXAMPLES()
   // examples
+  scratchpad
+  scratch pad
+  scratch_pad
 -- end UNIT TESTS ---------------------------------------------------------------------------------------------------------------------
 
 -- begin DECRYPTING ---------------------------------------------------------------------------------------------------------------------
@@ -128,6 +201,7 @@ directly call ffmpeg (decrypt only)
 
 -- begin ENHANCEMENT, UI: LONG RUNNING TASKS ---------------------------------------------------------------------------------------------------------------------
 long running tasks are appropriately async. however there's no way for the user to see that the task is running (vs nothing happened) except to wait and see if the final notification ever comes
+need events to update UI with progress
 -- end ENHANCEMENT, UI: LONG RUNNING TASKS ---------------------------------------------------------------------------------------------------------------------
 
 -- begin ENHANCEMENT, PERFORMANCE: TAGS ---------------------------------------------------------------------------------------------------------------------
@@ -136,10 +210,6 @@ unlikely to be an issue with file write. in fact, should probably roll back this
 also touches parts of code which: db write via a hook, search engine re-index
 -- end ENHANCEMENT, PERFORMANCE: TAGS ---------------------------------------------------------------------------------------------------------------------
 
--- begin ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
-when a book/pdf is NOT liberated, calculating the grid's [Liberated][NOT d/l'ed] label is very slow
--- end ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
-
 -- begin ENHANCEMENT: REMOVE BOOK ---------------------------------------------------------------------------------------------------------------------
 how to remove a book?
 previously difficult due to implementation details regarding scraping and importing. should be trivial after api replaces scraping