tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scraping => api transition almost complete

Browse Source
This commit is contained in:
Robert McRackan 2019-11-04 14:16:57 -05:00
parent df889a60a4
commit 591d84e719
41 changed files with 995 additions and 2494 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
AudibleApiDomainService/AudibleApiDomainService.csproj
View File

@ -1,13 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
<ProjectReference Include="..\FileManager\FileManager.csproj" />
<ProjectReference Include="..\ScrapingDomainServices\ScrapingDomainServices.csproj" />
</ItemGroup>
</Project>

151
AudibleApiDomainService/AudibleApiLibationClient.cs
View File

@ -1,151 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using AudibleApi;
using AudibleApi.Authentication;
using AudibleApi.Authorization;
using DTOs;
namespace AudibleApiDomainService
{
public class AudibleApiLibationClient
{
private Api _api;
#region initialize api
private AudibleApiLibationClient() { }
public async static Task<AudibleApiLibationClient> CreateClientAsync(Settings settings, IAudibleApiResponder responder)
{
Localization.SetLocale(settings.LocaleCountryCode);
Api api;
try
{
api = await EzApiCreator.GetApiAsync(settings.IdentityFilePath);
}
catch
{
var inMemoryIdentity = await loginAsync(responder);
api = await EzApiCreator.GetApiAsync(settings.IdentityFilePath, inMemoryIdentity);
}
return new AudibleApiLibationClient { _api = api };
}
// LOGIN PATTERN
// - Start with Authenticate. Submit email + pw
// - Each step in the login process will return a LoginResult
// - Each result which has required user input has a SubmitAsync method
// - The final LoginComplete result returns "Identity" -- in-memory authorization items
private static async Task<IIdentity> loginAsync(IAudibleApiResponder responder)
{
var login = new Authenticate();
var (email, password) = responder.GetLogin();
var loginResult = await login.SubmitCredentialsAsync(email, password);
while (true)
{
switch (loginResult)
{
case CredentialsPage credentialsPage:
var (emailInput, pwInput) = responder.GetLogin();
loginResult = await credentialsPage.SubmitAsync(emailInput, pwInput);
break;
case CaptchaPage captchaResult:
var imageBytes = await downloadImageAsync(captchaResult.CaptchaImage);
var guess = responder.GetCaptchaAnswer(imageBytes);
loginResult = await captchaResult.SubmitAsync(guess);
break;
case TwoFactorAuthenticationPage _2fa:
var _2faCode = responder.Get2faCode();
loginResult = await _2fa.SubmitAsync(_2faCode);
break;
case LoginComplete final:
return final.Identity;
default:
throw new Exception("Unknown LoginResult");
}
}
}
private static async Task<byte[]> downloadImageAsync(Uri imageUri)
{
using var client = new HttpClient();
using var contentStream = await client.GetStreamAsync(imageUri);
using var localStream = new MemoryStream();
await contentStream.CopyToAsync(localStream);
return localStream.ToArray();
}
#endregion
public async Task ImportLibraryAsync()
{
try
{
var items = await GetLibraryItemsAsync();
//var (total, newEntries) = await ScrapingDomainServices.Indexer.IndexLibraryAsync(items);
}
catch (Exception ex)
{
// catch here for easier debugging
throw;
}
}
private async Task<List<Item>> GetLibraryItemsAsync()
{
var allItems = new List<Item>();
for (var i = 1; ; i++)
{
var page = await _api.GetLibraryAsync(new LibraryOptions
{
NumberOfResultPerPage = 1000,
PageNumber = i,
PurchasedAfter = new DateTime(2000, 1, 1),
ResponseGroups = LibraryOptions.ResponseGroupOptions.ALL_OPTIONS
});
// important! use this convert method
var libResult = LibraryApiV10.FromJson(page.ToString());
if (!libResult.Items.Any())
break;
allItems.AddRange(libResult.Items);
}
return allItems;
}
//public async Task DownloadBookAsync(string asinToDownload)
//{
// // console example
// using var progressBar = new Dinah.Core.ConsoleLib.ProgressBar();
// var progress = new Progress<Dinah.Core.Net.Http.DownloadProgress>();
// progress.ProgressChanged += (_, e) => progressBar.Report(Math.Round((double)(100 * e.BytesReceived) / e.TotalFileSize.Value) / 100);
// logger.WriteLine("Download book");
// var finalFile = await _api.DownloadAaxWorkaroundAsync(asinToDownload, "downloadExample.xyz", progress);
// logger.WriteLine(" Done!");
// logger.WriteLine("final file: " + Path.GetFullPath(finalFile));
// // benefit of this small delay:
// // - if you try to delete a file too soon after it's created, the OS isn't done with the creation and you can get an unexpected error
// // - give progressBar's internal timer time to finish. if timer is disposed before the final message is processed, "100%" will never get a chance to be displayed
// await Task.Delay(100);
// File.Delete(finalFile);
//}
}
}

9
AudibleApiDomainService/IAudibleApiResponder.cs
View File

@ -1,9 +0,0 @@
namespace AudibleApiDomainService
{
public interface IAudibleApiResponder
{
(string email, string password) GetLogin();
string GetCaptchaAnswer(byte[] captchaImage);
string Get2faCode();
}
}

17
AudibleApiDomainService/Settings.cs
View File

@ -1,17 +0,0 @@
using System.IO;
using FileManager;
namespace AudibleApiDomainService
{
public class Settings
{
public string IdentityFilePath { get; }
public string LocaleCountryCode { get; }
public Settings(Configuration config)
{
IdentityFilePath = Path.Combine(config.LibationFiles, "IdentityTokens.json");
LocaleCountryCode = config.LocaleCountryCode;
}
}
}

2
CookieMonster/CookieMonster.csproj
View File

@ -5,7 +5,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="System.Data.SQLite.Core" Version="1.0.111" /> <PackageReference Include="System.Data.SQLite.Core" Version="1.0.112" />
<PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.6.0" /> <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.6.0" />
</ItemGroup> </ItemGroup>

11
DTOs/DTOs.csproj
View File

@ -1,11 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Dinah.Core\Dinah.Core\Dinah.Core.csproj" />
</ItemGroup>
</Project>

1829
DTOs/LibraryApiV10.cs
View File

File diff suppressed because it is too large Load Diff

148
DTOs/LibraryApiV10.custom.cs
View File

@ -1,148 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Dinah.Core.Collections.Generic;
namespace DTOs
{
public partial class LibraryApiV10
{
public IEnumerable<Person> AuthorsDistinct => Items.GetAuthorsDistinct();
public IEnumerable<string> NarratorsDistinct => Items.GetNarratorsDistinct();
public IEnumerable<string> PublishersDistinct => Items.GetPublishersDistinct();
public IEnumerable<Series> SeriesDistinct => Items.GetSeriesDistinct();
public IEnumerable<Ladder> ParentCategoriesDistinct => Items.GetParentCategoriesDistinct();
public IEnumerable<Ladder> ChildCategoriesDistinct => Items.GetChildCategoriesDistinct();
public override string ToString() => $"{Items.Length} {nameof(Items)}, {ResponseGroups.Length} {nameof(ResponseGroups)}";
}
public partial class Item
{
public string ProductId => Asin;
public int LengthInMinutes => RuntimeLengthMin ?? 0;
public string Description => PublisherSummary;
public bool Episodes
=> Relationships
?.Where(r => r.RelationshipToProduct == RelationshipToProduct.Child && r.RelationshipType == RelationshipType.Episode)
.Any()
?? false;
public string PictureId => ProductImages?.PictureId;
public string SupplementUrls => PdfUrl.AbsoluteUri; // item.PdfUrl == item.PdfLink
public DateTime DateAdded => PurchaseDate.UtcDateTime;
public float Product_OverallStars => Convert.ToSingle(Rating?.OverallDistribution.DisplayStars ?? 0);
public float Product_PerformanceStars => Convert.ToSingle(Rating?.PerformanceDistribution.DisplayStars ?? 0);
public float Product_StoryStars => Convert.ToSingle(Rating?.StoryDistribution.DisplayStars ?? 0);
public int MyUserRating_Overall => Convert.ToInt32(ProvidedReview?.Ratings.OverallRating ?? 0L);
public int MyUserRating_Performance => Convert.ToInt32(ProvidedReview?.Ratings.PerformanceRating ?? 0L);
public int MyUserRating_Story => Convert.ToInt32(ProvidedReview?.Ratings.StoryRating ?? 0L);
public bool IsAbridged
=> FormatType.HasValue
? FormatType == DTOs.FormatType.Abridged
: false;
public DateTime? DatePublished => IssueDate?.UtcDateTime; // item.IssueDate == item.ReleaseDate
public string Publisher => PublisherName;
// these category properties assume:
// - we're only exposing 1 category, irrespective of how many the Item actually has
// - each ladder will have either 1 or 2 levels: parent and optional child
public Ladder[] Categories => CategoryLadders?.FirstOrDefault()?.Ladder ?? new Ladder[0];
public Ladder ParentCategory => Categories[0];
public Ladder ChildCategory => Categories.Length > 1 ? Categories[1] : null;
// LibraryDTO.DownloadBookLink will be handled differently. see api.DownloadAaxWorkaroundAsync(asin)
public IEnumerable<Person> AuthorsDistinct => Authors.DistinctBy(a => new { a.Name, a.Asin });
public IEnumerable<Person> NarratorsDistinct => Narrators.DistinctBy(a => new { a.Name, a.Asin });
public override string ToString() => $"[{ProductId}] {Title}";
}
public partial class Person
{
public override string ToString() => $"{Name}";
}
public partial class AvailableCodec
{
public override string ToString() => $"{Name} {Format} {EnhancedCodec}";
}
public partial class CategoryLadder
{
public override string ToString() => Ladder.Select(l => l.CategoryName).Aggregate((a, b) => $"{a} | {b}");
}
public partial class Ladder
{
public string CategoryId => Id;
public string CategoryName => Name;
public override string ToString() => $"[{CategoryId}] {CategoryName}";
}
public partial class ContentRating
{
public override string ToString() => $"{Steaminess}";
}
public partial class Review
{
public override string ToString() => $"{this.Title}";
}
public partial class GuidedResponse
{
//public override string ToString() =>
}
public partial class Ratings
{
public override string ToString() => $"{OverallRating:0.0}|{PerformanceRating:0.0}|{StoryRating:0.0}";
}
public partial class ReviewContentScores
{
public override string ToString() => $"Helpful={NumHelpfulVotes}, Unhelpful={NumUnhelpfulVotes}";
}
public partial class Plan
{
public override string ToString() => $"{PlanName}";
}
public partial class Price
{
public override string ToString() => $"List={ListPrice}, Lowest={LowestPrice}";
}
public partial class ListPriceClass
{
public override string ToString() => $"{Base}";
}
public partial class ProductImages
{
public string PictureId
=> The500
.AbsoluteUri // https://m.media-amazon.com/images/I/51T1NWIkR4L._SL500_.jpg?foo=bar
?.Split('/').Last() // 51T1NWIkR4L._SL500_.jpg?foo=bar
?.Split('.').First() // 51T1NWIkR4L
;
public override string ToString() => $"{The500}";
}
public partial class Rating
{
public override string ToString() => $"{OverallDistribution}|{PerformanceDistribution}|{StoryDistribution}";
}
public partial class Distribution
{
public override string ToString() => $"{DisplayStars:0.0}";
}
public partial class Relationship
{
public override string ToString() => $"{RelationshipToProduct} {RelationshipType}";
}
public partial class Series
{
public string SeriesName => Title;
public string SeriesId => Asin;
public float Index
=> string.IsNullOrEmpty(Sequence)
? 0
// eg: a book containing volumes 5,6,7,8 has sequence "5-8"
: float.Parse(Sequence.Split('-').First());
public override string ToString() => $"[{SeriesId}] {SeriesName}";
}
}

31
DTOs/LibraryApiV10Extensions.cs
View File

@ -1,31 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Dinah.Core.Collections.Generic;
namespace DTOs
{
public static class LibraryApiV10Extensions
{
public static IEnumerable<Person> GetAuthorsDistinct(this IEnumerable<Item> items)
=> items.SelectMany(i => i.Authors).DistinctBy(a => new { a.Name, a.Asin });
public static IEnumerable<string> GetNarratorsDistinct(this IEnumerable<Item> items)
=> items.SelectMany(i => i.Narrators, (i, n) => n.Name).Distinct();
public static IEnumerable<string> GetPublishersDistinct(this IEnumerable<Item> items)
=> items.Select(i => i.Publisher).Distinct();
public static IEnumerable<Series> GetSeriesDistinct(this IEnumerable<Item> items)
=> items.SelectMany(i => i.Series).DistinctBy(s => new { s.SeriesName, s.SeriesId });
public static IEnumerable<Ladder> GetParentCategoriesDistinct(this IEnumerable<Item> items)
=> items.Select(l => l.ParentCategory).DistinctBy(l => new { l.CategoryName, l.CategoryId });
public static IEnumerable<Ladder> GetChildCategoriesDistinct(this IEnumerable<Item> items)
=> items
.Select(l => l.ChildCategory)
.Where(l => l != null)
.DistinctBy(l => new { l.CategoryName, l.CategoryId });
}
}

6
DataLayer/UNTESTED/EfClasses/Book.cs
View File

@ -62,8 +62,7 @@ namespace DataLayer
string title, string title,
string description, string description,
int lengthInMinutes, int lengthInMinutes,
IEnumerable<Contributor> authors, IEnumerable<Contributor> authors)
IEnumerable<Contributor> narrators)
{ {
// validate // validate
ArgumentValidator.EnsureNotNull(audibleProductId, nameof(audibleProductId)); ArgumentValidator.EnsureNotNull(audibleProductId, nameof(audibleProductId));
@ -88,7 +87,7 @@ namespace DataLayer
// assigns with biz logic // assigns with biz logic
ReplaceAuthors(authors); ReplaceAuthors(authors);
ReplaceNarrators(narrators); //ReplaceNarrators(narrators);
// import previously saved tags // import previously saved tags
// do this immediately. any save occurs before reloading tags will overwrite persistent tags with new blank entries; all old persisted tags will be lost // do this immediately. any save occurs before reloading tags will overwrite persistent tags with new blank entries; all old persisted tags will be lost
@ -218,7 +217,6 @@ namespace DataLayer
// Are automatically loaded, and can only be tracked by a DbContext alongside their owner." // Are automatically loaded, and can only be tracked by a DbContext alongside their owner."
ArgumentValidator.EnsureNotNullOrWhiteSpace(url, nameof(url)); ArgumentValidator.EnsureNotNullOrWhiteSpace(url, nameof(url));
url = FileManager.FileUtility.RestoreDeclawed(url);
if (!_supplements.Any(s => url.EqualsInsensitive(url))) if (!_supplements.Any(s => url.EqualsInsensitive(url)))
_supplements.Add(new Supplement(this, url)); _supplements.Add(new Supplement(this, url));

10
DataLayer/UNTESTED/EfClasses/LibraryBook.cs
View File

@ -10,16 +10,18 @@ namespace DataLayer
public DateTime DateAdded { get; private set; } public DateTime DateAdded { get; private set; }
/// <summary>For downloading AAX file</summary> /// <summary>For downloading AAX file</summary>
public string DownloadBookLink { get; private set; } public string DownloadBookLink { get; private set; }
private LibraryBook() { } private LibraryBook() { }
public LibraryBook(Book book, DateTime dateAdded, string downloadBookLink) public LibraryBook(Book book, DateTime dateAdded
, string downloadBookLink = null
)
{ {
ArgumentValidator.EnsureNotNull(book, nameof(book)); ArgumentValidator.EnsureNotNull(book, nameof(book));
Book = book; Book = book;
DateAdded = dateAdded; DateAdded = dateAdded;
DownloadBookLink = downloadBookLink; DownloadBookLink = downloadBookLink;
} }
} }
} }

4
DataLayer/UNTESTED/EfClasses/Rating.cs
View File

@ -64,11 +64,11 @@ namespace DataLayer
var items = new List<string>(); var items = new List<string>();
if (OverallRating > 0) if (OverallRating > 0)
items.Add($"Overall: {getStars(OverallRating)}"); items.Add($"Overall: {getStars(OverallRating)}");
if (PerformanceRating > 0) if (PerformanceRating > 0)
items.Add($"Perform: {getStars(PerformanceRating)}"); items.Add($"Perform: {getStars(PerformanceRating)}");
if (StoryRating > 0) if (StoryRating > 0)
items.Add($"Story: {getStars(StoryRating)}"); items.Add($"Story: {getStars(StoryRating)}");
return string.Join("\r\n", items); return string.Join("\r\n", items);
} }

6
DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs
View File

@ -11,7 +11,7 @@ namespace DataLayer
using var context = LibationContext.Create(); using var context = LibationContext.Create();
return context return context
.Library .Library
.AsNoTracking() //.AsNoTracking()
.GetLibrary() .GetLibrary()
.ToList(); .ToList();
} }
@ -21,8 +21,8 @@ namespace DataLayer
using var context = LibationContext.Create(); using var context = LibationContext.Create();
return context return context
.Library .Library
.AsNoTracking() //.AsNoTracking()
.GetLibraryBook(productId); .GetLibraryBook(productId);
} }
/// <summary>This is still IQueryable. YOU MUST CALL ToList() YOURSELF</summary> /// <summary>This is still IQueryable. YOU MUST CALL ToList() YOURSELF</summary>

136
DtoImporterService/BookImporter.cs Normal file
View File

@ -0,0 +1,136 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
namespace DtoImporterService
{
public class BookImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.Any(i => string.IsNullOrWhiteSpace(i.ProductId)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with blank {nameof(Item.ProductId)}", nameof(items)));
if (items.Any(i => string.IsNullOrWhiteSpace(i.Title)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with blank {nameof(Item.Title)}", nameof(items)));
if (items.Any(i => i.Authors is null))
exceptions.Add(new ArgumentException($"Collection contains item(s) with null {nameof(Item.Authors)}", nameof(items)));
return exceptions;
}
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{
// pre-req.s
new ContributorImporter().Import(items, context);
new SeriesImporter().Import(items, context);
new CategoryImporter().Import(items, context);
// get distinct
var productIds = items.Select(i => i.ProductId).ToList();
// load db existing => .Local
loadLocal_books(productIds, context);
// upsert
var qtyNew = upsertBooks(items, context);
return qtyNew;
}
private void loadLocal_books(List<string> productIds, LibationContext context)
{
var localProductIds = context.Books.Local.Select(b => b.AudibleProductId);
var remainingProductIds = productIds
.Distinct()
.Except(localProductIds)
.ToList();
// GetBooks() eager loads Series, category, et al
if (remainingProductIds.Any())
context.Books.GetBooks(b => remainingProductIds.Contains(b.AudibleProductId)).ToList();
}
private int upsertBooks(IEnumerable<Item> items, LibationContext context)
{
var qtyNew = 0;
foreach (var item in items)
{
var book = context.Books.Local.SingleOrDefault(p => p.AudibleProductId == item.ProductId);
if (book is null)
{
// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
var authors = item
.Authors
.Select(a => context.Contributors.Local.Single(c => a.Name == c.Name))
.ToList();
book = context.Books.Add(new Book(
new AudibleProductId(item.ProductId), item.Title, item.Description, item.LengthInMinutes, authors))
.Entity;
qtyNew++;
}
// if no narrators listed, author is the narrator
if (item.Narrators is null || !item.Narrators.Any())
item.Narrators = item.Authors;
// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
var narrators = item
.Narrators
.Select(n => context.Contributors.Local.Single(c => n.Name == c.Name))
.ToList();
// not all books have narrators. these will already be using author as narrator. don't undo this
if (narrators.Any())
book.ReplaceNarrators(narrators);
// set/update book-specific info which may have changed
book.PictureId = item.PictureId;
book.UpdateProductRating(item.Product_OverallStars, item.Product_PerformanceStars, item.Product_StoryStars);
if (!string.IsNullOrWhiteSpace(item.SupplementUrl))
book.AddSupplementDownloadUrl(item.SupplementUrl);
var publisherName = item.Publisher;
if (!string.IsNullOrWhiteSpace(publisherName))
{
var publisher = context.Contributors.Local.Single(c => publisherName == c.Name);
book.ReplacePublisher(publisher);
}
// important to update user-specific info. this will have changed if user has rated/reviewed the book since last library import
book.UserDefinedItem.UpdateRating(item.MyUserRating_Overall, item.MyUserRating_Performance, item.MyUserRating_Story);
//
// this was round 1 when it was a 2 step process
//
//// update series even for existing books. these are occasionally updated
//var seriesIds = item.Series.Select(kvp => kvp.SeriesId).ToList();
//var allSeries = context.Series.Local.Where(c => seriesIds.Contains(c.AudibleSeriesId)).ToList();
//foreach (var series in allSeries)
// book.UpsertSeries(series);
// these will upsert over library-scraped series, but will not leave orphans
if (item.Series != null)
{
foreach (var seriesEntry in item.Series)
{
var series = context.Series.Local.Single(s => seriesEntry.SeriesId == s.AudibleSeriesId);
book.UpsertSeries(series, seriesEntry.Index);
}
}
// categories are laid out for a breadcrumb. category is 1st, subcategory is 2nd
var category = context.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == item.Categories.LastOrDefault().CategoryId);
if (category != null)
book.UpdateCategory(category, context);
book.UpdateBookDetails(item.IsAbridged, item.DatePublished);
}
return qtyNew;
}
}
}

83
DtoImporterService/CategoryImporter.cs Normal file
View File

@ -0,0 +1,83 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
namespace DtoImporterService
{
public class CategoryImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
var distinct = items.GetCategoriesDistinct();
if (distinct.Any(s => s.CategoryId is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with null {nameof(Ladder.CategoryId)}", nameof(items)));
if (distinct.Any(s => s.CategoryName is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with null {nameof(Ladder.CategoryName)}", nameof(items)));
if (items.GetCategoryPairsDistinct().Any(p => p.Length > 2))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with wrong number of categories. Expecting 0, 1, or 2 categories per title", nameof(items)));
return exceptions;
}
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{
// get distinct
var categoryIds = items.GetCategoriesDistinct().Select(c => c.CategoryId).ToList();
// load db existing => .Local
loadLocal_categories(categoryIds, context);
// upsert
var categoryPairs = items.GetCategoryPairsDistinct().ToList();
var qtyNew = upsertCategories(categoryPairs, context);
return qtyNew;
}
private void loadLocal_categories(List<string> categoryIds, LibationContext context)
{
var localIds = context.Categories.Local.Select(c => c.AudibleCategoryId);
var remainingCategoryIds = categoryIds
.Distinct()
.Except(localIds)
.ToList();
if (remainingCategoryIds.Any())
context.Categories.Where(c => remainingCategoryIds.Contains(c.AudibleCategoryId)).ToList();
}
// only use after loading contributors => local
private int upsertCategories(List<Ladder[]> categoryPairs, LibationContext context)
{
var qtyNew = 0;
foreach (var pair in categoryPairs)
{
for (var i = 0; i < pair.Length; i++)
{
var id = pair[i].CategoryId;
var name = pair[i].CategoryName;
Category parentCategory = null;
if (i == 1)
parentCategory = context.Categories.Local.Single(c => c.AudibleCategoryId == pair[0].CategoryId);
var category = context.Categories.Local.SingleOrDefault(c => c.AudibleCategoryId == id);
if (category is null)
{
category = context.Categories.Add(new Category(new AudibleCategoryId(id), name)).Entity;
qtyNew++;
}
category.UpdateParentCategory(parentCategory);
}
}
return qtyNew;
}
}
}

106
DtoImporterService/ContributorImporter.cs Normal file
View File

@ -0,0 +1,106 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
namespace DtoImporterService
{
public class ContributorImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.GetAuthorsDistinct().Any(a => string.IsNullOrWhiteSpace(a.Name)))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Authors)} with null {nameof(Person.Name)}", nameof(items)));
if (items.GetNarratorsDistinct().Any(a => string.IsNullOrWhiteSpace(a.Name)))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Narrators)} with null {nameof(Person.Name)}", nameof(items)));
return exceptions;
}
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{
// get distinct
var authors = items.GetAuthorsDistinct().ToList();
var narrators = items.GetNarratorsDistinct().ToList();
var publishers = items.GetPublishersDistinct().ToList();
// load db existing => .Local
var allNames = authors
.Select(a => a.Name)
.Union(narrators.Select(n => n.Name))
.Union(publishers)
.ToList();
loadLocal_contributors(allNames, context);
// upsert
var qtyNew = 0;
qtyNew += upsertPeople(authors, context);
qtyNew += upsertPeople(narrators, context);
qtyNew += upsertPublishers(publishers, context);
return qtyNew;
}
private void loadLocal_contributors(List<string> contributorNames, LibationContext context)
{
contributorNames.Remove(null);
contributorNames.Remove("");
//// BAD: very inefficient
// var x = context.Contributors.Local.Where(c => !contribNames.Contains(c.Name));
// GOOD: Except() is efficient. Due to hashing, it's close to O(n)
var localNames = context.Contributors.Local.Select(c => c.Name);
var remainingContribNames = contributorNames
.Distinct()
.Except(localNames)
.ToList();
// load existing => local
if (remainingContribNames.Any())
context.Contributors.Where(c => remainingContribNames.Contains(c.Name)).ToList();
// _________________________________^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
// i tried to extract this pattern, but this part prohibits doing so
// wouldn't work anyway for Books.GetBooks()
}
// only use after loading contributors => local
private int upsertPeople(List<Person> people, LibationContext context)
{
var qtyNew = 0;
foreach (var p in people)
{
var person = context.Contributors.Local.SingleOrDefault(c => c.Name == p.Name);
if (person == null)
{
person = context.Contributors.Add(new Contributor(p.Name)).Entity;
qtyNew++;
}
person.UpdateAudibleAuthorId(p.Asin);
}
return qtyNew;
}
// only use after loading contributors => local
private int upsertPublishers(List<string> publishers, LibationContext context)
{
var qtyNew = 0;
foreach (var publisherName in publishers)
{
if (context.Contributors.Local.SingleOrDefault(c => c.Name == publisherName) == null)
{
context.Contributors.Add(new Contributor(publisherName));
qtyNew++;
}
}
return qtyNew;
}
}
}

12
DtoImporterService/DtoImporterService.csproj Normal file
View File

@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApiDTOs\AudibleApiDTOs.csproj" />
<ProjectReference Include="..\DataLayer\DataLayer.csproj" />
</ItemGroup>
</Project>

44
DtoImporterService/ImporterBase.cs Normal file
View File

@ -0,0 +1,44 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
namespace DtoImporterService
{
public interface IContextRunner<T>
{
public TResult Run<TResult>(Func<T, LibationContext, TResult> func, T param, LibationContext context = null)
{
if (context is null)
{
using (context = LibationContext.Create())
{
var r = Run(func, param, context);
context.SaveChanges();
return r;
}
}
var exceptions = Validate(param);
if (exceptions != null && exceptions.Any())
throw new AggregateException($"Device Jobs Service configuration validation failed", exceptions);
var result = func(param, context);
return result;
}
IEnumerable<Exception> Validate(T param);
}
public abstract class ImporterBase<T> : IContextRunner<T>
{
/// <summary>LONG RUNNING. call with await Task.Run</summary>
public int Import(T param, LibationContext context = null)
=> ((IContextRunner<T>)this).Run(DoImport, param, context);
protected abstract int DoImport(T elements, LibationContext context);
public abstract IEnumerable<Exception> Validate(T param);
}
public abstract class ItemsImporterBase : ImporterBase<IEnumerable<Item>> { }
}

50
DtoImporterService/LibraryImporter.cs Normal file
View File

@ -0,0 +1,50 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
namespace DtoImporterService
{
public class LibraryImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.Any(i => string.IsNullOrWhiteSpace(i.ProductId)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with null or blank {nameof(Item.ProductId)}", nameof(items)));
if (items.Any(i => i.DateAdded < new DateTime(1980, 1, 1)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with invalid {nameof(Item.DateAdded)}", nameof(items)));
return exceptions;
}
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{
new BookImporter().Import(items, context);
var qtyNew = upsertLibraryBooks(items, context);
return qtyNew;
}
private int upsertLibraryBooks(IEnumerable<Item> items, LibationContext context)
{
var currentLibraryProductIds = context.Library.Select(l => l.Book.AudibleProductId).ToList();
var newItems = items.Where(dto => !currentLibraryProductIds.Contains(dto.ProductId)).ToList();
foreach (var newItem in newItems)
{
var libraryBook = new LibraryBook(
context.Books.Local.Single(b => b.AudibleProductId == newItem.ProductId),
newItem.DateAdded
//,FileManager.FileUtility.RestoreDeclawed(newLibraryDTO.DownloadBookLink)
);
context.Library.Add(libraryBook);
}
var qtyNew = newItems.Count;
return qtyNew;
}
}
}

68
DtoImporterService/SeriesImporter.cs Normal file
View File

@ -0,0 +1,68 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
namespace DtoImporterService
{
public class SeriesImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
var distinct = items .GetSeriesDistinct();
if (distinct.Any(s => s.SeriesId is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Series)} with null {nameof(AudibleApiDTOs.Series.SeriesId)}", nameof(items)));
if (distinct.Any(s => s.SeriesName is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Series)} with null {nameof(AudibleApiDTOs.Series.SeriesName)}", nameof(items)));
return exceptions;
}
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{
// get distinct
var series = items.GetSeriesDistinct().ToList();
// load db existing => .Local
var seriesIds = series.Select(s => s.SeriesId).ToList();
loadLocal_series(seriesIds, context);
// upsert
var qtyNew = upsertSeries(series, context);
return qtyNew;
}
private void loadLocal_series(List<string> seriesIds, LibationContext context)
{
var localIds = context.Series.Local.Select(s => s.AudibleSeriesId);
var remainingSeriesIds = seriesIds
.Distinct()
.Except(localIds)
.ToList();
if (remainingSeriesIds.Any())
context.Series.Where(s => remainingSeriesIds.Contains(s.AudibleSeriesId)).ToList();
}
private int upsertSeries(List<AudibleApiDTOs.Series> requestedSeries, LibationContext context)
{
var qtyNew = 0;
foreach (var s in requestedSeries)
{
var series = context.Series.Local.SingleOrDefault(c => c.AudibleSeriesId == s.SeriesId);
if (series is null)
{
series = context.Series.Add(new DataLayer.Series(new AudibleSeriesId(s.SeriesId))).Entity;
qtyNew++;
}
series.UpdateName(s.SeriesName);
}
return qtyNew;
}
}
}

1
InternalUtilities/InternalUtilities.csproj
View File

@ -5,6 +5,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
<ProjectReference Include="..\LibationSearchEngine\LibationSearchEngine.csproj" /> <ProjectReference Include="..\LibationSearchEngine\LibationSearchEngine.csproj" />
<ProjectReference Include="..\Scraping\Scraping.csproj" /> <ProjectReference Include="..\Scraping\Scraping.csproj" />
</ItemGroup> </ItemGroup>

42
InternalUtilities/UNTESTED/AudibleApiExtensions.cs Normal file
View File

@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using AudibleApi;
using AudibleApiDTOs;
//
// probably not the best place for this
// but good enough for now
//
namespace InternalUtilities
{
public static class AudibleApiExtensions
{
public static async Task<List<Item>> GetAllLibraryItemsAsync(this Api api)
{
var allItems = new List<Item>();
for (var i = 1; ; i++)
{
var page = await api.GetLibraryAsync(new LibraryOptions
{
NumberOfResultPerPage = 1000,
PageNumber = i,
PurchasedAfter = new DateTime(2000, 1, 1),
ResponseGroups = LibraryOptions.ResponseGroupOptions.ALL_OPTIONS
});
// important! use this convert method
var libResult = LibraryApiV10.FromJson(page.ToString());
if (!libResult.Items.Any())
break;
allItems.AddRange(libResult.Items);
}
return allItems;
}
}
}

45
Libation.sln
View File

@ -57,8 +57,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GitAllRepos", "..\GitAllRep
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApi", "..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj", "{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApi", "..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj", "{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiClientExample", "..\audible api\AudibleApi\AudibleApiClientExample\AudibleApiClientExample.csproj", "{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApi.Tests", "..\audible api\AudibleApi\_Tests\AudibleApi.Tests\AudibleApi.Tests.csproj", "{111420E2-D4F0-4068-B46A-C4B6DCC823DC}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApi.Tests", "..\audible api\AudibleApi\_Tests\AudibleApi.Tests\AudibleApi.Tests.csproj", "{111420E2-D4F0-4068-B46A-C4B6DCC823DC}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LibationWinForm", "LibationWinForm\LibationWinForm.csproj", "{635F00E1-AAD1-45F7-BEB7-D909AD33B9F6}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LibationWinForm", "LibationWinForm\LibationWinForm.csproj", "{635F00E1-AAD1-45F7-BEB7-D909AD33B9F6}"
@ -81,9 +79,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2", "..\Lucene
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2.Tests", "..\LuceneNet303r2\LuceneNet303r2.Tests\LuceneNet303r2.Tests.csproj", "{5A7681A5-60D9-480B-9AC7-63E0812A2548}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2.Tests", "..\LuceneNet303r2\LuceneNet303r2.Tests\LuceneNet303r2.Tests.csproj", "{5A7681A5-60D9-480B-9AC7-63E0812A2548}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DTOs", "DTOs\DTOs.csproj", "{5FDA62B1-55FD-407A-BECA-38A969235541}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DtoImporterService", "DtoImporterService\DtoImporterService.csproj", "{401865F5-1942-4713-B230-04544C0A97B0}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiDomainService", "AudibleApiDomainService\AudibleApiDomainService.csproj", "{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiDTOs", "..\audible api\AudibleApi\AudibleApiDTOs\AudibleApiDTOs.csproj", "{C03C5D65-3B7F-453B-972F-23950B7E0604}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiDTOs.Tests", "..\audible api\AudibleApi\_Tests\AudibleApiDTOs.Tests\AudibleApiDTOs.Tests.csproj", "{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiClientExample", "..\audible api\AudibleApi\_Demos\AudibleApiClientExample\AudibleApiClientExample.csproj", "{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -151,10 +153,6 @@ Global
{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Debug|Any CPU.Build.0 = Debug|Any CPU {7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Release|Any CPU.ActiveCfg = Release|Any CPU {7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Release|Any CPU.Build.0 = Release|Any CPU {7EA01F9C-E579-4B01-A3B9-733B49DD0B60}.Release|Any CPU.Build.0 = Release|Any CPU
{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}.Debug|Any CPU.Build.0 = Debug|Any CPU
{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}.Release|Any CPU.ActiveCfg = Release|Any CPU
{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52}.Release|Any CPU.Build.0 = Release|Any CPU
{111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Debug|Any CPU.Build.0 = Debug|Any CPU {111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Release|Any CPU.ActiveCfg = Release|Any CPU {111420E2-D4F0-4068-B46A-C4B6DCC823DC}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -199,14 +197,22 @@ Global
{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Debug|Any CPU.Build.0 = Debug|Any CPU {5A7681A5-60D9-480B-9AC7-63E0812A2548}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.ActiveCfg = Release|Any CPU {5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.Build.0 = Release|Any CPU {5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.Build.0 = Release|Any CPU
{5FDA62B1-55FD-407A-BECA-38A969235541}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {401865F5-1942-4713-B230-04544C0A97B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5FDA62B1-55FD-407A-BECA-38A969235541}.Debug|Any CPU.Build.0 = Debug|Any CPU {401865F5-1942-4713-B230-04544C0A97B0}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5FDA62B1-55FD-407A-BECA-38A969235541}.Release|Any CPU.ActiveCfg = Release|Any CPU {401865F5-1942-4713-B230-04544C0A97B0}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5FDA62B1-55FD-407A-BECA-38A969235541}.Release|Any CPU.Build.0 = Release|Any CPU {401865F5-1942-4713-B230-04544C0A97B0}.Release|Any CPU.Build.0 = Release|Any CPU
{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C03C5D65-3B7F-453B-972F-23950B7E0604}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}.Debug|Any CPU.Build.0 = Debug|Any CPU {C03C5D65-3B7F-453B-972F-23950B7E0604}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}.Release|Any CPU.ActiveCfg = Release|Any CPU {C03C5D65-3B7F-453B-972F-23950B7E0604}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050}.Release|Any CPU.Build.0 = Release|Any CPU {C03C5D65-3B7F-453B-972F-23950B7E0604}.Release|Any CPU.Build.0 = Release|Any CPU
{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD}.Release|Any CPU.Build.0 = Release|Any CPU
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@ -227,7 +233,6 @@ Global
{FF12ADA0-8975-4E67-B6EA-4AC82E0C8994} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD} {FF12ADA0-8975-4E67-B6EA-4AC82E0C8994} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{AD1FDDC9-8D2A-436A-8EED-91FD74E7C7B4} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD} {AD1FDDC9-8D2A-436A-8EED-91FD74E7C7B4} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{7EA01F9C-E579-4B01-A3B9-733B49DD0B60} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} {7EA01F9C-E579-4B01-A3B9-733B49DD0B60} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
{959D01B4-5EF8-4D4E-BE06-AB1A580B0B52} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
{111420E2-D4F0-4068-B46A-C4B6DCC823DC} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD} {111420E2-D4F0-4068-B46A-C4B6DCC823DC} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{635F00E1-AAD1-45F7-BEB7-D909AD33B9F6} = {8679CAC8-9164-4007-BDD2-F004810EDA14} {635F00E1-AAD1-45F7-BEB7-D909AD33B9F6} = {8679CAC8-9164-4007-BDD2-F004810EDA14}
{0807616A-A77A-4B08-A65A-1582B09E114B} = {8679CAC8-9164-4007-BDD2-F004810EDA14} {0807616A-A77A-4B08-A65A-1582B09E114B} = {8679CAC8-9164-4007-BDD2-F004810EDA14}
@ -239,8 +244,10 @@ Global
{1255D9BA-CE6E-42E4-A253-6376540B9661} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1} {1255D9BA-CE6E-42E4-A253-6376540B9661} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
{35803735-B669-4090-9681-CC7F7FABDC71} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} {35803735-B669-4090-9681-CC7F7FABDC71} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
{5A7681A5-60D9-480B-9AC7-63E0812A2548} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD} {5A7681A5-60D9-480B-9AC7-63E0812A2548} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{5FDA62B1-55FD-407A-BECA-38A969235541} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} {401865F5-1942-4713-B230-04544C0A97B0} = {41CDCC73-9B81-49DD-9570-C54406E852AF}
{A1AB4B4B-6855-4BD0-BC54-C2FFDB20E050} = {41CDCC73-9B81-49DD-9570-C54406E852AF} {C03C5D65-3B7F-453B-972F-23950B7E0604} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
EndGlobalSection EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9} SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9}

2
LibationWinForm/LibationWinForm.csproj
View File

@ -11,7 +11,7 @@
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" /> <ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
<ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj" /> <ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj" />
<ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj" /> <ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj" />
<ProjectReference Include="..\AudibleApiDomainService\AudibleApiDomainService.csproj" /> <ProjectReference Include="..\DtoImporterService\DtoImporterService.csproj" />
<ProjectReference Include="..\ScrapingDomainServices\ScrapingDomainServices.csproj" /> <ProjectReference Include="..\ScrapingDomainServices\ScrapingDomainServices.csproj" />
</ItemGroup> </ItemGroup>

4
LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.cs
View File

@ -83,8 +83,8 @@ namespace LibationWinForm.BookLiberation
// close form on DOWNLOAD completed, not final Completed. Else for BackupBook this form won't close until DECRYPT is also complete // close form on DOWNLOAD completed, not final Completed. Else for BackupBook this form won't close until DECRYPT is also complete
void fileDownloadCompleted(object _, string __) => downloadDialog.Close(); void fileDownloadCompleted(object _, string __) => downloadDialog.Close();
void downloadProgressChanged(object _, System.Net.DownloadProgressChangedEventArgs arg) void downloadProgressChanged(object _, Dinah.Core.Net.Http.DownloadProgress progress)
=> downloadDialog.DownloadProgressChanged(arg.BytesReceived, arg.TotalBytesToReceive); => downloadDialog.DownloadProgressChanged(progress.BytesReceived, progress.TotalBytesToReceive.Value);
void unsubscribe(object _ = null, EventArgs __ = null) void unsubscribe(object _ = null, EventArgs __ = null)
{ {

2
LibationWinForm/UNTESTED/Dialogs/Login/WinformResponder.cs
View File

@ -3,7 +3,7 @@ using LibationWinForm.Dialogs.Login;
namespace LibationWinForm.Login namespace LibationWinForm.Login
{ {
public class WinformResponder : AudibleApiDomainService.IAudibleApiResponder public class WinformResponder : AudibleApi.ILoginCallback
{ {
public string Get2faCode() public string Get2faCode()
{ {

56
LibationWinForm/UNTESTED/Form1.cs
View File

@ -357,18 +357,48 @@ namespace LibationWinForm
private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e) private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e)
{ {
// audible api // legacy/scraping method
var settings = new AudibleApiDomainService.Settings(config); //await indexDialog(new ScanLibraryDialog());
var responder = new Login.WinformResponder(); // new/api method
var client = await AudibleApiDomainService.AudibleApiLibationClient.CreateClientAsync(settings, responder); await audibleApi();
await client.ImportLibraryAsync();
// scrape
await indexDialog(new ScanLibraryDialog());
} }
private async void reimportMostRecentLibraryScanToolStripMenuItem_Click(object sender, EventArgs e) private async Task audibleApi()
{
var identityFilePath = System.IO.Path.Combine(config.LibationFiles, "IdentityTokens.json");
var callback = new Login.WinformResponder();
var api = await AudibleApi.EzApiCreator.GetApiAsync(identityFilePath, callback, config.LocaleCountryCode);
int totalCount;
int newCount;
// seems to be very common the 1st time after long absence. either figure out why, or run 2x before declaring error
try
{
var items = await InternalUtilities.AudibleApiExtensions.GetAllLibraryItemsAsync(api);
totalCount = items.Count;
newCount = await Task.Run(() => new DtoImporterService.LibraryImporter().Import(items));
}
catch
{
try
{
var items = await InternalUtilities.AudibleApiExtensions.GetAllLibraryItemsAsync(api);
totalCount = items.Count;
newCount = await Task.Run(() => new DtoImporterService.LibraryImporter().Import(items));
}
catch (Exception ex)
{
MessageBox.Show("Error importing library.\r\n" + ex.Message, "Error importing library", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
}
await InternalUtilities.SearchEngineActions.FullReIndexAsync();
await indexComplete(totalCount, newCount);
}
private async void reimportMostRecentLibraryScanToolStripMenuItem_Click(object sender, EventArgs e)
{ {
// DO NOT ConfigureAwait(false) // DO NOT ConfigureAwait(false)
// this would result in index() => reloadGrid() => setGrid() => "gridPanel.Controls.Remove(currProductsGrid);" // this would result in index() => reloadGrid() => setGrid() => "gridPanel.Controls.Remove(currProductsGrid);"
@ -377,15 +407,15 @@ await client.ImportLibraryAsync();
MessageBox.Show($"Total processed: {TotalBooksProcessed}\r\nNew: {NewBooksAdded}"); MessageBox.Show($"Total processed: {TotalBooksProcessed}\r\nNew: {NewBooksAdded}");
await index(NewBooksAdded, TotalBooksProcessed); await indexComplete(TotalBooksProcessed, NewBooksAdded);
} }
private async Task indexDialog(IIndexLibraryDialog dialog) private async Task indexDialog(IIndexLibraryDialog dialog)
{ {
if (!dialog.RunDialog().In(DialogResult.Abort, DialogResult.Cancel, DialogResult.None)) if (!dialog.RunDialog().In(DialogResult.Abort, DialogResult.Cancel, DialogResult.None))
await index(dialog.NewBooksAdded, dialog.TotalBooksProcessed); await indexComplete(dialog.TotalBooksProcessed, dialog.NewBooksAdded);
} }
private async Task index(int newBooksAdded, int totalBooksProcessed) private async Task indexComplete(int totalBooksProcessed, int newBooksAdded)
{ {
// update backup counts if we have new library items // update backup counts if we have new library items
if (newBooksAdded > 0) if (newBooksAdded > 0)

173
LibationWinForm/UNTESTED/GridEntry.cs
View File

@ -7,103 +7,132 @@ using DataLayer;
namespace LibationWinForm namespace LibationWinForm
{ {
internal class GridEntry internal class GridEntry
{ {
private LibraryBook libraryBook; private LibraryBook libraryBook;
private Book book => libraryBook.Book; private Book book => libraryBook.Book;
public Book GetBook() => book; public Book GetBook() => book;
// this special case is obvious and ugly // this special case is obvious and ugly
public void REPLACE_Library_Book(LibraryBook libraryBook) => this.libraryBook = libraryBook; public void REPLACE_Library_Book(LibraryBook libraryBook) => this.libraryBook = libraryBook;
public GridEntry(LibraryBook libraryBook) => this.libraryBook = libraryBook; public GridEntry(LibraryBook libraryBook) => this.libraryBook = libraryBook;
// hide from public fields from Data Source GUI with [Browsable(false)] // hide from public fields from Data Source GUI with [Browsable(false)]
[Browsable(false)] [Browsable(false)]
public string Tags => book.UserDefinedItem.Tags; public string Tags => book.UserDefinedItem.Tags;
[Browsable(false)] [Browsable(false)]
public IEnumerable<string> TagsEnumerated => book.UserDefinedItem.TagsEnumerated; public IEnumerable<string> TagsEnumerated => book.UserDefinedItem.TagsEnumerated;
private Dictionary<string, string> formatReplacements { get; } = new Dictionary<string, string>(); // formatReplacements is what gets displayed
public bool TryGetFormatted(string key, out string value) => formatReplacements.TryGetValue(key, out value); // the value that gets returned from the property is the cell's value
// this allows for the value to be sorted one way and displayed another
// eg:
// orig title: The Computer
// formatReplacement: The Computer
// value for sorting: Computer
private Dictionary<string, string> formatReplacements { get; } = new Dictionary<string, string>();
public bool TryGetFormatted(string key, out string value) => formatReplacements.TryGetValue(key, out value);
public Image Cover => public Image Cover =>
Dinah.Core.Drawing.ImageConverter.GetPictureFromBytes( Dinah.Core.Drawing.ImageConverter.GetPictureFromBytes(
FileManager.PictureStorage.GetImage(book.PictureId, FileManager.PictureStorage.PictureSize._80x80) FileManager.PictureStorage.GetImage(book.PictureId, FileManager.PictureStorage.PictureSize._80x80)
); );
public string Title public string Title
{ {
get get
{ {
formatReplacements[nameof(Title)] = book.Title; formatReplacements[nameof(Title)] = book.Title;
var sortName = book.Title var sortName = book.Title
.Replace("|", "") .Replace("|", "")
.Replace(":", "") .Replace(":", "")
.ToLowerInvariant(); .ToLowerInvariant();
if (sortName.StartsWith("the ") || sortName.StartsWith("a ") || sortName.StartsWith("an ")) if (sortName.StartsWith("the ") || sortName.StartsWith("a ") || sortName.StartsWith("an "))
sortName = sortName.Substring(sortName.IndexOf(" ") + 1); sortName = sortName.Substring(sortName.IndexOf(" ") + 1);
return sortName; return sortName;
} }
} }
public string Authors => book.AuthorNames; public string Authors => book.AuthorNames;
public string Narrators => book.NarratorNames; public string Narrators => book.NarratorNames;
public int Length public int Length
{ {
get get
{ {
formatReplacements[nameof(Length)] formatReplacements[nameof(Length)]
= book.LengthInMinutes == 0 = book.LengthInMinutes == 0
? "[pre-release]" ? ""
: $"{book.LengthInMinutes / 60} hr {book.LengthInMinutes % 60} min"; : $"{book.LengthInMinutes / 60} hr {book.LengthInMinutes % 60} min";
return book.LengthInMinutes; return book.LengthInMinutes;
} }
} }
public string Series => book.SeriesNames; public string Series => book.SeriesNames;
public string Description private string descriptionCache = null;
=> book.Description == null ? "" public string Description
: book.Description.Length < 63 ? book.Description {
: book.Description.Substring(0, 60) + "..."; get
{
// HtmlAgilityPack is expensive. cache results
if (descriptionCache is null)
{
if (book.Description is null)
descriptionCache = "";
else
{
var doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(book.Description);
var noHtml = doc.DocumentNode.InnerText;
descriptionCache
= noHtml.Length < 63
? noHtml
: noHtml.Substring(0, 60) + "...";
}
}
public string Category => string.Join(" > ", book.CategoriesNames); return descriptionCache;
}
}
// star ratings retain numeric value but display star text. this is needed because just using star text doesn't sort correctly: public string Category => string.Join(" > ", book.CategoriesNames);
// - star
// - star star
// - star 1/2
public string Product_Rating // star ratings retain numeric value but display star text. this is needed because just using star text doesn't sort correctly:
{ // - star
get // - star star
{ // - star 1/2
Rating rating = book.Rating;
formatReplacements[nameof(Product_Rating)] = starString(rating); public string Product_Rating
{
get
{
formatReplacements[nameof(Product_Rating)] = starString(book.Rating);
return firstScore(book.Rating);
}
}
return firstScore(rating); public string Purchase_Date
} {
} get
{
public DateTime? Purchase_Date => libraryBook.DateAdded; formatReplacements[nameof(Purchase_Date)] = libraryBook.DateAdded.ToString("d");
return libraryBook.DateAdded.ToString("yyyy-MM-dd HH:mm:ss");
}
}
public string My_Rating public string My_Rating
{ {
get get
{ {
Rating rating = book.UserDefinedItem.Rating; formatReplacements[nameof(My_Rating)] = starString(book.UserDefinedItem.Rating);
return firstScore(book.UserDefinedItem.Rating);
formatReplacements[nameof(My_Rating)] = starString(rating);
return firstScore(rating);
} }
} }

26
LibationWinForm/UNTESTED/ProductsGrid.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Windows.Forms; using System.Windows.Forms;
using Dinah.Core.DataBinding; using Dinah.Core.DataBinding;
using DataLayer; using DataLayer;
using Dinah.Core.Collections.Generic;
namespace LibationWinForm namespace LibationWinForm
{ {
@ -76,11 +77,14 @@ namespace LibationWinForm
col.HeaderText = col.HeaderText.Replace("_", " "); col.HeaderText = col.HeaderText.Replace("_", " ");
if (col.Name == nameof(GridEntry.Title)) col.Width = col.Name switch
col.Width *= 2; {
nameof(GridEntry.Cover) => 80,
if (col.Name == nameof(GridEntry.Misc)) nameof(GridEntry.Title) => col.Width * 2,
col.Width = (int)(col.Width * 1.35); nameof(GridEntry.Misc) => (int)(col.Width * 1.35),
var n when n.In(nameof(GridEntry.My_Rating), nameof(GridEntry.Product_Rating)) => col.Width + 8,
_ => col.Width
};
} }
@ -88,12 +92,14 @@ namespace LibationWinForm
// transform into sorted GridEntry.s BEFORE binding // transform into sorted GridEntry.s BEFORE binding
// //
var lib = LibraryQueries.GetLibrary_Flat_NoTracking(); var lib = LibraryQueries.GetLibrary_Flat_NoTracking();
var orderedGridEntries = lib var orderedGridEntries = lib
.Select(lb => new GridEntry(lb)).ToList() .Select(lb => new GridEntry(lb)).ToList()
// default load order: sort by author, then series, then title // default load order
.OrderBy(ge => ge.Authors) .OrderByDescending(ge => ge.Purchase_Date)
.ThenBy(ge => ge.Series) //// more advanced example: sort by author, then series, then title
.ThenBy(ge => ge.Title) //.OrderBy(ge => ge.Authors)
// .ThenBy(ge => ge.Series)
// .ThenBy(ge => ge.Title)
.ToList(); .ToList();
// //

1
Scraping/Scraping.csproj
View File

@ -6,7 +6,6 @@
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\AudibleDotCom\AudibleDotCom.csproj" /> <ProjectReference Include="..\AudibleDotCom\AudibleDotCom.csproj" />
<ProjectReference Include="..\DTOs\DTOs.csproj" />
</ItemGroup> </ItemGroup>
</Project> </Project>

0
DTOs/BookDetailDTO.cs → Scraping/UNTESTED/DTOs/BookDetailDTO.cs
View File

0
DTOs/LibraryDTO.cs → Scraping/UNTESTED/DTOs/LibraryDTO.cs
View File

108
ScrapingDomainServices/UNTESTED/DownloadBook.cs
View File

@ -22,48 +22,78 @@ namespace ScrapingDomainServices
&& !await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId); && !await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId);
public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook) public override async Task<StatusHandler> ProcessItemAsync(LibraryBook libraryBook)
{ {
var tempAaxFilename = FileUtility.GetValidFilename( var tempAaxFilename = FileUtility.GetValidFilename(
AudibleFileStorage.DownloadsInProgress, AudibleFileStorage.DownloadsInProgress,
libraryBook.Book.Title, libraryBook.Book.Title,
"aax", "aax",
libraryBook.Book.AudibleProductId); libraryBook.Book.AudibleProductId);
// if getting from full title: // if getting from full title:
// '?' is allowed // '?' is allowed
// colons are inconsistent but not problematic to just leave them // colons are inconsistent but not problematic to just leave them
// - 1 colon: sometimes full title is used. sometimes only the part before the colon is used // - 1 colon: sometimes full title is used. sometimes only the part before the colon is used
// - multple colons: only the part before the final colon is used // - multple colons: only the part before the final colon is used
// e.g. Alien: Out of the Shadows: An Audible Original Drama => Alien: Out of the Shadows // e.g. Alien: Out of the Shadows: An Audible Original Drama => Alien: Out of the Shadows
// in cases where title includes '&', just use everything before the '&' and ignore the rest // in cases where title includes '&', just use everything before the '&' and ignore the rest
//// var adhTitle = product.Title.Split('&')[0] //// var adhTitle = product.Title.Split('&')[0]
var aaxDownloadLink = libraryBook.DownloadBookLink // legacy/scraping method
.Replace("/admhelper", "") //await performDownloadAsync(libraryBook, tempAaxFilename);
.Replace("&DownloadType=Now", "") // new/api method
+ "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download"; tempAaxFilename = await performApiDownloadAsync(libraryBook, tempAaxFilename);
var uri = new Uri(aaxDownloadLink);
using var webClient = await GetWebClient(tempAaxFilename); // move
// for book downloads only: pretend to be the audible download manager. from inAudible: var aaxFilename = FileUtility.GetValidFilename(
webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601"; AudibleFileStorage.DownloadsFinal,
await webClient.DownloadFileTaskAsync(uri, tempAaxFilename); libraryBook.Book.Title,
"aax",
libraryBook.Book.AudibleProductId);
File.Move(tempAaxFilename, aaxFilename);
// move var statusHandler = new StatusHandler();
var aaxFilename = FileUtility.GetValidFilename( var isDownloaded = await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId);
AudibleFileStorage.DownloadsFinal, if (isDownloaded)
libraryBook.Book.Title, DoStatusUpdate($"Downloaded: {aaxFilename}");
"aax", else
libraryBook.Book.AudibleProductId); statusHandler.AddError("Downloaded AAX file cannot be found");
File.Move(tempAaxFilename, aaxFilename); return statusHandler;
}
var statusHandler = new StatusHandler(); // GetWebClientAsync:
var isDownloaded = await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId); // wires up webClient events
if (isDownloaded) // [DownloadProgressChanged, DownloadFileCompleted, DownloadDataCompleted, DownloadStringCompleted]
DoStatusUpdate($"Downloaded: {aaxFilename}"); // to DownloadableBase events
else // DownloadProgressChanged, DownloadCompleted
statusHandler.AddError("Downloaded AAX file cannot be found"); // fires DownloadBegin event
return statusHandler; // method begins async file download
} private async Task performDownloadAsync(LibraryBook libraryBook, string tempAaxFilename)
} {
var aaxDownloadLink = libraryBook.DownloadBookLink
.Replace("/admhelper", "")
.Replace("&DownloadType=Now", "")
+ "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download";
var uri = new Uri(aaxDownloadLink);
using var webClient = await GetWebClientAsync(tempAaxFilename);
// for book downloads only: pretend to be the audible download manager. from inAudible:
webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601";
await webClient.DownloadFileTaskAsync(uri, tempAaxFilename);
}
private async Task<string> performApiDownloadAsync(LibraryBook libraryBook, string tempAaxFilename)
{
var identityFilePath = Path.Combine(Configuration.Instance.LibationFiles, "IdentityTokens.json");
var api = await AudibleApi.EzApiCreator.GetApiAsync(identityFilePath);
var progress = new Progress<Dinah.Core.Net.Http.DownloadProgress>();
progress.ProgressChanged += (_, e) => Invoke_DownloadProgressChanged(this, e);
Invoke_DownloadBegin(tempAaxFilename);
var actualFilePath = await api.DownloadAaxWorkaroundAsync(libraryBook.Book.AudibleProductId, tempAaxFilename, progress);
Invoke_DownloadCompleted(this, $"Completed: {actualFilePath}");
return actualFilePath;
}
}
} }

2
ScrapingDomainServices/UNTESTED/DownloadPdf.cs
View File

@ -43,7 +43,7 @@ namespace ScrapingDomainServices
var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url)); var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url));
using var webClient = await GetWebClient(destinationFilename); using var webClient = await GetWebClientAsync(destinationFilename);
await webClient.DownloadFileTaskAsync(url, destinationFilename); await webClient.DownloadFileTaskAsync(url, destinationFilename);
var statusHandler = new StatusHandler(); var statusHandler = new StatusHandler();

48
ScrapingDomainServices/UNTESTED/DownloadableBase.cs
View File

@ -7,18 +7,23 @@ using Dinah.Core.Humanizer;
namespace ScrapingDomainServices namespace ScrapingDomainServices
{ {
public abstract class DownloadableBase : IDownloadable public abstract class DownloadableBase : IDownloadable
{ {
public event EventHandler<string> Begin; public event EventHandler<string> Begin;
public event EventHandler<string> StatusUpdate; public event EventHandler<string> StatusUpdate;
protected void DoStatusUpdate(string message) => StatusUpdate?.Invoke(this, message); protected void DoStatusUpdate(string message) => StatusUpdate?.Invoke(this, message);
public event EventHandler<string> DownloadBegin; public event EventHandler<string> DownloadBegin;
public event DownloadProgressChangedEventHandler DownloadProgressChanged; public event EventHandler<Dinah.Core.Net.Http.DownloadProgress> DownloadProgressChanged;
public event EventHandler<string> DownloadCompleted; public event EventHandler<string> DownloadCompleted;
public event EventHandler<string> Completed; protected void Invoke_DownloadBegin(string downloadMessage) => DownloadBegin?.Invoke(this, downloadMessage);
protected void Invoke_DownloadProgressChanged(object sender, Dinah.Core.Net.Http.DownloadProgress progress) => DownloadProgressChanged?.Invoke(sender, progress);
protected void Invoke_DownloadCompleted(object sender, string str) => DownloadCompleted?.Invoke(sender, str);
public event EventHandler<string> Completed;
static DownloadableBase() static DownloadableBase()
{ {
@ -54,13 +59,14 @@ namespace ScrapingDomainServices
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36",
}; "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36",
};
// we need a minimum delay between tries when hitting audible.com // we need a minimum delay between tries when hitting audible.com
// in every case except decrypt (which is already long running), we hit audible.com // in every case except decrypt (which is already long running), we hit audible.com
static Humanizer humanizer { get; } = new Humanizer { Minimum = 5, Maximum = 20 }; static Humanizer humanizer { get; } = new Humanizer { Minimum = 5, Maximum = 20 };
static Random rnd = new Random(); static Random rnd { get; } = new Random();
protected async Task<WebClient> GetWebClient(string downloadMessage) protected async Task<WebClient> GetWebClientAsync(string downloadMessage)
{ {
await humanizer.Wait(); await humanizer.Wait();
@ -75,16 +81,16 @@ namespace ScrapingDomainServices
webClient.Headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"; webClient.Headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
webClient.Headers["Accept-Language"] = "en-US,en;q=0.9"; webClient.Headers["Accept-Language"] = "en-US,en;q=0.9";
// this breaks pdf download which uses: http://download.audible.com // this breaks pdf download which uses: http://download.audible.com
// weirdly, it works for book download even though it uses https://cds.audible.com // weirdly, it works for book download even though it uses https://cds.audible.com
//webClient.Headers["Host"] = "www.audible.com"; //webClient.Headers["Host"] = "www.audible.com";
webClient.DownloadProgressChanged += (s, e) => DownloadProgressChanged?.Invoke(s, e); webClient.DownloadProgressChanged += (s, e) => Invoke_DownloadProgressChanged(s, new Dinah.Core.Net.Http.DownloadProgress { BytesReceived = e.BytesReceived, ProgressPercentage = e.ProgressPercentage, TotalBytesToReceive = e.TotalBytesToReceive });
webClient.DownloadFileCompleted += (s, e) => DownloadCompleted?.Invoke(s, $"Completed: {downloadMessage}"); webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
webClient.DownloadDataCompleted += (s, e) => DownloadCompleted?.Invoke(s, $"Completed: {downloadMessage}"); webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
webClient.DownloadStringCompleted += (s, e) => DownloadCompleted?.Invoke(s, $"Completed: {downloadMessage}"); webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}");
DownloadBegin?.Invoke(this, downloadMessage); Invoke_DownloadBegin(downloadMessage);
return webClient; return webClient;
} }

13
ScrapingDomainServices/UNTESTED/DtoImporter.cs
View File

@ -93,7 +93,7 @@ namespace ScrapingDomainServices
// if no narrators listed, author is the narrator // if no narrators listed, author is the narrator
if (!libraryDTO.Narrators.Any()) if (!libraryDTO.Narrators.Any())
libraryDTO.Narrators = libraryDTO.Narrators = authors.Select(a => a.Name).ToArray(); libraryDTO.Narrators = authors.Select(a => a.Name).ToArray();
// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db // nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
var narrators = libraryDTO var narrators = libraryDTO
@ -102,18 +102,19 @@ namespace ScrapingDomainServices
.ToList(); .ToList();
book = context.Books.Add(new Book( book = context.Books.Add(new Book(
new AudibleProductId(libraryDTO.ProductId), libraryDTO.Title, libraryDTO.Description, libraryDTO.LengthInMinutes, authors, narrators)) new AudibleProductId(libraryDTO.ProductId), libraryDTO.Title, libraryDTO.Description, libraryDTO.LengthInMinutes, authors))
.Entity; .Entity;
book.ReplaceNarrators(narrators);
} }
// set/update book-specific info which may have changed // set/update book-specific info which may have changed
book.PictureId = libraryDTO.PictureId; book.PictureId = libraryDTO.PictureId;
book.UpdateProductRating(libraryDTO.Product_OverallStars, libraryDTO.Product_PerformanceStars, libraryDTO.Product_StoryStars); book.UpdateProductRating(libraryDTO.Product_OverallStars, libraryDTO.Product_PerformanceStars, libraryDTO.Product_StoryStars);
foreach (var url in libraryDTO.SupplementUrls) foreach (var url in libraryDTO.SupplementUrls)
book.AddSupplementDownloadUrl(url); book.AddSupplementDownloadUrl(FileManager.FileUtility.RestoreDeclawed(url));
// important to update user-specific info. this will have changed if user has rated/reviewed the book since last library import // important to update user-specific info. this will have changed if user has rated/reviewed the book since last library import
book.UserDefinedItem.UpdateRating(libraryDTO.MyUserRating_Overall, libraryDTO.MyUserRating_Performance, libraryDTO.MyUserRating_Story); book.UserDefinedItem.UpdateRating(libraryDTO.MyUserRating_Overall, libraryDTO.MyUserRating_Performance, libraryDTO.MyUserRating_Story);
// update series even for existing books. these are occasionally updated // update series even for existing books. these are occasionally updated
var seriesIds = libraryDTO.Series.Select(kvp => kvp.Key).ToList(); var seriesIds = libraryDTO.Series.Select(kvp => kvp.Key).ToList();

3
ScrapingDomainServices/UNTESTED/IDownloadable.cs
View File

@ -1,12 +1,11 @@
using System; using System;
using System.Net;
namespace ScrapingDomainServices namespace ScrapingDomainServices
{ {
public interface IDownloadable : IProcessable public interface IDownloadable : IProcessable
{ {
event EventHandler<string> DownloadBegin; event EventHandler<string> DownloadBegin;
event DownloadProgressChangedEventHandler DownloadProgressChanged; event EventHandler<Dinah.Core.Net.Http.DownloadProgress> DownloadProgressChanged;
event EventHandler<string> DownloadCompleted; event EventHandler<string> DownloadCompleted;
} }
} }

9
ScrapingDomainServices/UNTESTED/IProcessableExt.cs
View File

@ -1,6 +1,7 @@
using System; using System;
using System.Threading.Tasks; using System.Threading.Tasks;
using DataLayer; using DataLayer;
using Dinah.Core.Collections.Generic;
using Dinah.Core.ErrorHandling; using Dinah.Core.ErrorHandling;
namespace ScrapingDomainServices namespace ScrapingDomainServices
@ -37,8 +38,12 @@ namespace ScrapingDomainServices
var libraryBooks = LibraryQueries.GetLibrary_Flat_NoTracking(); var libraryBooks = LibraryQueries.GetLibrary_Flat_NoTracking();
foreach (var libraryBook in libraryBooks) foreach (var libraryBook in libraryBooks)
if (await processable.ValidateAsync(libraryBook)) if (
return libraryBook; // hardcoded blacklist
//episodes
!libraryBook.Book.AudibleProductId.In("B079ZTTL4J", "B0779LK1TX", "B0779H7B38", "B0779M3KGC", "B076PQ6G9Z", "B07D4M18YC") &&
await processable.ValidateAsync(libraryBook))
return libraryBook;
return null; return null;
} }

28
ScrapingDomainServices/UNTESTED/Indexer.cs
View File

@ -26,7 +26,8 @@ namespace ScrapingDomainServices
public static async Task<(int total, int newEntries)> IndexLibraryAsync(List<FileInfo> jsonFileInfos) public static async Task<(int total, int newEntries)> IndexLibraryAsync(List<FileInfo> jsonFileInfos)
{ {
var productItems = jsonFileInfos.SelectMany(fi => json2libraryDtos(fi)).ToList(); var productItems = jsonFileInfos.SelectMany(fi => json2libraryDtos(fi)).ToList();
return await IndexLibraryAsync(productItems); var newEntries = await IndexLibraryAsync(productItems);
return (productItems.Count, newEntries);
} }
private static Regex jsonIsCollectionRegex = new Regex(@"^\s*\[\s*\{", RegexOptions.Compiled); private static Regex jsonIsCollectionRegex = new Regex(@"^\s*\[\s*\{", RegexOptions.Compiled);
private static IEnumerable<LibraryDTO> json2libraryDtos(FileInfo jsonFileInfo) private static IEnumerable<LibraryDTO> json2libraryDtos(FileInfo jsonFileInfo)
@ -44,13 +45,11 @@ namespace ScrapingDomainServices
} }
// new full index or library-file import: re-create search index // new full index or library-file import: re-create search index
public static async Task<(int total, int newEntries)> IndexLibraryAsync(List<LibraryDTO> productItems) /// <returns>qty new entries</returns>
=> await IndexLibraryAsync(productItems, SearchEngineActions.FullReIndexAsync); public static async Task<int> IndexLibraryAsync(List<LibraryDTO> productItems)
private static async Task<(int total, int newEntries)> IndexLibraryAsync(List<LibraryDTO> productItems, Func<Task> postIndexActionAsync)
{ {
if (productItems == null || !productItems.Any()) if (productItems == null || !productItems.Any())
return (0, 0); return 0;
productItems = filterAndValidate(productItems); productItems = filterAndValidate(productItems);
@ -74,9 +73,9 @@ namespace ScrapingDomainServices
await Task.Run(() => dtoImporter.ReloadBookDetails(productItems)); await Task.Run(() => dtoImporter.ReloadBookDetails(productItems));
await context.SaveChangesAsync(); await context.SaveChangesAsync();
await postIndexActionAsync?.Invoke(); await SearchEngineActions.FullReIndexAsync();
return (productItems.Count, newEntries); return newEntries;
} }
private static List<LibraryDTO> filterAndValidate(List<LibraryDTO> collection) private static List<LibraryDTO> filterAndValidate(List<LibraryDTO> collection)
{ {
@ -121,19 +120,6 @@ namespace ScrapingDomainServices
#endregion #endregion
#region book details #region book details
public static async Task IndexBookDetailsAsync(FileInfo jsonFileInfo)
{
var bookDetailDTO = json2bookDetailDto(jsonFileInfo);
await IndexBookDetailsAsync(bookDetailDTO);
}
private static BookDetailDTO json2bookDetailDto(FileInfo jsonFileInfo)
{
validateJsonFile(jsonFileInfo);
var serialized = File.ReadAllText(jsonFileInfo.FullName);
return JsonConvert.DeserializeObject<BookDetailDTO>(serialized);
}
public static async Task IndexBookDetailsAsync(BookDetailDTO bookDetailDTO) public static async Task IndexBookDetailsAsync(BookDetailDTO bookDetailDTO)
=> await indexBookDetailsAsync(bookDetailDTO, () => SearchEngineActions.ProductReIndexAsync(bookDetailDTO.ProductId)); => await indexBookDetailsAsync(bookDetailDTO, () => SearchEngineActions.ProductReIndexAsync(bookDetailDTO.ProductId));

2
ScrapingDomainServices/UNTESTED/ScrapeBookDetails.cs
View File

@ -72,7 +72,7 @@ namespace ScrapingDomainServices
// download htm // download htm
string source; string source;
var url = AudiblePage.Product.GetUrl(productId); var url = AudiblePage.Product.GetUrl(productId);
using var webClient = await GetWebClient($"Getting Book Details for {libraryBook.Book.Title}"); using var webClient = await GetWebClientAsync($"Getting Book Details for {libraryBook.Book.Title}");
try try
{ {
source = await webClient.DownloadStringTaskAsync(url); source = await webClient.DownloadStringTaskAsync(url);

188
__TODO.txt
View File

@ -1,4 +1,121 @@
-- begin CONFIG FILES --------------------------------------------------------------------------------------------------------------------- -- begin REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
integrate API into libation. replace all authentication, audible communication
IN-PROGRESS
-----------
download via api with UI integration
- functionality: episodes
library import UI
- disable main ui
- updates on which stage and how long it's expected to take
- error handling
- dialog/pop up when done. show how many new and total
move biz logic out of UI (Form1.scanLibraryToolStripMenuItem_Click)
- non-db dependent: InternalUtilities. see: SearchEngineActions
InternalUtilities.AudibleApiExtensions may not belong here. not sure
- db dependent: eg DtoImporterService
extract IdentityTokens.json into FileManager
replace all hardcoded occurances
FIX
// hardcoded blacklist
datalayer stuff (eg: Book) need better ToString
MOVE TO LEGACY
--------------
- scraping code
- "legacy inAudible wire-up code"
Book.IsEpisodes
Book.HasBookDetails
LibraryBook.DownloadBookLink
humanizer
replace all scraping with audible api
public partial class ScanLibraryDialog : Form, IIndexLibraryDialog
public async Task DoMainWorkAsync()
using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
scraping stuff to remove. order matters. go from top to bottom:
REMOVED PROJECT REFERENCES
3.2 Domain Utilities (post database)
DomainServices
AudibleDotComAutomation
3.1 Domain Internal Utilities
InternalUtilities
Scraping
REMOVED PROJECTS
2 Utilities (domain ignorant)
Scraping
AudibleDotComAutomation
AudibleDotCom
CookieMonster
REMOVED FILES
\Libation\InternalUtilities\UNTESTED\DataConverter.cs
\Libation\DomainServices\UNTESTED\DownloadLibrary.cs
\Libation\DomainServices\UNTESTED\ScrapeBookDetails.cs
ADDED PROJECT REFERENCES
3.1 Domain Internal Utilities
InternalUtilities
DTOs
-- end REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, PERFORMANCE: IMPORT ---------------------------------------------------------------------------------------------------------------------
imports are PAINFULLY slow for just a few hundred items. wtf is taking so long?
-- end ENHANCEMENT, PERFORMANCE: IMPORT ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
when a book/pdf is NOT liberated, calculating the grid's [Liberated][NOT d/l'ed] label is very slow
https://stackoverflow.com/a/12046333
https://codereview.stackexchange.com/a/135074
// do NOT use lock() or Monitor with async/await
private static int _lockFlag = 0; // 0 - free
if (Interlocked.CompareExchange(ref _lockFlag, 1, 0) != 0) return;
// only 1 thread will enter here without locking the object/put the other threads to sleep
try { await DoWorkAsync(); }
// free the lock
finally { Interlocked.Decrement(ref _lockFlag); }
use stop light icons for liberated state: red=none, yellow=downloaded encrypted, green=liberated
need a way to liberate ad hoc books and pdf.s
use pdf icon with and without and X over it to indicate status
-- end ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENTS, GET LIBRARY ---------------------------------------------------------------------------------------------------------------------
Audible API. GET /1.0/library , GET /1.0/library/{asin}
TONS of expensive conversion: GetLibraryAsync > string > JObject > string > LibraryApiV10
-- end ENHANCEMENTS, GET LIBRARY ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENTS, EPISODES ---------------------------------------------------------------------------------------------------------------------
grid: episodes need a better Download_Status and Length/LengthInMinutes
download: need a way to liberate episodes. show in grid 'x/y downloaded/liberated' etc
-- end ENHANCEMENTS, EPISODES ---------------------------------------------------------------------------------------------------------------------
-- begin BUG, MOVING FILES ---------------------------------------------------------------------------------------------------------------------
with libation closed, move files
start libation
can get error below
fixed on restart
Form1_Load ... await setBackupCountsAsync();
Collection was modified; enumeration operation may not execute.
stack trace
at System.ThrowHelper.ThrowInvalidOperationException_InvalidOperation_EnumFailedVersion()
at System.Collections.Generic.List`1.Enumerator.MoveNextRare()
at System.Linq.Enumerable.SingleOrDefault[TSource](IEnumerable`1 source, Func`2 predicate)
at FileManager.FilePathCache.GetPath(String id, FileType type) in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\FileManager\UNTESTED\FilePathCache.cs:line 33
at FileManager.AudibleFileStorage.<getAsync>d__32.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\FileManager\UNTESTED\AudibleFileStorage.cs:line 112
at FileManager.AudibleFileStorage.<GetAsync>d__31.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\FileManager\UNTESTED\AudibleFileStorage.cs:line 107
at FileManager.AudibleFileStorage.<ExistsAsync>d__30.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\FileManager\UNTESTED\AudibleFileStorage.cs:line 104
at LibationWinForm.Form1.<<setBookBackupCountsAsync>g__getAudioFileStateAsync|15_1>d.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\LibationWinForm\UNTESTED\Form1.cs:line 110
at LibationWinForm.Form1.<setBookBackupCountsAsync>d__15.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\LibationWinForm\UNTESTED\Form1.cs:line 117
at LibationWinForm.Form1.<setBackupCountsAsync>d__13.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\LibationWinForm\UNTESTED\Form1.cs:line 81
at LibationWinForm.Form1.<Form1_Load>d__11.MoveNext() in C:\Dropbox\Dinah's folder\coding\_NET\Visual Studio 2019\Libation\LibationWinForm\UNTESTED\Form1.cs:line 60
-- end BUG, MOVING FILES ---------------------------------------------------------------------------------------------------------------------
-- begin CONFIG FILES ---------------------------------------------------------------------------------------------------------------------
.\appsettings.json should only be a pointer to the real settings file location: LibationSettings.json .\appsettings.json should only be a pointer to the real settings file location: LibationSettings.json
replace complex config saving throughout with new way in my ConsoleDependencyInjection solution replace complex config saving throughout with new way in my ConsoleDependencyInjection solution
all settings should be strongly typed all settings should be strongly typed
@ -16,68 +133,21 @@ BasePath => recursively search directories upward-only until fild dir with .sln
from here can set up a shared dir anywhere. use recursive upward search to find it. store shared files here. eg: identityTokens.json, ComputedTestValues from here can set up a shared dir anywhere. use recursive upward search to find it. store shared files here. eg: identityTokens.json, ComputedTestValues
-- end CONFIG FILES --------------------------------------------------------------------------------------------------------------------- -- end CONFIG FILES ---------------------------------------------------------------------------------------------------------------------
-- begin REPLACE SCRAPING WITH API --------------------------------------------------------------------------------------------------------------------- -- begin TAGS ---------------------------------------------------------------------------------------------------------------------
integrate API into libation. eventually entirely replace all other authentication methods, audible communication. incl book download pulling previous tags into new Books. think: reloading db
logged-in user's own ratings (aka rating-star(s)) -- is "provided_review" move out of Book and into DtoMapper?
unlike traditional scraping, audible api calls do not need to be humanized. can likely delete humanizer altogether. still needed for downloading pdf.s?
incl episodes. eg: "Bill Bryson's Appliance of Science" Extract file and tag stuff from domain objects. This should exist only in data layer. If domain objects are able to call EF context, it should go through data layer
refining episode retrieval might also get rid of the need for IsEpisodes property Why are tags in file AND database?
replace all scraping with audible api
public partial class ScanLibraryDialog : Form, IIndexLibraryDialog
public async Task DoMainWorkAsync()
using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
move old DTOs back into scraping so it's easier to move them all to new legacy area extract FileManager dependency from data layer
library to return strongly typed LibraryApiV10 -- end TAGS ---------------------------------------------------------------------------------------------------------------------
note: some json importing may still be relevant. might want to allow custom importing
scraping stuff to remove:
order matters. go from top to bottom:
REMOVED PROJECT REFERENCES
3.2 Domain Utilities (post database)
DomainServices
AudibleDotComAutomation
3.1 Domain Internal Utilities
InternalUtilities
Scraping
REMOVED PROJECTS
2 Utilities (domain ignorant)
Scraping
AudibleDotComAutomation
AudibleDotCom
CookieMonster
REMOVED FILES
\Libation\InternalUtilities\UNTESTED\DataConverter.cs
\Libation\DomainServices\UNTESTED\DownloadLibrary.cs
\Libation\DomainServices\UNTESTED\ScrapeBookDetails.cs
ADDED PROJECT REFERENCES
3.1 Domain Internal Utilities
InternalUtilities
DTOs
Audible API. GET /1.0/library , GET /1.0/library/{asin}
TONS of expensive conversion: GetLibraryAsync > string > JObject > string > LibraryApiV10
-- end REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, CATEGORIES --------------------------------------------------------------------------------------------------------------------- -- begin ENHANCEMENT, CATEGORIES ---------------------------------------------------------------------------------------------------------------------
add support for multiple categories add support for multiple categories
when i do this, learn about the different CategoryLadder.Root enums. probably only need Root.Genres when i do this, learn about the different CategoryLadder.Root enums. probably only need Root.Genres
-- end ENHANCEMENT, CATEGORIES --------------------------------------------------------------------------------------------------------------------- -- end ENHANCEMENT, CATEGORIES ---------------------------------------------------------------------------------------------------------------------
-- begin LEGACY ---------------------------------------------------------------------------------------------------------------------
retain legacy functionality in out-of-the-way area. turn on/off in settings?
> scraping code
> "legacy inAudible wire-up code"
-- end LEGACY ---------------------------------------------------------------------------------------------------------------------
-- begin CLEAN UP ARCHITECTURE --------------------------------------------------------------------------------------------------------------------- -- begin CLEAN UP ARCHITECTURE ---------------------------------------------------------------------------------------------------------------------
my ui sucks. it's also tightly coupled with biz logic. can't replace ui until biz logic is extracted and loosely. remove all biz logic from presentation/winforms layer my ui sucks. it's also tightly coupled with biz logic. can't replace ui until biz logic is extracted and loosely. remove all biz logic from presentation/winforms layer
-- end CLEAN UP ARCHITECTURE --------------------------------------------------------------------------------------------------------------------- -- end CLEAN UP ARCHITECTURE ---------------------------------------------------------------------------------------------------------------------
@ -98,6 +168,9 @@ Turn into unit tests or demos
search 'example code' on: LibationWinForm\...\Form1.cs search 'example code' on: LibationWinForm\...\Form1.cs
EnumerationFlagsExtensions.EXAMPLES() EnumerationFlagsExtensions.EXAMPLES()
// examples // examples
scratchpad
scratch pad
scratch_pad
-- end UNIT TESTS --------------------------------------------------------------------------------------------------------------------- -- end UNIT TESTS ---------------------------------------------------------------------------------------------------------------------
-- begin DECRYPTING --------------------------------------------------------------------------------------------------------------------- -- begin DECRYPTING ---------------------------------------------------------------------------------------------------------------------
@ -128,6 +201,7 @@ directly call ffmpeg (decrypt only)
-- begin ENHANCEMENT, UI: LONG RUNNING TASKS --------------------------------------------------------------------------------------------------------------------- -- begin ENHANCEMENT, UI: LONG RUNNING TASKS ---------------------------------------------------------------------------------------------------------------------
long running tasks are appropriately async. however there's no way for the user to see that the task is running (vs nothing happened) except to wait and see if the final notification ever comes long running tasks are appropriately async. however there's no way for the user to see that the task is running (vs nothing happened) except to wait and see if the final notification ever comes
need events to update UI with progress
-- end ENHANCEMENT, UI: LONG RUNNING TASKS --------------------------------------------------------------------------------------------------------------------- -- end ENHANCEMENT, UI: LONG RUNNING TASKS ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, PERFORMANCE: TAGS --------------------------------------------------------------------------------------------------------------------- -- begin ENHANCEMENT, PERFORMANCE: TAGS ---------------------------------------------------------------------------------------------------------------------
@ -136,10 +210,6 @@ unlikely to be an issue with file write. in fact, should probably roll back this
also touches parts of code which: db write via a hook, search engine re-index also touches parts of code which: db write via a hook, search engine re-index
-- end ENHANCEMENT, PERFORMANCE: TAGS --------------------------------------------------------------------------------------------------------------------- -- end ENHANCEMENT, PERFORMANCE: TAGS ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
when a book/pdf is NOT liberated, calculating the grid's [Liberated][NOT d/l'ed] label is very slow
-- end ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT: REMOVE BOOK --------------------------------------------------------------------------------------------------------------------- -- begin ENHANCEMENT: REMOVE BOOK ---------------------------------------------------------------------------------------------------------------------
how to remove a book? how to remove a book?
previously difficult due to implementation details regarding scraping and importing. should be trivial after api replaces scraping previously difficult due to implementation details regarding scraping and importing. should be trivial after api replaces scraping