tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finalize move from library scraping to api

Browse Source
This commit is contained in:
Robert McRackan 2019-11-04 23:07:40 -05:00
parent 55b57cf9a9
commit 664fcc50e2
29 changed files with 604 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
ApplicationService/ApplicationService.csproj Normal file
View File

@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApiDTOs\AudibleApiDTOs.csproj" />
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
<ProjectReference Include="..\DtoImporterService\DtoImporterService.csproj" />
<ProjectReference Include="..\InternalUtilities\InternalUtilities.csproj" />
<ProjectReference Include="..\LibationSearchEngine\LibationSearchEngine.csproj" />
</ItemGroup>
</Project>

25
ApplicationService/LibraryIndexer.cs Normal file
View File

@ -0,0 +1,25 @@
using System;
using System.Threading.Tasks;
using AudibleApi;
using DtoImporterService;
using InternalUtilities;
namespace ApplicationService
{
public class LibraryIndexer
{
public async Task<(int totalCount, int newCount)> IndexAsync(ILoginCallback callback)
{
var audibleApiActions = new AudibleApiActions();
var items = await audibleApiActions.GetAllLibraryItemsAsync(callback);
var totalCount = items.Count;
var libImporter = new LibraryImporter();
var newCount = await Task.Run(() => libImporter.Import(items));
await SearchEngineActions.FullReIndexAsync();
return (totalCount, newCount);
}
}
}

26
ApplicationService/SearchEngineActions.cs Normal file
View File

@ -0,0 +1,26 @@
using System.Threading.Tasks;
using DataLayer;
namespace ApplicationService
{
public static class SearchEngineActions
{
public static async Task FullReIndexAsync()
{
var engine = new LibationSearchEngine.SearchEngine();
await engine.CreateNewIndexAsync().ConfigureAwait(false);
}
public static void UpdateBookTags(Book book)
{
var engine = new LibationSearchEngine.SearchEngine();
engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
}
public static async Task ProductReIndexAsync(string productId)
{
var engine = new LibationSearchEngine.SearchEngine();
await engine.UpdateBookAsync(productId).ConfigureAwait(false);
}
}
}

15
DtoImporterService/BookImporter.cs
View File

@ -3,24 +3,13 @@ using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
using InternalUtilities;
namespace DtoImporterService
{
public class BookImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.Any(i => string.IsNullOrWhiteSpace(i.ProductId)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with blank {nameof(Item.ProductId)}", nameof(items)));
if (items.Any(i => string.IsNullOrWhiteSpace(i.Title)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with blank {nameof(Item.Title)}", nameof(items)));
if (items.Any(i => i.Authors is null))
exceptions.Add(new ArgumentException($"Collection contains item(s) with null {nameof(Item.Authors)}", nameof(items)));
return exceptions;
}
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new BookValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{

17
DtoImporterService/CategoryImporter.cs
View File

@ -3,26 +3,13 @@ using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
using InternalUtilities;
namespace DtoImporterService
{
public class CategoryImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
var distinct = items.GetCategoriesDistinct();
if (distinct.Any(s => s.CategoryId is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with null {nameof(Ladder.CategoryId)}", nameof(items)));
if (distinct.Any(s => s.CategoryName is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with null {nameof(Ladder.CategoryName)}", nameof(items)));
if (items.GetCategoryPairsDistinct().Any(p => p.Length > 2))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with wrong number of categories. Expecting 0, 1, or 2 categories per title", nameof(items)));
return exceptions;
}
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new CategoryValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{

13
DtoImporterService/ContributorImporter.cs
View File

@ -3,22 +3,13 @@ using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
using InternalUtilities;
namespace DtoImporterService
{
public class ContributorImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.GetAuthorsDistinct().Any(a => string.IsNullOrWhiteSpace(a.Name)))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Authors)} with null {nameof(Person.Name)}", nameof(items)));
if (items.GetNarratorsDistinct().Any(a => string.IsNullOrWhiteSpace(a.Name)))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Narrators)} with null {nameof(Person.Name)}", nameof(items)));
return exceptions;
}
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new ContributorValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{

1
DtoImporterService/DtoImporterService.csproj
View File

@ -7,6 +7,7 @@
<ItemGroup>
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApiDTOs\AudibleApiDTOs.csproj" />
<ProjectReference Include="..\DataLayer\DataLayer.csproj" />
<ProjectReference Include="..\InternalUtilities\InternalUtilities.csproj" />
</ItemGroup>
</Project>

13
DtoImporterService/LibraryImporter.cs
View File

@ -3,22 +3,13 @@ using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
using InternalUtilities;
namespace DtoImporterService
{
public class LibraryImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.Any(i => string.IsNullOrWhiteSpace(i.ProductId)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with null or blank {nameof(Item.ProductId)}", nameof(items)));
if (items.Any(i => i.DateAdded < new DateTime(1980, 1, 1)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with invalid {nameof(Item.DateAdded)}", nameof(items)));
return exceptions;
}
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new LibraryValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{

14
DtoImporterService/SeriesImporter.cs
View File

@ -3,23 +3,13 @@ using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
using DataLayer;
using InternalUtilities;
namespace DtoImporterService
{
public class SeriesImporter : ItemsImporterBase
{
public override IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
var distinct = items .GetSeriesDistinct();
if (distinct.Any(s => s.SeriesId is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Series)} with null {nameof(AudibleApiDTOs.Series.SeriesId)}", nameof(items)));
if (distinct.Any(s => s.SeriesName is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Series)} with null {nameof(AudibleApiDTOs.Series.SeriesName)}", nameof(items)));
return exceptions;
}
public override IEnumerable<Exception> Validate(IEnumerable<Item> items) => new SeriesValidator().Validate(items);
protected override int DoImport(IEnumerable<Item> items, LibationContext context)
{

9
FileManager/UNTESTED/AudibleApiStorage.cs Normal file
View File

@ -0,0 +1,9 @@
using System.IO;
namespace FileManager
{
public static class AudibleApiStorage
{
public static string IdentityTokensFile => Path.Combine(Configuration.Instance.LibationFiles, "IdentityTokens.json");
}
}

1
InternalUtilities/InternalUtilities.csproj
View File

@ -6,7 +6,6 @@
<ItemGroup>
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
<ProjectReference Include="..\LibationSearchEngine\LibationSearchEngine.csproj" />
<ProjectReference Include="..\Scraping\Scraping.csproj" />
</ItemGroup>

76
InternalUtilities/UNTESTED/AudibleApiActions.cs Normal file
View File

@ -0,0 +1,76 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using AudibleApi;
using AudibleApiDTOs;
using FileManager;
namespace InternalUtilities
{
public class AudibleApiActions
{
public async Task<List<Item>> GetAllLibraryItemsAsync(ILoginCallback callback)
{
// bug on audible's side. the 1st time after a long absence, a query to get library will return without titles or authors. a subsequent identical query will be successful. this is true whether or tokens are refreshed
// worse, this 1st dummy call doesn't seem to help:
// var page = await api.GetLibraryAsync(new AudibleApi.LibraryOptions { NumberOfResultPerPage = 1, PageNumber = 1, PurchasedAfter = DateTime.Now.AddYears(-20), ResponseGroups = AudibleApi.LibraryOptions.ResponseGroupOptions.ALL_OPTIONS });
// i don't want to incur the cost of making a full dummy call every time because it fails sometimes
try
{
return await getItemsAsync(callback);
}
catch
{
return await getItemsAsync(callback);
}
}
private async Task<List<Item>> getItemsAsync(ILoginCallback callback)
{
var api = await EzApiCreator.GetApiAsync(AudibleApiStorage.IdentityTokensFile, callback, Configuration.Instance.LocaleCountryCode);
var items = await AudibleApiExtensions.GetAllLibraryItemsAsync(api);
// remove episode parents
items.RemoveAll(i => i.Episodes);
#region // episode handling. doesn't quite work
// // add individual/children episodes
// var childIds = items
// .Where(i => i.Episodes)
// .SelectMany(ep => ep.Relationships)
// .Where(r => r.RelationshipToProduct == AudibleApiDTOs.RelationshipToProduct.Child && r.RelationshipType == AudibleApiDTOs.RelationshipType.Episode)
// .Select(c => c.Asin)
// .ToList();
// foreach (var childId in childIds)
// {
// var bookResult = await api.GetLibraryBookAsync(childId, AudibleApi.LibraryOptions.ResponseGroupOptions.ALL_OPTIONS);
// var bookItem = AudibleApiDTOs.LibraryApiV10.FromJson(bookResult.ToString()).Item;
// items.Add(bookItem);
// }
#endregion
var validators = new List<IValidator>();
validators.AddRange(getValidators());
foreach (var v in validators)
{
var exceptions = v.Validate(items);
if (exceptions != null && exceptions.Any())
throw new AggregateException(exceptions);
}
return items;
}
private static List<IValidator> getValidators()
{
var type = typeof(IValidator);
var types = AppDomain.CurrentDomain.GetAssemblies()
.SelectMany(s => s.GetTypes())
.Where(p => type.IsAssignableFrom(p) && !p.IsInterface);
return types.Select(t => Activator.CreateInstance(t) as IValidator).ToList();
}
}
}

89
InternalUtilities/UNTESTED/AudibleApiValidators.cs Normal file
View File

@ -0,0 +1,89 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AudibleApiDTOs;
namespace InternalUtilities
{
public interface IValidator
{
IEnumerable<Exception> Validate(IEnumerable<Item> items);
}
public class LibraryValidator : IValidator
{
public IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.Any(i => string.IsNullOrWhiteSpace(i.ProductId)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with null or blank {nameof(Item.ProductId)}", nameof(items)));
if (items.Any(i => i.DateAdded < new DateTime(1980, 1, 1)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with invalid {nameof(Item.DateAdded)}", nameof(items)));
return exceptions;
}
}
public class BookValidator : IValidator
{
public IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.Any(i => string.IsNullOrWhiteSpace(i.ProductId)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with blank {nameof(Item.ProductId)}", nameof(items)));
if (items.Any(i => string.IsNullOrWhiteSpace(i.Title)))
exceptions.Add(new ArgumentException($"Collection contains item(s) with blank {nameof(Item.Title)}", nameof(items)));
if (items.Any(i => i.Authors is null))
exceptions.Add(new ArgumentException($"Collection contains item(s) with null {nameof(Item.Authors)}", nameof(items)));
return exceptions;
}
}
public class CategoryValidator : IValidator
{
public IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
var distinct = items.GetCategoriesDistinct();
if (distinct.Any(s => s.CategoryId is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with null {nameof(Ladder.CategoryId)}", nameof(items)));
if (distinct.Any(s => s.CategoryName is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with null {nameof(Ladder.CategoryName)}", nameof(items)));
if (items.GetCategoryPairsDistinct().Any(p => p.Length > 2))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Categories)} with wrong number of categories. Expecting 0, 1, or 2 categories per title", nameof(items)));
return exceptions;
}
}
public class ContributorValidator : IValidator
{
public IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
if (items.GetAuthorsDistinct().Any(a => string.IsNullOrWhiteSpace(a.Name)))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Authors)} with null {nameof(Person.Name)}", nameof(items)));
if (items.GetNarratorsDistinct().Any(a => string.IsNullOrWhiteSpace(a.Name)))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Narrators)} with null {nameof(Person.Name)}", nameof(items)));
return exceptions;
}
}
public class SeriesValidator : IValidator
{
public IEnumerable<Exception> Validate(IEnumerable<Item> items)
{
var exceptions = new List<Exception>();
var distinct = items.GetSeriesDistinct();
if (distinct.Any(s => s.SeriesId is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Series)} with null {nameof(AudibleApiDTOs.Series.SeriesId)}", nameof(items)));
if (distinct.Any(s => s.SeriesName is null))
exceptions.Add(new ArgumentException($"Collection contains {nameof(Item.Series)} with null {nameof(AudibleApiDTOs.Series.SeriesName)}", nameof(items)));
return exceptions;
}
}
}

26
InternalUtilities/UNTESTED/SearchEngineActions.cs
View File

@ -1,26 +0,0 @@
using System.Threading.Tasks;
using DataLayer;
namespace InternalUtilities
{
public static class SearchEngineActions
{
public static async Task FullReIndexAsync()
{
var engine = new LibationSearchEngine.SearchEngine();
await engine.CreateNewIndexAsync().ConfigureAwait(false);
}
public static void UpdateBookTags(Book book)
{
var engine = new LibationSearchEngine.SearchEngine();
engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags);
}
public static async Task ProductReIndexAsync(string productId)
{
var engine = new LibationSearchEngine.SearchEngine();
await engine.UpdateBookAsync(productId).ConfigureAwait(false);
}
}
}

9
Libation.sln
View File

@ -87,6 +87,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiDTOs.Tests", "..\
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiClientExample", "..\audible api\AudibleApi\_Demos\AudibleApiClientExample\AudibleApiClientExample.csproj", "{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApplicationService", "ApplicationService\ApplicationService.csproj", "{B95650EA-25F0-449E-BA5D-99126BC5D730}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -213,6 +215,10 @@ Global
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6}.Release|Any CPU.Build.0 = Release|Any CPU
{B95650EA-25F0-449E-BA5D-99126BC5D730}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B95650EA-25F0-449E-BA5D-99126BC5D730}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B95650EA-25F0-449E-BA5D-99126BC5D730}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B95650EA-25F0-449E-BA5D-99126BC5D730}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -227,7 +233,7 @@ Global
{7BD02E29-3430-4D06-88D2-5CECEE9ABD01} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
{393B5B27-D15C-4F77-9457-FA14BA8F3C73} = {41CDCC73-9B81-49DD-9570-C54406E852AF}
{06882742-27A6-4347-97D9-56162CEC9C11} = {F0CBB7A7-D3FB-41FF-8F47-CF3F6A592249}
{2E1F5DB4-40CC-4804-A893-5DCE0193E598} = {F0CBB7A7-D3FB-41FF-8F47-CF3F6A592249}
{2E1F5DB4-40CC-4804-A893-5DCE0193E598} = {41CDCC73-9B81-49DD-9570-C54406E852AF}
{9F1AA3DE-962F-469B-82B2-46F93491389B} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
{E874D000-AD3A-4629-AC65-7219C2C7C1F0} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{FF12ADA0-8975-4E67-B6EA-4AC82E0C8994} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
@ -248,6 +254,7 @@ Global
{C03C5D65-3B7F-453B-972F-23950B7E0604} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
{B95650EA-25F0-449E-BA5D-99126BC5D730} = {41CDCC73-9B81-49DD-9570-C54406E852AF}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9}

8
LibationWinForm/LibationWinForm.csproj
View File

@ -11,7 +11,9 @@
<ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
<ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj" />
<ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj" />
<ProjectReference Include="..\ApplicationService\ApplicationService.csproj" />
<ProjectReference Include="..\DtoImporterService\DtoImporterService.csproj" />
<ProjectReference Include="..\LibationSearchEngine\LibationSearchEngine.csproj" />
<ProjectReference Include="..\ScrapingDomainServices\ScrapingDomainServices.csproj" />
</ItemGroup>
@ -21,6 +23,12 @@
<AutoGen>True</AutoGen>
<DependentUpon>Resources.resx</DependentUpon>
</Compile>
<Compile Update="UNTESTED\Dialogs\IndexLibraryDialog.cs">
<SubType>Form</SubType>
</Compile>
<Compile Update="UNTESTED\Dialogs\IndexLibraryDialog.Designer.cs">
<DependentUpon>IndexLibraryDialog.cs</DependentUpon>
</Compile>
</ItemGroup>
<ItemGroup>

2
LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs
View File

@ -17,7 +17,7 @@ namespace LibationWinForm
public string StringBasedValidate() => null;
List<string> successMessages = new List<string>();
List<string> successMessages { get; } = new List<string>();
public string SuccessMessage => string.Join("\r\n", successMessages);
public int NewBooksAdded { get; private set; }

66
LibationWinForm/UNTESTED/Dialogs/IndexLibraryDialog.Designer.cs generated Normal file
View File

@ -0,0 +1,66 @@
namespace LibationWinForm
{
partial class IndexLibraryDialog
{
/// <summary>
/// Required designer variable.
/// </summary>
private System.ComponentModel.IContainer components = null;
/// <summary>
/// Clean up any resources being used.
/// </summary>
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
protected override void Dispose(bool disposing)
{
if (disposing && (components != null))
{
components.Dispose();
}
base.Dispose(disposing);
}
#region Windows Form Designer generated code
/// <summary>
/// Required method for Designer support - do not modify
/// the contents of this method with the code editor.
/// </summary>
private void InitializeComponent()
{
this.label1 = new System.Windows.Forms.Label();
this.SuspendLayout();
//
// label1
//
this.label1.AutoSize = true;
this.label1.Location = new System.Drawing.Point(28, 24);
this.label1.Name = "label1";
this.label1.Size = new System.Drawing.Size(260, 13);
this.label1.TabIndex = 0;
this.label1.Text = "Scanning Audible library. This may take a few minutes";
//
// IndexLibraryDialog
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(319, 63);
this.Controls.Add(this.label1);
this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedDialog;
this.MaximizeBox = false;
this.MinimizeBox = false;
this.Name = "IndexLibraryDialog";
this.ShowIcon = false;
this.ShowInTaskbar = false;
this.StartPosition = System.Windows.Forms.FormStartPosition.CenterParent;
this.Text = "Scan Library";
this.ResumeLayout(false);
this.PerformLayout();
}
#endregion
private System.Windows.Forms.Label label1;
}
}

43
LibationWinForm/UNTESTED/Dialogs/IndexLibraryDialog.cs Normal file
View File

@ -0,0 +1,43 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Windows.Forms;
using ApplicationService;
namespace LibationWinForm
{
public partial class IndexLibraryDialog : Form, IIndexLibraryDialog
{
public IndexLibraryDialog()
{
InitializeComponent();
var btn = new Button();
AcceptButton = btn;
btn.Location = new System.Drawing.Point(this.Size.Width + 10, 0);
// required for FindForm() to work
this.Controls.Add(btn);
this.Shown += (_, __) => AcceptButton.PerformClick();
}
public string StringBasedValidate() => null;
List<string> successMessages { get; } = new List<string>();
public string SuccessMessage => string.Join("\r\n", successMessages);
public int NewBooksAdded { get; private set; }
public int TotalBooksProcessed { get; private set; }
public async Task DoMainWorkAsync()
{
var callback = new Login.WinformResponder();
var refresher = new LibraryIndexer();
(TotalBooksProcessed, NewBooksAdded) = await refresher.IndexAsync(callback);
successMessages.Add($"Total processed: {TotalBooksProcessed}");
successMessages.Add($"New: {NewBooksAdded}");
}
}
}

61
LibationWinForm/UNTESTED/Form1.cs
View File

@ -360,66 +360,7 @@ namespace LibationWinForm
// legacy/scraping method
//await indexDialog(new ScanLibraryDialog());
// new/api method
await audibleApi();
}
private async Task audibleApi()
{
var identityFilePath = System.IO.Path.Combine(config.LibationFiles, "IdentityTokens.json");
var callback = new Login.WinformResponder();
var api = await AudibleApi.EzApiCreator.GetApiAsync(identityFilePath, callback, config.LocaleCountryCode);
int totalCount;
int newCount;
// bug on audible's side. the 1st time after a long absence, a query to get library will return without titles or authors. a subsequent identical query will be successful. this is true whether or tokens are refreshed
// worse, this 1st dummy call doesn't seem to help:
// var page = await api.GetLibraryAsync(new AudibleApi.LibraryOptions { NumberOfResultPerPage = 1, PageNumber = 1, PurchasedAfter = DateTime.Now.AddYears(-20), ResponseGroups = AudibleApi.LibraryOptions.ResponseGroupOptions.ALL_OPTIONS });
// i don't want to incur the cost of making a full dummy call every time because it fails sometimes
try
{
var items = await InternalUtilities.AudibleApiExtensions.GetAllLibraryItemsAsync(api);
// remove episode parents
items.RemoveAll(i => i.Episodes);
// // add individual/children episodes
// var childIds = items
// .Where(i => i.Episodes)
// .SelectMany(ep => ep.Relationships)
// .Where(r => r.RelationshipToProduct == AudibleApiDTOs.RelationshipToProduct.Child && r.RelationshipType == AudibleApiDTOs.RelationshipType.Episode)
// .Select(c => c.Asin)
// .ToList();
// foreach (var childId in childIds)
// {
//// clean this up
// var bookResult = await api.GetLibraryBookAsync(childId, AudibleApi.LibraryOptions.ResponseGroupOptions.ALL_OPTIONS);
// var bookResultString = bookResult.ToString();
// var bookResultJson = AudibleApiDTOs.LibraryApiV10.FromJson(bookResultString);
// var bookItem = bookResultJson.Item;
// items.Add(bookItem);
// }
// extract code in 'try' so retry in 'catch' isn't duplicate code
totalCount = items.Count;
newCount = await Task.Run(() => new DtoImporterService.LibraryImporter().Import(items));
}
catch (Exception ex1)
{
try
{
var items = await InternalUtilities.AudibleApiExtensions.GetAllLibraryItemsAsync(api);
items.RemoveAll(i => i.Episodes);
totalCount = items.Count;
newCount = await Task.Run(() => new DtoImporterService.LibraryImporter().Import(items));
}
catch (Exception ex2)
{
MessageBox.Show("Error importing library.\r\n" + ex2.Message, "Error importing library", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
}
await InternalUtilities.SearchEngineActions.FullReIndexAsync();
await indexComplete(totalCount, newCount);
await indexDialog(new IndexLibraryDialog());
}
private async void reimportMostRecentLibraryScanToolStripMenuItem_Click(object sender, EventArgs e)

2
ScrapingDomainServices/ScrapingDomainServices.csproj
View File

@ -6,7 +6,9 @@
<ItemGroup>
<ProjectReference Include="..\AaxDecrypter\AaxDecrypter.csproj" />
<ProjectReference Include="..\ApplicationService\ApplicationService.csproj" />
<ProjectReference Include="..\AudibleDotComAutomation\AudibleDotComAutomation.csproj" />
<ProjectReference Include="..\DataLayer\DataLayer.csproj" />
<ProjectReference Include="..\InternalUtilities\InternalUtilities.csproj" />
</ItemGroup>

3
ScrapingDomainServices/UNTESTED/DownloadBook.cs
View File

@ -83,8 +83,7 @@ tempAaxFilename = await performApiDownloadAsync(libraryBook, tempAaxFilename);
private async Task<string> performApiDownloadAsync(LibraryBook libraryBook, string tempAaxFilename)
{
var identityFilePath = Path.Combine(Configuration.Instance.LibationFiles, "IdentityTokens.json");
var api = await AudibleApi.EzApiCreator.GetApiAsync(identityFilePath);
var api = await AudibleApi.EzApiCreator.GetApiAsync(AudibleApiStorage.IdentityTokensFile);
var progress = new Progress<Dinah.Core.Net.Http.DownloadProgress>();
progress.ProgressChanged += (_, e) => Invoke_DownloadProgressChanged(this, e);

8
ScrapingDomainServices/UNTESTED/IProcessableExt.cs
View File

@ -38,12 +38,8 @@ namespace ScrapingDomainServices
var libraryBooks = LibraryQueries.GetLibrary_Flat_NoTracking();
foreach (var libraryBook in libraryBooks)
if (
// hardcoded blacklist
//episodes
!libraryBook.Book.AudibleProductId.In("B079ZTTL4J", "B0779LK1TX", "B0779H7B38", "B0779M3KGC", "B076PQ6G9Z", "B07D4M18YC") &&
await processable.ValidateAsync(libraryBook))
return libraryBook;
if (await processable.ValidateAsync(libraryBook))
return libraryBook;
return null;
}

2
ScrapingDomainServices/UNTESTED/Indexer.cs
View File

@ -4,12 +4,12 @@ using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using ApplicationService;
using DataLayer;
using Dinah.Core;
using Dinah.Core.Collections.Generic;
using DTOs;
using FileManager;
using InternalUtilities;
using Newtonsoft.Json;
namespace ScrapingDomainServices

66
WinFormsDesigner/Dialogs/IndexLibraryDialog.Designer.cs generated Normal file
View File

@ -0,0 +1,66 @@
namespace LibationWinForm_Framework.Dialogs
{
partial class IndexLibraryDialog
{
/// <summary>
/// Required designer variable.
/// </summary>
private System.ComponentModel.IContainer components = null;
/// <summary>
/// Clean up any resources being used.
/// </summary>
/// <param name="disposing">true if managed resources should be disposed; otherwise, false.</param>
protected override void Dispose(bool disposing)
{
if (disposing && (components != null))
{
components.Dispose();
}
base.Dispose(disposing);
}
#region Windows Form Designer generated code
/// <summary>
/// Required method for Designer support - do not modify
/// the contents of this method with the code editor.
/// </summary>
private void InitializeComponent()
{
this.label1 = new System.Windows.Forms.Label();
this.SuspendLayout();
//
// label1
//
this.label1.AutoSize = true;
this.label1.Location = new System.Drawing.Point(28, 24);
this.label1.Name = "label1";
this.label1.Size = new System.Drawing.Size(260, 13);
this.label1.TabIndex = 0;
this.label1.Text = "Scanning Audible library. This may take a few minutes";
//
// IndexLibraryDialog
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(319, 63);
this.Controls.Add(this.label1);
this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedDialog;
this.MaximizeBox = false;
this.MinimizeBox = false;
this.Name = "IndexLibraryDialog";
this.ShowIcon = false;
this.ShowInTaskbar = false;
this.StartPosition = System.Windows.Forms.FormStartPosition.CenterParent;
this.Text = "Scan Library";
this.ResumeLayout(false);
this.PerformLayout();
}
#endregion
private System.Windows.Forms.Label label1;
}
}

20
WinFormsDesigner/Dialogs/IndexLibraryDialog.cs Normal file
View File

@ -0,0 +1,20 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
namespace LibationWinForm_Framework.Dialogs
{
public partial class IndexLibraryDialog : Form
{
public IndexLibraryDialog()
{
InitializeComponent();
}
}
}

120
WinFormsDesigner/Dialogs/IndexLibraryDialog.resx Normal file
View File

@ -0,0 +1,120 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
Version 2.0
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.
Example:
... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
<resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
<resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
<data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
<data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
<data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
<value>[base64 mime encoded serialized .NET Framework object]</value>
</data>
<data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>
There are any number of "resheader" rows that contain simple
name/value pairs.
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.
mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
<xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
<xsd:element name="root" msdata:IsDataSet="true">
<xsd:complexType>
<xsd:choice maxOccurs="unbounded">
<xsd:element name="metadata">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" />
</xsd:sequence>
<xsd:attribute name="name" use="required" type="xsd:string" />
<xsd:attribute name="type" type="xsd:string" />
<xsd:attribute name="mimetype" type="xsd:string" />
<xsd:attribute ref="xml:space" />
</xsd:complexType>
</xsd:element>
<xsd:element name="assembly">
<xsd:complexType>
<xsd:attribute name="alias" type="xsd:string" />
<xsd:attribute name="name" type="xsd:string" />
</xsd:complexType>
</xsd:element>
<xsd:element name="data">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
<xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
<xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
<xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
<xsd:attribute ref="xml:space" />
</xsd:complexType>
</xsd:element>
<xsd:element name="resheader">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" />
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:schema>
<resheader name="resmimetype">
<value>text/microsoft-resx</value>
</resheader>
<resheader name="version">
<value>2.0</value>
</resheader>
<resheader name="reader">
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
</root>

9
WinFormsDesigner/WinFormsDesigner.csproj
View File

@ -77,6 +77,12 @@
<Compile Include="Dialogs\EditQuickFilters.Designer.cs">
<DependentUpon>EditQuickFilters.cs</DependentUpon>
</Compile>
<Compile Include="Dialogs\IndexLibraryDialog.cs">
<SubType>Form</SubType>
</Compile>
<Compile Include="Dialogs\IndexLibraryDialog.Designer.cs">
<DependentUpon>IndexLibraryDialog.cs</DependentUpon>
</Compile>
<Compile Include="Dialogs\Login\AudibleLoginDialog.cs">
<SubType>Form</SubType>
</Compile>
@ -157,6 +163,9 @@
<EmbeddedResource Include="Dialogs\EditTagsDialog.resx">
<DependentUpon>EditTagsDialog.cs</DependentUpon>
</EmbeddedResource>
<EmbeddedResource Include="Dialogs\IndexLibraryDialog.resx">
<DependentUpon>IndexLibraryDialog.cs</DependentUpon>
</EmbeddedResource>
<EmbeddedResource Include="Dialogs\Login\AudibleLoginDialog.resx">
<DependentUpon>AudibleLoginDialog.cs</DependentUpon>
</EmbeddedResource>

25
__TODO.txt
View File

@ -1,25 +1,11 @@
-- begin REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
integrate API into libation. replace all authentication, audible communication
IN-PROGRESS
-----------
library import UI
- disable main ui
- updates on which stage and how long it's expected to take
- error handling
- dialog/pop up when done. show how many new and total
move biz logic out of UI (Form1.scanLibraryToolStripMenuItem_Click)
- non-db dependent: InternalUtilities. see: SearchEngineActions
InternalUtilities.AudibleApiExtensions may not belong here. not sure
- db dependent: eg DtoImporterService
extract IdentityTokens.json into FileManager
replace all hardcoded occurances
FIX
// hardcoded blacklist
datalayer stuff (eg: Book) need better ToString
MOVE TO LEGACY
--------------
@ -81,15 +67,14 @@ need a way to liberate ad hoc books and pdf.s
use pdf icon with and without and X over it to indicate status
-- end ENHANCEMENT, PERFORMANCE: GRID ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENTS, GET LIBRARY ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, GET LIBRARY ---------------------------------------------------------------------------------------------------------------------
Audible API. GET /1.0/library , GET /1.0/library/{asin}
TONS of expensive conversion: GetLibraryAsync > string > JObject > string > LibraryApiV10
-- end ENHANCEMENTS, GET LIBRARY ---------------------------------------------------------------------------------------------------------------------
-- end ENHANCEMENT, GET LIBRARY ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENTS, EPISODES ---------------------------------------------------------------------------------------------------------------------
grid: episodes need a better Download_Status and Length/LengthInMinutes
download: need a way to liberate episodes. show in grid 'x/y downloaded/liberated' etc
-- end ENHANCEMENTS, EPISODES ---------------------------------------------------------------------------------------------------------------------
-- begin ENHANCEMENT, DEBUGGING ---------------------------------------------------------------------------------------------------------------------
datalayer stuff (eg: Book) need better ToString
-- end ENHANCEMENT, DEBUGGING ---------------------------------------------------------------------------------------------------------------------
-- begin BUG, MOVING FILES ---------------------------------------------------------------------------------------------------------------------
with libation closed, move files