diff --git a/Scraping/UNTESTED/BookDetail/BookDetailDTO.cs b/DTOs/BookDetailDTO.cs similarity index 97% rename from Scraping/UNTESTED/BookDetail/BookDetailDTO.cs rename to DTOs/BookDetailDTO.cs index 648c15de..28a90da5 100644 --- a/Scraping/UNTESTED/BookDetail/BookDetailDTO.cs +++ b/DTOs/BookDetailDTO.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace Scraping.BookDetail +namespace DTOs { public class SeriesEntry { diff --git a/DTOs/DTOs.csproj b/DTOs/DTOs.csproj new file mode 100644 index 00000000..d4c395e8 --- /dev/null +++ b/DTOs/DTOs.csproj @@ -0,0 +1,7 @@ + + + + netstandard2.1 + + + diff --git a/Scraping/UNTESTED/Library/LibraryDTO.cs b/DTOs/LibraryDTO.cs similarity index 98% rename from Scraping/UNTESTED/Library/LibraryDTO.cs rename to DTOs/LibraryDTO.cs index d941711e..4ffaa5c9 100644 --- a/Scraping/UNTESTED/Library/LibraryDTO.cs +++ b/DTOs/LibraryDTO.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace Scraping.Library +namespace DTOs { public class LibraryDTO { diff --git a/InternalUtilities/UNTESTED/DtoImporter.cs b/DomainServices/UNTESTED/DtoImporter.cs similarity index 99% rename from InternalUtilities/UNTESTED/DtoImporter.cs rename to DomainServices/UNTESTED/DtoImporter.cs index 9ae6d2b4..d3986163 100644 --- a/InternalUtilities/UNTESTED/DtoImporter.cs +++ b/DomainServices/UNTESTED/DtoImporter.cs @@ -2,10 +2,9 @@ using System.Collections.Generic; using System.Linq; using DataLayer; -using Scraping.BookDetail; -using Scraping.Library; +using DTOs; -namespace InternalUtilities +namespace DomainServices { public class DtoImporter { diff --git a/DomainServices/UNTESTED/Indexer.cs b/DomainServices/UNTESTED/Indexer.cs index ee22a177..1e72f34c 100644 --- a/DomainServices/UNTESTED/Indexer.cs +++ b/DomainServices/UNTESTED/Indexer.cs @@ -7,11 +7,10 @@ using System.Threading.Tasks; using DataLayer; using Dinah.Core; using Dinah.Core.Collections.Generic; +using DTOs; using FileManager; using InternalUtilities; using Newtonsoft.Json; -using Scraping.BookDetail; -using Scraping.Library; namespace DomainServices { diff --git a/DomainServices/UNTESTED/ScrapeBookDetails.cs b/DomainServices/UNTESTED/ScrapeBookDetails.cs index 51ffb987..3e99891c 100644 --- a/DomainServices/UNTESTED/ScrapeBookDetails.cs +++ b/DomainServices/UNTESTED/ScrapeBookDetails.cs @@ -6,9 +6,9 @@ using System.Threading.Tasks; using AudibleDotCom; using DataLayer; using Dinah.Core.ErrorHandling; +using DTOs; using InternalUtilities; using Scraping; -using Scraping.BookDetail; namespace DomainServices { diff --git a/InternalUtilities/InternalUtilities.csproj b/InternalUtilities/InternalUtilities.csproj index 1743dfd0..6493853d 100644 --- a/InternalUtilities/InternalUtilities.csproj +++ b/InternalUtilities/InternalUtilities.csproj @@ -5,7 +5,6 @@ - diff --git a/Libation.sln b/Libation.sln index 76f8142c..3202d151 100644 --- a/Libation.sln +++ b/Libation.sln @@ -81,6 +81,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2", "..\Lucene EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2.Tests", "..\LuceneNet303r2\LuceneNet303r2.Tests\LuceneNet303r2.Tests.csproj", "{5A7681A5-60D9-480B-9AC7-63E0812A2548}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DTOs", "DTOs\DTOs.csproj", "{5FDA62B1-55FD-407A-BECA-38A969235541}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -195,6 +197,10 @@ Global {5A7681A5-60D9-480B-9AC7-63E0812A2548}.Debug|Any CPU.Build.0 = Debug|Any CPU {5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.ActiveCfg = Release|Any CPU {5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.Build.0 = Release|Any CPU + {5FDA62B1-55FD-407A-BECA-38A969235541}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5FDA62B1-55FD-407A-BECA-38A969235541}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5FDA62B1-55FD-407A-BECA-38A969235541}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5FDA62B1-55FD-407A-BECA-38A969235541}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -227,6 +233,7 @@ Global {1255D9BA-CE6E-42E4-A253-6376540B9661} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1} {35803735-B669-4090-9681-CC7F7FABDC71} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} {5A7681A5-60D9-480B-9AC7-63E0812A2548} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD} + {5FDA62B1-55FD-407A-BECA-38A969235541} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9} diff --git a/LibationWinForm/UNTESTED/Form1.cs b/LibationWinForm/UNTESTED/Form1.cs index cd1f9285..e2af8823 100644 --- a/LibationWinForm/UNTESTED/Form1.cs +++ b/LibationWinForm/UNTESTED/Form1.cs @@ -179,7 +179,7 @@ namespace LibationWinForm } #endregion - #region grid select + #region reload grid bool isProcessingGridSelect = false; private void reloadGrid() { @@ -355,7 +355,13 @@ namespace LibationWinForm #endregion } - private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e) => await indexDialog(new ScanLibraryDialog()); + private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e) + { + // audible api + + // scrape + await indexDialog(new ScanLibraryDialog()); + } private async void reimportMostRecentLibraryScanToolStripMenuItem_Click(object sender, EventArgs e) { diff --git a/REFERENCE.txt b/REFERENCE.txt index 83f6bedb..9d897df7 100644 --- a/REFERENCE.txt +++ b/REFERENCE.txt @@ -4,40 +4,6 @@ alternate book id (eg BK_RAND_006061) is called 'sku' , 'sku_lite' , 'prod_id' , -- end AUDIBLE DETAILS --------------------------------------------------------------------------------------------------------------------- --- begin DOTNET CORE DI --------------------------------------------------------------------------------------------------------------------- -from: https://andrewlock.net/using-dependency-injection-in-a-net-core-console-application/ - -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -public class Program -{ - public static void Main(string[] args) - { - //setup our DI - var serviceProvider = new ServiceCollection() - .AddLogging() - .AddSingleton() - .AddSingleton() - .BuildServiceProvider(); - - //configure console logging - serviceProvider - .GetService() - .AddConsole(LogLevel.Debug); - - var logger = serviceProvider.GetService().CreateLogger(); - logger.LogDebug("Starting application"); - - //do the actual work here - var bar = serviceProvider.GetService(); - bar.DoSomeRealWork(); - - logger.LogDebug("All done!"); - } -} --- end DOTNET CORE DI --------------------------------------------------------------------------------------------------------------------- - -- begin SOLUTION LAYOUT --------------------------------------------------------------------------------------------------------------------- core libraries extend Standard Libraries @@ -102,51 +68,4 @@ https://blog.stephencleary.com/2012/02/async-and-await.html Async-Await - Best Practices in Asynchronous Programming -- Stephen Cleary https://msdn.microsoft.com/en-us/magazine/jj991977.aspx - - -ASYNC PARALLELISM -- pre C# 8 -================= -private async Task WaitThreeSeconds(int param) -{ - Console.WriteLine($"{param} started ------ ({DateTime.Now:hh:mm:ss}) ---"); - await Task.Delay(3000); - Console.WriteLine($"{ param} finished ------({ DateTime.Now:hh:mm: ss}) ---"); -} -/////////////////////////////////////////////////////////////////////// -// SERIAL CODE -var listOfInts = new List() { 1, 2, 3 }; -foreach (var integer in listOfInts) - await WaitThreeSeconds(integer); -// SERIAL RESULTS -1 started ------ (00:00:40) --- -1 finished ------(00:00: 43) --- -2 started ------ (00:00:43) --- -2 finished ------(00:00: 46) --- -3 started ------ (00:00:46) --- -2 finished ------(00:00: 49) --- -/////////////////////////////////////////////////////////////////////// -// PARALLEL CODE -var listOfInts = new List() { 1, 2, 3 }; -var tasks = new List(); -foreach (var integer in listOfInts) - tasks.Add(WaitThreeSeconds(integer)); -await Task.WhenAll(tasks); -// PARALLEL RESULTS -1 started ------ (00:00:02) --- -2 started ------ (00:00:02) --- -3 started ------ (00:00:02) --- -3 finished ------(00:00: 05) --- -2 finished ------(00:00: 05) --- -1 finished ------(00:00: 05) --- -/////////////////////////////////////////////////////////////////////// -// similar. note that await IS NOT being used in the top method calls. TASKS are returned. those tasks are later awaited in parallel -var catTask = FeedCatAsync(); -var houseTask = SellHouseAsync(); -var carTask = BuyCarAsync(); - -await Task.WhenAll(catTask, houseTask, carTask).ConfigureAwait(false); - -var cat = await catTask; -var house = await houseTask; -var car = await carTask; -- end ASYNC/AWAIT --------------------------------------------------------------------------------------------------------------------- diff --git a/Scraping/Scraping.csproj b/Scraping/Scraping.csproj index 49e55777..8d208b90 100644 --- a/Scraping/Scraping.csproj +++ b/Scraping/Scraping.csproj @@ -6,6 +6,7 @@ + diff --git a/Scraping/UNTESTED/AudibleScraper.cs b/Scraping/UNTESTED/AudibleScraper.cs index a94343b5..3ea53311 100644 --- a/Scraping/UNTESTED/AudibleScraper.cs +++ b/Scraping/UNTESTED/AudibleScraper.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using AudibleDotCom; using Dinah.Core; +using DTOs; using Scraping.BookDetail; using Scraping.Library; diff --git a/Scraping/UNTESTED/BookDetail/BookDetailScraper.cs b/Scraping/UNTESTED/BookDetail/BookDetailScraper.cs index c1d89de1..ae4d5a93 100644 --- a/Scraping/UNTESTED/BookDetail/BookDetailScraper.cs +++ b/Scraping/UNTESTED/BookDetail/BookDetailScraper.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Linq; using AudibleDotCom; using Dinah.Core; +using DTOs; using Newtonsoft.Json.Linq; using Scraping.Selectors; diff --git a/Scraping/UNTESTED/BookDetail/BookDetailScraperRules.cs b/Scraping/UNTESTED/BookDetail/BookDetailScraperRules.cs index 1530399d..81ab36db 100644 --- a/Scraping/UNTESTED/BookDetail/BookDetailScraperRules.cs +++ b/Scraping/UNTESTED/BookDetail/BookDetailScraperRules.cs @@ -2,7 +2,7 @@ using Scraping.Rules; using Scraping.Selectors; -using DTO = Scraping.BookDetail.BookDetailDTO; +using DTO = DTOs.BookDetailDTO; namespace Scraping.BookDetail { diff --git a/Scraping/UNTESTED/Library/LibraryScraper.cs b/Scraping/UNTESTED/Library/LibraryScraper.cs index d4a14b3d..889b5d7b 100644 --- a/Scraping/UNTESTED/Library/LibraryScraper.cs +++ b/Scraping/UNTESTED/Library/LibraryScraper.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text.RegularExpressions; using AudibleDotCom; using Dinah.Core; +using DTOs; using FileManager; using Scraping.Selectors; diff --git a/Scraping/UNTESTED/Library/LibraryScraperBase.cs b/Scraping/UNTESTED/Library/LibraryScraperBase.cs index 21832d06..b7cf477a 100644 --- a/Scraping/UNTESTED/Library/LibraryScraperBase.cs +++ b/Scraping/UNTESTED/Library/LibraryScraperBase.cs @@ -2,6 +2,7 @@ using System.Linq; using System.Threading.Tasks; using AudibleDotCom; +using DTOs; using Scraping.Selectors; namespace Scraping.Library diff --git a/Scraping/UNTESTED/Library/LibraryScraperRules.cs b/Scraping/UNTESTED/Library/LibraryScraperRules.cs index e305ce01..c8c9061d 100644 --- a/Scraping/UNTESTED/Library/LibraryScraperRules.cs +++ b/Scraping/UNTESTED/Library/LibraryScraperRules.cs @@ -2,7 +2,7 @@ using Scraping.Rules; using Scraping.Selectors; -using DTO = Scraping.Library.LibraryDTO; +using DTO = DTOs.LibraryDTO; namespace Scraping.Library { diff --git a/__TODO.txt b/__TODO.txt index d39909b7..553520ab 100644 --- a/__TODO.txt +++ b/__TODO.txt @@ -1,8 +1,5 @@ -- begin CONFIG FILES --------------------------------------------------------------------------------------------------------------------- try saving back into config -https://stackoverflow.com/questions/40970944/how-to-update-values-into-appsetting-json - -move ConsoleConfigDemo into TestCommon multiple files named "appsettings.json" will overwrite each other @@ -22,6 +19,10 @@ from here can set up a shared dir anywhere. use recursive upward search to find -- end CONFIG FILES --------------------------------------------------------------------------------------------------------------------- -- begin REPLACE SCRAPING WITH API --------------------------------------------------------------------------------------------------------------------- +integrate API into libation. eventually entirely replace all other authentication methods, audible communication. incl book download +logged-in user's own ratings (aka rating-star(s)) -- is "provided_review" +unlike traditional scraping, audible api calls do not need to be humanized. can likely delete humanizer altogether. still needed for downloading pdf.s? + incl episodes. eg: "Bill Bryson's Appliance of Science" refining episode retrieval might also get rid of the need for IsEpisodes property replace all scraping with audible api @@ -30,8 +31,6 @@ replace all scraping with audible api using var pageRetriever = websiteProcessorControl1.GetPageRetriever(); jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false); -break out DTOs. currently coupled with scraping - note: some json importing may still be relevant. might want to allow custom importing scraping stuff to remove: