tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Decouple DTOs from Scraping

Browse Source
This commit is contained in:
Robert McRackan 2019-10-21 09:25:56 -04:00
parent a1ebe1b0c8
commit fbc9824f12
18 changed files with 39 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Scraping/UNTESTED/BookDetail/BookDetailDTO.cs → DTOs/BookDetailDTO.cs
View File

@ -1,7 +1,7 @@
using System;
using System.Collections.Generic;
namespace Scraping.BookDetail
namespace DTOs
{
public class SeriesEntry
{

7
DTOs/DTOs.csproj Normal file
View File

@ -0,0 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
</PropertyGroup>
</Project>

2
Scraping/UNTESTED/Library/LibraryDTO.cs → DTOs/LibraryDTO.cs
View File

@ -1,7 +1,7 @@
using System;
using System.Collections.Generic;
namespace Scraping.Library
namespace DTOs
{
public class LibraryDTO
{

5
InternalUtilities/UNTESTED/DtoImporter.cs → DomainServices/UNTESTED/DtoImporter.cs
View File

@ -2,10 +2,9 @@
using System.Collections.Generic;
using System.Linq;
using DataLayer;
using Scraping.BookDetail;
using Scraping.Library;
using DTOs;
namespace InternalUtilities
namespace DomainServices
{
public class DtoImporter
{

3
DomainServices/UNTESTED/Indexer.cs
View File

@ -7,11 +7,10 @@ using System.Threading.Tasks;
using DataLayer;
using Dinah.Core;
using Dinah.Core.Collections.Generic;
using DTOs;
using FileManager;
using InternalUtilities;
using Newtonsoft.Json;
using Scraping.BookDetail;
using Scraping.Library;
namespace DomainServices
{

2
DomainServices/UNTESTED/ScrapeBookDetails.cs
View File

@ -6,9 +6,9 @@ using System.Threading.Tasks;
using AudibleDotCom;
using DataLayer;
using Dinah.Core.ErrorHandling;
using DTOs;
using InternalUtilities;
using Scraping;
using Scraping.BookDetail;
namespace DomainServices
{

1
InternalUtilities/InternalUtilities.csproj
View File

@ -5,7 +5,6 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\DataLayer\DataLayer.csproj" />
<ProjectReference Include="..\LibationSearchEngine\LibationSearchEngine.csproj" />
<ProjectReference Include="..\Scraping\Scraping.csproj" />
</ItemGroup>

7
Libation.sln
View File

@ -81,6 +81,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2", "..\Lucene
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2.Tests", "..\LuceneNet303r2\LuceneNet303r2.Tests\LuceneNet303r2.Tests.csproj", "{5A7681A5-60D9-480B-9AC7-63E0812A2548}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DTOs", "DTOs\DTOs.csproj", "{5FDA62B1-55FD-407A-BECA-38A969235541}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -195,6 +197,10 @@ Global
{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5A7681A5-60D9-480B-9AC7-63E0812A2548}.Release|Any CPU.Build.0 = Release|Any CPU
{5FDA62B1-55FD-407A-BECA-38A969235541}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5FDA62B1-55FD-407A-BECA-38A969235541}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5FDA62B1-55FD-407A-BECA-38A969235541}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5FDA62B1-55FD-407A-BECA-38A969235541}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -227,6 +233,7 @@ Global
{1255D9BA-CE6E-42E4-A253-6376540B9661} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
{35803735-B669-4090-9681-CC7F7FABDC71} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
{5A7681A5-60D9-480B-9AC7-63E0812A2548} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{5FDA62B1-55FD-407A-BECA-38A969235541} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9}

10
LibationWinForm/UNTESTED/Form1.cs
View File

@ -179,7 +179,7 @@ namespace LibationWinForm
}
#endregion
#region grid select
#region reload grid
bool isProcessingGridSelect = false;
private void reloadGrid()
{
@ -355,7 +355,13 @@ namespace LibationWinForm
#endregion
}
private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e) => await indexDialog(new ScanLibraryDialog());
private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e)
{
// audible api
// scrape
await indexDialog(new ScanLibraryDialog());
}
private async void reimportMostRecentLibraryScanToolStripMenuItem_Click(object sender, EventArgs e)
{

81
REFERENCE.txt
View File

@ -4,40 +4,6 @@ alternate book id (eg BK_RAND_006061) is called 'sku' , 'sku_lite' , 'prod_id' ,
-- end AUDIBLE DETAILS ---------------------------------------------------------------------------------------------------------------------
-- begin DOTNET CORE DI ---------------------------------------------------------------------------------------------------------------------
from: https://andrewlock.net/using-dependency-injection-in-a-net-core-console-application/
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
public class Program
{
public static void Main(string[] args)
{
//setup our DI
var serviceProvider = new ServiceCollection()
.AddLogging()
.AddSingleton<IFooService, FooService>()
.AddSingleton<IBarService, BarService>()
.BuildServiceProvider();
//configure console logging
serviceProvider
.GetService<ILoggerFactory>()
.AddConsole(LogLevel.Debug);
var logger = serviceProvider.GetService<ILoggerFactory>().CreateLogger<Program>();
logger.LogDebug("Starting application");
//do the actual work here
var bar = serviceProvider.GetService<IBarService>();
bar.DoSomeRealWork();
logger.LogDebug("All done!");
}
}
-- end DOTNET CORE DI ---------------------------------------------------------------------------------------------------------------------
-- begin SOLUTION LAYOUT ---------------------------------------------------------------------------------------------------------------------
core libraries
extend Standard Libraries
@ -102,51 +68,4 @@ https://blog.stephencleary.com/2012/02/async-and-await.html
Async-Await - Best Practices in Asynchronous Programming -- Stephen Cleary
https://msdn.microsoft.com/en-us/magazine/jj991977.aspx
ASYNC PARALLELISM -- pre C# 8
=================
private async Task WaitThreeSeconds(int param)
{
Console.WriteLine($"{param} started ------ ({DateTime.Now:hh:mm:ss}) ---");
await Task.Delay(3000);
Console.WriteLine($"{ param} finished ------({ DateTime.Now:hh:mm: ss}) ---");
}
///////////////////////////////////////////////////////////////////////
// SERIAL CODE
var listOfInts = new List<int>() { 1, 2, 3 };
foreach (var integer in listOfInts)
await WaitThreeSeconds(integer);
// SERIAL RESULTS
1 started ------ (00:00:40) ---
1 finished ------(00:00: 43) ---
2 started ------ (00:00:43) ---
2 finished ------(00:00: 46) ---
3 started ------ (00:00:46) ---
2 finished ------(00:00: 49) ---
///////////////////////////////////////////////////////////////////////
// PARALLEL CODE
var listOfInts = new List<int>() { 1, 2, 3 };
var tasks = new List<Task>();
foreach (var integer in listOfInts)
tasks.Add(WaitThreeSeconds(integer));
await Task.WhenAll(tasks);
// PARALLEL RESULTS
1 started ------ (00:00:02) ---
2 started ------ (00:00:02) ---
3 started ------ (00:00:02) ---
3 finished ------(00:00: 05) ---
2 finished ------(00:00: 05) ---
1 finished ------(00:00: 05) ---
///////////////////////////////////////////////////////////////////////
// similar. note that await IS NOT being used in the top method calls. TASKS are returned. those tasks are later awaited in parallel
var catTask = FeedCatAsync();
var houseTask = SellHouseAsync();
var carTask = BuyCarAsync();
await Task.WhenAll(catTask, houseTask, carTask).ConfigureAwait(false);
var cat = await catTask;
var house = await houseTask;
var car = await carTask;
-- end ASYNC/AWAIT ---------------------------------------------------------------------------------------------------------------------

1
Scraping/Scraping.csproj
View File

@ -6,6 +6,7 @@
<ItemGroup>
<ProjectReference Include="..\AudibleDotCom\AudibleDotCom.csproj" />
<ProjectReference Include="..\DTOs\DTOs.csproj" />
</ItemGroup>
</Project>

1
Scraping/UNTESTED/AudibleScraper.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using AudibleDotCom;
using Dinah.Core;
using DTOs;
using Scraping.BookDetail;
using Scraping.Library;

1
Scraping/UNTESTED/BookDetail/BookDetailScraper.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using AudibleDotCom;
using Dinah.Core;
using DTOs;
using Newtonsoft.Json.Linq;
using Scraping.Selectors;

2
Scraping/UNTESTED/BookDetail/BookDetailScraperRules.cs
View File

@ -2,7 +2,7 @@
using Scraping.Rules;
using Scraping.Selectors;
using DTO = Scraping.BookDetail.BookDetailDTO;
using DTO = DTOs.BookDetailDTO;
namespace Scraping.BookDetail
{

1
Scraping/UNTESTED/Library/LibraryScraper.cs
View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Text.RegularExpressions;
using AudibleDotCom;
using Dinah.Core;
using DTOs;
using FileManager;
using Scraping.Selectors;

1
Scraping/UNTESTED/Library/LibraryScraperBase.cs
View File

@ -2,6 +2,7 @@
using System.Linq;
using System.Threading.Tasks;
using AudibleDotCom;
using DTOs;
using Scraping.Selectors;
namespace Scraping.Library

2
Scraping/UNTESTED/Library/LibraryScraperRules.cs
View File

@ -2,7 +2,7 @@
using Scraping.Rules;
using Scraping.Selectors;
using DTO = Scraping.Library.LibraryDTO;
using DTO = DTOs.LibraryDTO;
namespace Scraping.Library
{

9
__TODO.txt
View File

@ -1,8 +1,5 @@
-- begin CONFIG FILES ---------------------------------------------------------------------------------------------------------------------
try saving back into config
https://stackoverflow.com/questions/40970944/how-to-update-values-into-appsetting-json
move ConsoleConfigDemo into TestCommon
multiple files named "appsettings.json" will overwrite each other
@ -22,6 +19,10 @@ from here can set up a shared dir anywhere. use recursive upward search to find
-- end CONFIG FILES ---------------------------------------------------------------------------------------------------------------------
-- begin REPLACE SCRAPING WITH API ---------------------------------------------------------------------------------------------------------------------
integrate API into libation. eventually entirely replace all other authentication methods, audible communication. incl book download
logged-in user's own ratings (aka rating-star(s)) -- is "provided_review"
unlike traditional scraping, audible api calls do not need to be humanized. can likely delete humanizer altogether. still needed for downloading pdf.s?
incl episodes. eg: "Bill Bryson's Appliance of Science"
refining episode retrieval might also get rid of the need for IsEpisodes property
replace all scraping with audible api
@ -30,8 +31,6 @@ replace all scraping with audible api
using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
break out DTOs. currently coupled with scraping
note: some json importing may still be relevant. might want to allow custom importing
scraping stuff to remove: