diff --git a/ApplicationServices/UNTESTED/LibraryIndexer.cs b/ApplicationServices/UNTESTED/LibraryIndexer.cs index c111e5a2..9bdcd4b5 100644 --- a/ApplicationServices/UNTESTED/LibraryIndexer.cs +++ b/ApplicationServices/UNTESTED/LibraryIndexer.cs @@ -4,7 +4,7 @@ using AudibleApi; using DtoImporterService; using InternalUtilities; -namespace ApplicationService +namespace ApplicationServices { public class LibraryIndexer { diff --git a/ApplicationServices/UNTESTED/SearchEngineActions.cs b/ApplicationServices/UNTESTED/SearchEngineActions.cs index e6508abb..2d8781e3 100644 --- a/ApplicationServices/UNTESTED/SearchEngineActions.cs +++ b/ApplicationServices/UNTESTED/SearchEngineActions.cs @@ -1,7 +1,7 @@ using System.Threading.Tasks; using DataLayer; -namespace ApplicationService +namespace ApplicationServices { public static class SearchEngineActions { @@ -16,11 +16,5 @@ namespace ApplicationService var engine = new LibationSearchEngine.SearchEngine(); engine.UpdateTags(book.AudibleProductId, book.UserDefinedItem.Tags); } - - public static async Task ProductReIndexAsync(string productId) - { - var engine = new LibationSearchEngine.SearchEngine(); - await engine.UpdateBookAsync(productId).ConfigureAwait(false); - } } } diff --git a/ApplicationServices/UNTESTED/TagUpdater.cs b/ApplicationServices/UNTESTED/TagUpdater.cs new file mode 100644 index 00000000..ddef5057 --- /dev/null +++ b/ApplicationServices/UNTESTED/TagUpdater.cs @@ -0,0 +1,21 @@ +using DataLayer; + +namespace ApplicationServices +{ + public static class TagUpdater + { + public static int IndexChangedTags(Book book) + { + // update disconnected entity + using var context = LibationContext.Create(); + context.Update(book); + var qtyChanges = context.SaveChanges(); + + // this part is tags-specific + if (qtyChanges > 0) + SearchEngineActions.UpdateBookTags(book); + + return qtyChanges; + } + } +} diff --git a/AudibleDotCom/AudibleDotCom.csproj b/AudibleDotCom/AudibleDotCom.csproj deleted file mode 100644 index a1930702..00000000 --- a/AudibleDotCom/AudibleDotCom.csproj +++ /dev/null @@ -1,11 +0,0 @@ - - - - netstandard2.1 - - - - - - - diff --git a/AudibleDotCom/UNTESTED/AudiblePage.cs b/AudibleDotCom/UNTESTED/AudiblePage.cs deleted file mode 100644 index 4b993cd3..00000000 --- a/AudibleDotCom/UNTESTED/AudiblePage.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System; -using System.Linq; -using Dinah.Core; - -namespace AudibleDotCom -{ - public enum AudiblePageType - { - ProductDetails = 1, - - Library = 2 - } - public static class AudiblePageExt - { - public static AudiblePage GetAudiblePageRobust(this AudiblePageType audiblePage) => AudiblePage.FromPageType(audiblePage); - } - - public abstract partial class AudiblePage : Enumeration - { - // useful for generic classes: - // public abstract class PageScraper where T : AudiblePageRobust { - // public AudiblePage AudiblePage => AudiblePageRobust.GetAudiblePageFromType(typeof(T)); - public static AudiblePageType GetAudiblePageFromType(Type audiblePageRobustType) - => (AudiblePageType)GetAll().Single(t => t.GetType() == audiblePageRobustType).Id; - - public AudiblePageType AudiblePageType { get; } - - protected AudiblePage(AudiblePageType audiblePage, string abbreviation) : base((int)audiblePage, abbreviation) => AudiblePageType = audiblePage; - - public static AudiblePage FromPageType(AudiblePageType audiblePage) => FromValue((int)audiblePage); - - /// For pages which need a param, the param is marked with {0} - protected abstract string Url { get; } - public string GetUrl(string id) => string.Format(Url, id); - - public string Abbreviation => DisplayName; - } - public abstract partial class AudiblePage : Enumeration - { - public static AudiblePage Library { get; } = LibraryPage.Instance; - public class LibraryPage : AudiblePage - { - #region singleton stuff - public static LibraryPage Instance { get; } = new LibraryPage(); - static LibraryPage() { } - private LibraryPage() : base(AudiblePageType.Library, "LIB") { } - #endregion - - protected override string Url => "http://www.audible.com/lib"; - } - } - public abstract partial class AudiblePage : Enumeration - { - public static AudiblePage Product { get; } = ProductDetailPage.Instance; - public class ProductDetailPage : AudiblePage - { - #region singleton stuff - public static ProductDetailPage Instance { get; } = new ProductDetailPage(); - static ProductDetailPage() { } - private ProductDetailPage() : base(AudiblePageType.ProductDetails, "PD") { } - #endregion - - protected override string Url => "http://www.audible.com/pd/{0}"; - } - } -} diff --git a/AudibleDotCom/UNTESTED/AudiblePageSource.cs b/AudibleDotCom/UNTESTED/AudiblePageSource.cs deleted file mode 100644 index 05bc5bbc..00000000 --- a/AudibleDotCom/UNTESTED/AudiblePageSource.cs +++ /dev/null @@ -1,43 +0,0 @@ -using FileManager; - -namespace AudibleDotCom -{ - public class AudiblePageSource - { - public AudiblePageType AudiblePage { get; } - public string Source { get; } - public string PageId { get; } - - public AudiblePageSource(AudiblePageType audiblePage, string source, string pageId) - { - AudiblePage = audiblePage; - Source = source; - PageId = pageId; - } - - /// declawed allows local file to safely be reloaded in chrome - /// NOTE ABOUT DECLAWED FILES - /// making them safer also breaks functionality - /// eg: previously hidden parts become visible. this changes how selenium can parse pages. - /// hidden elements don't expose .Text property - public AudiblePageSource Declawed() => new AudiblePageSource(AudiblePage, FileUtility.Declaw(Source), PageId); - - public string Serialized() => $"\r\n" + Source; - - public static AudiblePageSource Deserialize(string serializedSource) - { - var endOfLine1 = serializedSource.IndexOf('\n'); - - var parameters = serializedSource - .Substring(0, endOfLine1) - .Split('|'); - var abbrev = parameters[1]; - var pageId = parameters[2]; - - var source = serializedSource.Substring(endOfLine1 + 1); - var audiblePage = AudibleDotCom.AudiblePage.FromDisplayName(abbrev).AudiblePageType; - - return new AudiblePageSource(audiblePage, source, pageId); - } - } -} diff --git a/AudibleDotComAutomation/AudibleDotComAutomation.csproj b/AudibleDotComAutomation/AudibleDotComAutomation.csproj deleted file mode 100644 index 35a53f2b..00000000 --- a/AudibleDotComAutomation/AudibleDotComAutomation.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - netstandard2.1 - - - - - - - - - - - - - - - Always - - - - diff --git a/AudibleDotComAutomation/UNTESTED/Page Retrievers/Abstract_SeleniumRetriever.cs b/AudibleDotComAutomation/UNTESTED/Page Retrievers/Abstract_SeleniumRetriever.cs deleted file mode 100644 index 29354f96..00000000 --- a/AudibleDotComAutomation/UNTESTED/Page Retrievers/Abstract_SeleniumRetriever.cs +++ /dev/null @@ -1,184 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using AudibleDotCom; -using Dinah.Core.Humanizer; -using OpenQA.Selenium; -using OpenQA.Selenium.Chrome; -using OpenQA.Selenium.Support.UI; - -namespace AudibleDotComAutomation -{ - /// browser manipulation. web driver access - /// browser operators. create and store web driver, browser navigation which can vary depending on whether anon or auth'd - /// - /// this base class: is online. no auth. used for most pages. retain no chrome cookies - public abstract class SeleniumRetriever : IPageRetriever - { - #region // chrome driver details - /* - HIDING CHROME CONSOLE WINDOW - hiding chrome console window has proven to cause more headaches than it solves. here's how to do it though: - // can also use CreateDefaultService() overloads to specify driver path and/or file name - var chromeDriverService = ChromeDriverService.CreateDefaultService(); - chromeDriverService.HideCommandPromptWindow = true; - return new ChromeDriver(chromeDriverService, options); - - HEADLESS CHROME - this WOULD be how to do headless. but amazon/audible are far too tricksy about their changes and anti-scraping measures - which renders 'headless' mode useless - var options = new ChromeOptions(); - options.AddArgument("--headless"); - - SPECIFYING DRIVER LOCATION - if continues to have trouble finding driver: - var driver = new ChromeDriver(@"C:\my\path\to\chromedriver\directory"); - var chromeDriverService = ChromeDriverService.CreateDefaultService(@"C:\my\path\to\chromedriver\directory"); - */ - #endregion - - protected IWebDriver Driver { get; } - Humanizer humanizer { get; } = new Humanizer(); - - protected SeleniumRetriever() - { - Driver = new ChromeDriver(ctorCreateChromeOptions()); - } - - /// no auth. retain no chrome cookies - protected virtual ChromeOptions ctorCreateChromeOptions() => new ChromeOptions(); - - protected async Task AudibleLinkClickAsync(IWebElement element) - { - // EACH CALL to audible should have a small random wait to reduce chances of scrape detection - await humanizer.Wait(); - - await Task.Run(() => Driver.Click(element)); - - await waitForSpinnerAsync(); - - // sometimes these clicks just take a while. add a few more seconds - await Task.Delay(5000); - } - - By spinnerLocator { get; } = By.Id("library-main-overlay"); - private async Task waitForSpinnerAsync() - { - // if loading overlay w/spinner exists: pause, wait for it to end - - await Task.Delay(100); - - if (Driver.FindElements(spinnerLocator).Count > 0) - new WebDriverWait(Driver, TimeSpan.FromSeconds(60)) - .Until(ExpectedConditions.InvisibilityOfElementLocated(spinnerLocator)); - } - - private bool isFirstRun = true; - protected virtual async Task FirstRunAsync() - { - // load with no beginning wait. then wait 7 seconds to allow for page flicker. it usually happens after ~5 seconds. can happen irrespective of login state - await Task.Run(() => Driver.Navigate().GoToUrl("http://www.audible.com/")); - await Task.Delay(7000); - } - - public async Task> GetPageSourcesAsync(AudiblePageType audiblePage, string pageId = null) - { - if (isFirstRun) - { - await FirstRunAsync(); - isFirstRun = false; - } - - await initFirstPageAsync(audiblePage, pageId); - - return await processUrl(audiblePage, pageId); - } - - private async Task initFirstPageAsync(AudiblePageType audiblePage, string pageId) - { - // EACH CALL to audible should have a small random wait to reduce chances of scrape detection - await humanizer.Wait(); - - var url = audiblePage.GetAudiblePageRobust().GetUrl(pageId); - await Task.Run(() => Driver.Navigate().GoToUrl(url)); - - await waitForSpinnerAsync(); - } - - private async Task> processUrl(AudiblePageType audiblePage, string pageId) - { - var pageSources = new List(); - do - { - pageSources.Add(new AudiblePageSource(audiblePage, Driver.PageSource, pageId)); - } - while (await hasMorePagesAsync()); - - return pageSources; - } - - #region has more pages - /// if no more pages, return false. else, navigate to next page and return true - private async Task hasMorePagesAsync() - { - var next = //old_hasMorePages() ?? - new_hasMorePages(); - if (next == null) - return false; - - await AudibleLinkClickAsync(next); - return true; - } - - private IWebElement old_hasMorePages() - { - var parentElements = Driver.FindElements(By.ClassName("adbl-page-next")); - if (parentElements.Count == 0) - return null; - - var childElements = parentElements[0].FindElements(By.LinkText("NEXT")); - if (childElements.Count != 1) - return null; - - return childElements[0]; - } - - // ~ oct 2017 - private IWebElement new_hasMorePages() - { - // get all active/enabled navigation links - var pageNavLinks = Driver.FindElements(By.ClassName("library-load-page")); - if (pageNavLinks.Count == 0) - return null; - - // get only the right chevron if active. - // note: there are also right chevrons which are not for wish list navigation which is why we first filter by library-load-page - var nextLink = pageNavLinks - .Where(p => p.FindElements(By.ClassName("bc-icon-chevron-right")).Count > 0) - .ToList(); // cut-off delayed execution - if (nextLink.Count == 0) - return null; - - return nextLink.Single().FindElement(By.TagName("button")); - } - #endregion - - #region IDisposable pattern - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool disposing) - { - if (disposing && Driver != null) - { - // Quit() does cleanup AND disposes - Driver.Quit(); - } - } - #endregion - } -} diff --git a/AudibleDotComAutomation/UNTESTED/Page Retrievers/AuthSeleniumRetriever.cs b/AudibleDotComAutomation/UNTESTED/Page Retrievers/AuthSeleniumRetriever.cs deleted file mode 100644 index 5e32f4b4..00000000 --- a/AudibleDotComAutomation/UNTESTED/Page Retrievers/AuthSeleniumRetriever.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using OpenQA.Selenium; - -namespace AudibleDotComAutomation -{ - /// for user collections: lib, WL - public abstract class AuthSeleniumRetriever : SeleniumRetriever - { - protected bool IsLoggedIn => GetListenerPageLink() != null; - - // needed? - protected AuthSeleniumRetriever() : base() { } - - protected IWebElement GetListenerPageLink() - { - var listenerPageElement = Driver.FindElements(By.XPath("//a[contains(@href, '/review-by-author')]")); - if (listenerPageElement.Count > 0) - return listenerPageElement[0]; - return null; - } - } -} diff --git a/AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs b/AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs deleted file mode 100644 index abc2eeca..00000000 --- a/AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs +++ /dev/null @@ -1,130 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Threading.Tasks; -using AudibleDotCom; -using CookieMonster; -using Dinah.Core; -using Dinah.Core.Humanizer; - -namespace AudibleDotComAutomation -{ - public class BrowserlessRetriever : IPageRetriever - { - Humanizer humanizer { get; } = new Humanizer(); - - public async Task> GetPageSourcesAsync(AudiblePageType audiblePage, string pageId = null) - { - switch (audiblePage) - { -case AudiblePageType.Library: return await getLibraryPageSourcesAsync(); - default: throw new NotImplementedException(); - } - } - - private async Task> getLibraryPageSourcesAsync() - { - var collection = new List(); - - var cookies = await getAudibleCookiesAsync(); - - var currPageNum = 1; - bool hasMorePages; - do - { - // EACH CALL to audible should have a small random wait to reduce chances of scrape detection - await humanizer.Wait(); - -var html = await getLibraryPageAsync(cookies, currPageNum); -var pageSource = new AudiblePageSource(AudiblePageType.Library, html, null); - collection.Add(pageSource); - - hasMorePages = getHasMorePages(pageSource.Source); - - currPageNum++; - } while (hasMorePages); - - return collection; - } - - private static async Task getAudibleCookiesAsync() - { - var liveCookies = await CookiesHelper.GetLiveCookieValuesAsync(); - - var audibleCookies = liveCookies.Where(c - => c.Domain.ContainsInsensitive("audible.com") - || c.Domain.ContainsInsensitive("adbl") - || c.Domain.ContainsInsensitive("amazon.com")) - .ToList(); - - var cookies = new CookieContainer(); - foreach (var c in audibleCookies) - cookies.Add(new Cookie(c.Name, c.Value, "/", c.Domain)); - - return cookies; - } - - private static bool getHasMorePages(string html) - { - var doc = new HtmlAgilityPack.HtmlDocument(); - doc.LoadHtml(html); - - // final page, invalid page: - // - // only page: ??? - // has more pages: - // - var next_active_link = doc - .DocumentNode - .Descendants() - .FirstOrDefault(n => - n.HasClass("nextButton") && - !n.HasClass("bc-button-disabled")); - - return next_active_link != null; - } - - private static async Task getLibraryPageAsync(CookieContainer cookies, int pageNum) - { - #region // POST example (from 2017 ajax) - // var destination = "https://www.audible.com/lib-ajax"; - // var webRequest = (HttpWebRequest)WebRequest.Create(destination); - // webRequest.Method = "POST"; - // webRequest.Accept = "*/*"; - // webRequest.AllowAutoRedirect = false; - // webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)"; - // webRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8"; - // webRequest.Credentials = null; - // - // webRequest.CookieContainer = new CookieContainer(); - // webRequest.CookieContainer.Add(cookies.GetCookies(new Uri(destination))); - // - // var postData = $"progType=all&timeFilter=all&itemsPerPage={itemsPerPage}&searchTerm=&searchType=&sortColumn=&sortType=down&page={pageNum}&mode=normal&subId=&subTitle="; - // var data = Encoding.UTF8.GetBytes(postData); - // webRequest.ContentLength = data.Length; - // using var dataStream = webRequest.GetRequestStream(); - // dataStream.Write(data, 0, data.Length); - #endregion - - var destination = "https://" + $"www.audible.com/lib?purchaseDateFilter=all&programFilter=all&sortBy=PURCHASE_DATE.dsc&page={pageNum}"; - var webRequest = (HttpWebRequest)WebRequest.Create(destination); - webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)"; - - webRequest.CookieContainer = new CookieContainer(); - webRequest.CookieContainer.Add(cookies.GetCookies(new Uri(destination))); - - var webResponse = await webRequest.GetResponseAsync(); - return new StreamReader(webResponse.GetResponseStream()).ReadToEnd(); - } - - public void Dispose() { } - } -} diff --git a/AudibleDotComAutomation/UNTESTED/Page Retrievers/ManualLoginSeleniumRetriever.cs b/AudibleDotComAutomation/UNTESTED/Page Retrievers/ManualLoginSeleniumRetriever.cs deleted file mode 100644 index caa00502..00000000 --- a/AudibleDotComAutomation/UNTESTED/Page Retrievers/ManualLoginSeleniumRetriever.cs +++ /dev/null @@ -1,75 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using OpenQA.Selenium; -using OpenQA.Selenium.Support.UI; - -namespace AudibleDotComAutomation -{ - /// online. get auth by logging in with provided username and password - /// retain no chrome cookies. enter user + pw login - public class ManualLoginSeleniumRetriever : AuthSeleniumRetriever - { - string _username; - string _password; - public ManualLoginSeleniumRetriever(string username, string password) : base() - { - _username = username; - _password = password; - } - protected override async Task FirstRunAsync() - { - await base.FirstRunAsync(); - - // can't extract this into AuthSeleniumRetriever ctor. can't use username/pw until prev ctors are complete - - // click login link - await AudibleLinkClickAsync(getLoginLink()); - - // wait until login page loads - new WebDriverWait(Driver, TimeSpan.FromSeconds(60)).Until(ExpectedConditions.ElementIsVisible(By.Id("ap_email"))); - - // insert credentials - Driver - .FindElement(By.Id("ap_email")) - .SendKeys(_username); - Driver - .FindElement(By.Id("ap_password")) - .SendKeys(_password); - - // submit - var submitElement - = Driver.FindElements(By.Id("signInSubmit")).FirstOrDefault() - ?? Driver.FindElement(By.Id("signInSubmit-input")); - await AudibleLinkClickAsync(submitElement); - - // wait until audible page loads - new WebDriverWait(Driver, TimeSpan.FromSeconds(60)) - .Until(d => GetListenerPageLink()); - - if (!IsLoggedIn) - throw new Exception("not logged in"); - } - private IWebElement getLoginLink() - { - { - var loginLinkElements1 = Driver.FindElements(By.XPath("//a[contains(@href, '/signin')]")); - if (loginLinkElements1.Any()) - return loginLinkElements1[0]; - } - - // - // ADD ADDITIONAL ACCEPTABLE PATTERNS HERE - // - //{ - // var loginLinkElements2 = Driver.FindElements(By.XPath("//a[contains(@href, '/signin')]")); - // if (loginLinkElements2.Any()) - // return loginLinkElements2[0]; - //} - - throw new NotFoundException("Cannot locate login link"); - } - } -} diff --git a/AudibleDotComAutomation/UNTESTED/Page Retrievers/UserDataSeleniumRetriever.cs b/AudibleDotComAutomation/UNTESTED/Page Retrievers/UserDataSeleniumRetriever.cs deleted file mode 100644 index 33219974..00000000 --- a/AudibleDotComAutomation/UNTESTED/Page Retrievers/UserDataSeleniumRetriever.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using OpenQA.Selenium.Chrome; - -namespace AudibleDotComAutomation -{ - /// online. load auth, cookies etc from user data - public class UserDataSeleniumRetriever : AuthSeleniumRetriever - { - public UserDataSeleniumRetriever() : base() - { - // can't extract this into AuthSeleniumRetriever ctor. can't use username/pw until prev ctors are complete - if (!IsLoggedIn) - throw new Exception("not logged in"); - } - - /// Use current user data/chrome cookies. DO NOT use if chrome is already open - protected override ChromeOptions ctorCreateChromeOptions() - { - var options = base.ctorCreateChromeOptions(); - - // load user data incl cookies. default on windows: - // %LOCALAPPDATA%\Google\Chrome\User Data - // C:\Users\username\AppData\Local\Google\Chrome\User Data - var chromeDefaultWindowsUserDataDir = System.IO.Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), - "Google", - "Chrome", - "User Data"); - options.AddArguments($"user-data-dir={chromeDefaultWindowsUserDataDir}"); - - return options; - } - } -} diff --git a/AudibleDotComAutomation/UNTESTED/Page Retrievers/_IPageRetriever.cs b/AudibleDotComAutomation/UNTESTED/Page Retrievers/_IPageRetriever.cs deleted file mode 100644 index 70ca5596..00000000 --- a/AudibleDotComAutomation/UNTESTED/Page Retrievers/_IPageRetriever.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using AudibleDotCom; - -namespace AudibleDotComAutomation -{ - public interface IPageRetriever : IDisposable - { - Task> GetPageSourcesAsync(AudiblePageType audiblePage, string pageId = null); - } -} diff --git a/AudibleDotComAutomation/UNTESTED/Selenium.Examples.cs b/AudibleDotComAutomation/UNTESTED/Selenium.Examples.cs deleted file mode 100644 index 03a509c6..00000000 --- a/AudibleDotComAutomation/UNTESTED/Selenium.Examples.cs +++ /dev/null @@ -1,115 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Linq; -using OpenQA.Selenium; -using OpenQA.Selenium.Support.UI; - -namespace AudibleDotComAutomation.Examples -{ - public class SeleniumExamples - { - public IWebDriver Driver { get; set; } - - IWebElement GetListenerPageLink() - { - var listenerPageElement = Driver.FindElements(By.XPath("//a[contains(@href, '/review-by-author')]")); - if (listenerPageElement.Count > 0) - return listenerPageElement[0]; - return null; - } - void wait_examples() - { - new WebDriverWait(Driver, TimeSpan.FromSeconds(60)) - .Until(ExpectedConditions.ElementIsVisible(By.Id("mast-member-acct-name"))); - - new WebDriverWait(Driver, TimeSpan.FromSeconds(60)) - .Until(d => GetListenerPageLink()); - - // https://stackoverflow.com/questions/21339339/how-to-add-custom-expectedconditions-for-selenium - new WebDriverWait(Driver, TimeSpan.FromSeconds(60)) - .Until((d) => - { - // could be refactored into OR, AND per the java selenium library - - // check 1 - var e1 = Driver.FindElements(By.Id("mast-member-acct-name")); - if (e1.Count > 0) - return e1[0]; - // check 2 - var e2 = Driver.FindElements(By.Id("header-account-info-0")); - if (e2.Count > 0) - return e2[0]; - return null; - }); - } - void XPath_examples() - { - // - // 1 - // 2 - // - // - // 3 - // 4 - // - - ReadOnlyCollection all_tr = Driver.FindElements(By.XPath("/tr")); - IWebElement first_tr = Driver.FindElement(By.XPath("/tr")); - IWebElement second_tr = Driver.FindElement(By.XPath("/tr[2]")); - // beginning with a single / starts from root - IWebElement ERROR_not_at_root = Driver.FindElement(By.XPath("/td")); - // 2 slashes searches all, NOT just descendants - IWebElement td1 = Driver.FindElement(By.XPath("//td")); - - // 2 slashes still searches all, NOT just descendants - IWebElement still_td1 = first_tr.FindElement(By.XPath("//td")); - - // dot operator starts from current node specified by first_tr - // single slash: immediate descendant - IWebElement td3 = first_tr.FindElement(By.XPath( - ".//td")); - // double slash: descendant at any depth - IWebElement td3_also = first_tr.FindElement(By.XPath( - "./td")); - - // - IWebElement find_anywhere_in_doc = first_tr.FindElement(By.XPath( - "//input[@name='asin']")); - IWebElement find_in_subsection = first_tr.FindElement(By.XPath( - ".//input[@name='asin']")); - - // search entire page. useful for: - // - RulesLocator to find something that only appears once on the page - // - non-list pages. eg: product details - var onePerPageRules = new RuleFamily - { - RowsLocator = By.XPath("/*"), // search entire page - Rules = new RuleSet { - (row, productItem) => productItem.CustomerId = row.FindElement(By.XPath("//input[@name='cust_id']")).GetValue(), - (row, productItem) => productItem.UserName = row.FindElement(By.XPath("//input[@name='user_name']")).GetValue() - } - }; - // - applying conditionals to entire page - var ruleFamily = new RuleFamily - { - RowsLocator = By.XPath("//*[starts-with(@id,'adbl-library-content-row-')]"), - // Rules = getRuleSet() - }; - } - #region Rules classes stubs - public class RuleFamily { public By RowsLocator; public IRuleClass Rules; } - public interface IRuleClass { } - public class RuleSet : IRuleClass, IEnumerable - { - public void Add(IRuleClass ruleClass) { } - public void Add(RuleAction action) { } - - public IEnumerator GetEnumerator() => throw new NotImplementedException(); - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => throw new NotImplementedException(); - } - public delegate void RuleAction(IWebElement row, ProductItem productItem); - public class ProductItem { public string CustomerId; public string UserName; } - #endregion - } -} diff --git a/AudibleDotComAutomation/UNTESTED/SeleniumExt.cs b/AudibleDotComAutomation/UNTESTED/SeleniumExt.cs deleted file mode 100644 index 386cecbb..00000000 --- a/AudibleDotComAutomation/UNTESTED/SeleniumExt.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using OpenQA.Selenium; -using OpenQA.Selenium.Interactions; - -namespace AudibleDotComAutomation -{ - public static class IWebElementExt - { - // allows getting Text from elements even if hidden - // this only works on visible elements: webElement.Text - // http://yizeng.me/2014/04/08/get-text-from-hidden-elements-using-selenium-webdriver/#c-sharp - // - public static string GetText(this IWebElement webElement) => webElement.GetAttribute("textContent"); - - public static string GetValue(this IWebElement webElement) => webElement.GetAttribute("value"); - } - - public static class IWebDriverExt - { - /// Use this instead of element.Click() to ensure that the element is clicked even if it's not currently scrolled into view - public static void Click(this IWebDriver driver, IWebElement element) - { - // from: https://stackoverflow.com/questions/12035023/selenium-webdriver-cant-click-on-a-link-outside-the-page - - - //// this works but isn't really the same - //element.SendKeys(Keys.Enter); - - - //// didn't work for me - //new Actions(driver) - // .MoveToElement(element) - // .Click() - // .Build() - // .Perform(); - - driver.ScrollIntoView(element); - element.Click(); - } - public static void ScrollIntoView(this IWebDriver driver, IWebElement element) - => ((IJavaScriptExecutor)driver).ExecuteScript($"window.scroll({element.Location.X}, {element.Location.Y})"); - } -} diff --git a/AudibleDotComAutomation/chromedriver.exe b/AudibleDotComAutomation/chromedriver.exe deleted file mode 100644 index f8b34f1b..00000000 Binary files a/AudibleDotComAutomation/chromedriver.exe and /dev/null differ diff --git a/CookieMonster/CookieMonster.csproj b/CookieMonster/CookieMonster.csproj deleted file mode 100644 index 62e79934..00000000 --- a/CookieMonster/CookieMonster.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - netstandard2.1 - - - - - - - - - - - - diff --git a/CookieMonster/UNTESTED/Browsers/Chrome.cs b/CookieMonster/UNTESTED/Browsers/Chrome.cs deleted file mode 100644 index fc55e613..00000000 --- a/CookieMonster/UNTESTED/Browsers/Chrome.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data.SQLite; -using System.IO; -using System.Text; -using System.Threading.Tasks; -using FileManager; - -namespace CookieMonster -{ - internal class Chrome : IBrowser - { - public async Task> GetAllCookiesAsync() - { - var col = new List(); - - var strPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"Google\Chrome\User Data\Default\Cookies"); - if (!FileUtility.FileExists(strPath)) - return col; - - // - // IF WE GET AN ERROR HERE - // then add a reference to sqlite core in the project which is ultimately calling this. - // a project which directly references CookieMonster doesn't need to also ref sqlite. - // however, for any further number of abstractions, the project needs to directly ref sqlite. - // eg: this will not work unless the winforms proj adds sqlite to ref.s: - // LibationWinForm > AudibleDotComAutomation > CookieMonster - // - using var conn = new SQLiteConnection("Data Source=" + strPath + ";pooling=false"); - using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT host_key, name, value, encrypted_value, last_access_utc, expires_utc FROM cookies;"; - - conn.Open(); - using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (reader.Read()) - { - var host_key = reader.GetString(0); - var name = reader.GetString(1); - var value = reader.GetString(2); - var last_access_utc = reader.GetInt64(4); - var expires_utc = reader.GetInt64(5); - - // https://stackoverflow.com/a/25874366 - if (string.IsNullOrWhiteSpace(value)) - { - var encrypted_value = (byte[])reader[3]; - var decodedData = System.Security.Cryptography.ProtectedData.Unprotect(encrypted_value, null, System.Security.Cryptography.DataProtectionScope.CurrentUser); - value = Encoding.ASCII.GetString(decodedData); - } - - try - { - // if something goes wrong in this step (eg: a cookie has an invalid filetime), then just skip this cookie - col.Add(new CookieValue { Browser = "chrome", Domain = host_key, Name = name, Value = value, LastAccess = chromeTimeToDateTimeUtc(last_access_utc), Expires = chromeTimeToDateTimeUtc(expires_utc) }); - } - catch { } - } - - return col; - } - - // Chrome uses 1601-01-01 00:00:00 UTC as the epoch (ie the starting point for the millisecond time counter). - // this is the same as "FILETIME" in Win32 except FILETIME uses 100ns ticks instead of ms. - private static DateTime chromeTimeToDateTimeUtc(long time) => DateTime.SpecifyKind(DateTime.FromFileTime(time * 10), DateTimeKind.Utc); - } -} diff --git a/CookieMonster/UNTESTED/Browsers/FireFox.cs b/CookieMonster/UNTESTED/Browsers/FireFox.cs deleted file mode 100644 index 6be33c6e..00000000 --- a/CookieMonster/UNTESTED/Browsers/FireFox.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data.SQLite; -using System.IO; -using System.Threading.Tasks; -using FileManager; - -namespace CookieMonster -{ - internal class FireFox : IBrowser - { - public async Task> GetAllCookiesAsync() - { - var col = new List(); - - string strPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), @"Mozilla\Firefox\Profiles"); - if (!FileUtility.FileExists(strPath)) - return col; - var dirs = new DirectoryInfo(strPath).GetDirectories("*.default"); - if (dirs.Length != 1) - return col; - strPath = Path.Combine(strPath, dirs[0].Name, "cookies.sqlite"); - if (!FileUtility.FileExists(strPath)) - return col; - - // First copy the cookie jar so that we can read the cookies from unlocked copy while FireFox is running - var strTemp = strPath + ".temp"; - - File.Copy(strPath, strTemp, true); - - // Now open the temporary cookie jar and extract Value from the cookie if we find it. - using var conn = new SQLiteConnection("Data Source=" + strTemp + ";pooling=false"); - using var cmd = conn.CreateCommand(); - cmd.CommandText = "SELECT host, name, value, lastAccessed, expiry FROM moz_cookies; "; - - conn.Open(); - using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); - while (reader.Read()) - { - var host_key = reader.GetString(0); - var name = reader.GetString(1); - var value = reader.GetString(2); - var lastAccessed = reader.GetInt32(3); - var expiry = reader.GetInt32(4); - - col.Add(new CookieValue { Browser = "firefox", Domain = host_key, Name = name, Value = value, LastAccess = lastAccessedToDateTime(lastAccessed), Expires = expiryToDateTime(expiry) }); - } - - if (FileUtility.FileExists(strTemp)) - File.Delete(strTemp); - - return col; - } - - // time is in microseconds since unix epoch - private static DateTime lastAccessedToDateTime(int time) => new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddMilliseconds(time); - - // time is in normal seconds since unix epoch - private static DateTime expiryToDateTime(int time) => new DateTime(1970, 1, 1, 0, 0, 0, 0, System.DateTimeKind.Utc).AddSeconds(time); - } -} diff --git a/CookieMonster/UNTESTED/Browsers/IBrowser.cs b/CookieMonster/UNTESTED/Browsers/IBrowser.cs deleted file mode 100644 index cb37cece..00000000 --- a/CookieMonster/UNTESTED/Browsers/IBrowser.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; - -namespace CookieMonster -{ - internal interface IBrowser - { - Task> GetAllCookiesAsync(); - } -} diff --git a/CookieMonster/UNTESTED/Browsers/InternetExplorer.cs b/CookieMonster/UNTESTED/Browsers/InternetExplorer.cs deleted file mode 100644 index 39f95c7f..00000000 --- a/CookieMonster/UNTESTED/Browsers/InternetExplorer.cs +++ /dev/null @@ -1,87 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace CookieMonster -{ - internal class InternetExplorer : IBrowser - { - public async Task> GetAllCookiesAsync() - { - // real locations of Windows Cookies folders - // - // Windows 7: - // C:\Users\username\AppData\Roaming\Microsoft\Windows\Cookies - // C:\Users\username\AppData\Roaming\Microsoft\Windows\Cookies\Low - // - // Windows 8, Windows 8.1, Windows 10: - // C:\Users\username\AppData\Local\Microsoft\Windows\INetCookies - // C:\Users\username\AppData\Local\Microsoft\Windows\INetCookies\Low - - var strPath = Environment.GetFolderPath(Environment.SpecialFolder.Cookies); - - var col = (await getIECookiesAsync(strPath).ConfigureAwait(false)).ToList(); - col = col.Concat(await getIECookiesAsync(Path.Combine(strPath, "Low"))).ToList(); - - return col; - } - - private static async Task> getIECookiesAsync(string strPath) - { - var cookies = new List(); - - var files = await Task.Run(() => Directory.EnumerateFiles(strPath, "*.txt")); - foreach (string path in files) - { - var cookiesInFile = new List(); - - var cookieLines = File.ReadAllLines(path); - CookieValue currCookieVal = null; - for (var i = 0; i < cookieLines.Length; i++) - { - var line = cookieLines[i]; - - // IE cookie format - // 0 Cookie name - // 1 Cookie value - // 2 Host / path for the web server setting the cookie - // 3 Flags - // 4 Expiration time (low int) - // 5 Expiration time (high int) - // 6 Creation time (low int) - // 7 Creation time (high int) - // 8 Record delimiter == "*" - var pos = i % 9; - long expLoTemp = 0; - long creatLoTemp = 0; - if (pos == 0) - { - currCookieVal = new CookieValue { Browser = "ie", Name = line }; - cookiesInFile.Add(currCookieVal); - } - else if (pos == 1) - currCookieVal.Value = line; - else if (pos == 2) - currCookieVal.Domain = line; - else if (pos == 4) - expLoTemp = Int64.Parse(line); - else if (pos == 5) - currCookieVal.Expires = LoHiToDateTime(expLoTemp, Int64.Parse(line)); - else if (pos == 6) - creatLoTemp = Int64.Parse(line); - else if (pos == 7) - currCookieVal.LastAccess = LoHiToDateTime(creatLoTemp, Int64.Parse(line)); - } - - cookies.AddRange(cookiesInFile); - } - - return cookies; - } - - private static DateTime LoHiToDateTime(long lo, long hi) => DateTime.FromFileTimeUtc(((hi << 32) + lo)); - } -} diff --git a/CookieMonster/UNTESTED/CookieValue.cs b/CookieMonster/UNTESTED/CookieValue.cs deleted file mode 100644 index c1fdde97..00000000 --- a/CookieMonster/UNTESTED/CookieValue.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; - -namespace CookieMonster -{ - public class CookieValue - { - public string Browser { get; set; } - - public string Name { get; set; } - public string Value { get; set; } - public string Domain { get; set; } - - public DateTime LastAccess { get; set; } - public DateTime Expires { get; set; } - - public bool IsValid - { - get - { - // sanity check. datetimes are stored weird in each cookie type. make sure i haven't converted these incredibly wrong. - // some early conversion attempts produced years like 42, 1955, 4033 - var _5yearsPast = DateTime.UtcNow.AddYears(-5); - if (LastAccess < _5yearsPast || LastAccess > DateTime.UtcNow) - return false; - // don't check expiry. some sites are setting stupid values for year. eg: 9999 - return true; - } - } - - public bool HasExpired => Expires < DateTime.UtcNow; - } -} diff --git a/CookieMonster/UNTESTED/CookiesHelper.cs b/CookieMonster/UNTESTED/CookiesHelper.cs deleted file mode 100644 index 3e7ad1f7..00000000 --- a/CookieMonster/UNTESTED/CookiesHelper.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Dinah.Core.Collections.Generic; - -namespace CookieMonster -{ - public static class CookiesHelper - { - internal static IEnumerable GetBrowsers() - => AppDomain.CurrentDomain - .GetAssemblies() - .SelectMany(s => s.GetTypes()) - .Where(p => typeof(IBrowser).IsAssignableFrom(p) && !p.IsAbstract && !p.IsInterface) - .Select(t => Activator.CreateInstance(t) as IBrowser) - .ToList(); - - /// all. including expired - public static async Task> GetAllCookieValuesAsync() - { - //// foreach{await} runs in serial - //var allCookies = new List(); - //foreach (var b in GetBrowsers()) - //{ - // var browserCookies = await b.GetAllCookiesAsync().ConfigureAwait(false); - // allCookies.AddRange(browserCookies); - //} - - //// WhenAll runs in parallel - // this 1st step LOOKS like a bug which runs each method until completion. However, since we don't use await, it's actually returning a Task. That resulting task is awaited asynchronously - var browserTasks = GetBrowsers().Select(b => b.GetAllCookiesAsync()); - var results = await Task.WhenAll(browserTasks).ConfigureAwait(false); - var allCookies = results.SelectMany(a => a).ToList(); - - if (allCookies.Any(c => !c.IsValid)) - throw new Exception("some date time was converted way too far"); - - foreach (var c in allCookies) - c.Domain = c.Domain.TrimEnd('/'); - - // for each domain+name, only keep the 1 with the most recent access - var sortedCookies = allCookies - .OrderByDescending(c => c.LastAccess) - .DistinctBy(c => new { c.Domain, c.Name }) - .ToList(); - - return sortedCookies; - } - - /// not expired - public static async Task> GetLiveCookieValuesAsync() - => (await GetAllCookieValuesAsync().ConfigureAwait(false)) - .Where(c => !c.HasExpired) - .ToList(); - } -} diff --git a/DataLayer/Migrations/20191105183104_NoScraping.Designer.cs b/DataLayer/Migrations/20191105183104_NoScraping.Designer.cs new file mode 100644 index 00000000..37db1775 --- /dev/null +++ b/DataLayer/Migrations/20191105183104_NoScraping.Designer.cs @@ -0,0 +1,335 @@ +// +using System; +using DataLayer; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace DataLayer.Migrations +{ + [DbContext(typeof(LibationContext))] + [Migration("20191105183104_NoScraping")] + partial class NoScraping + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "3.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 128) + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + modelBuilder.Entity("DataLayer.Book", b => + { + b.Property("BookId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b.Property("AudibleProductId") + .HasColumnType("nvarchar(450)"); + + b.Property("CategoryId") + .HasColumnType("int"); + + b.Property("DatePublished") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("IsAbridged") + .HasColumnType("bit"); + + b.Property("LengthInMinutes") + .HasColumnType("int"); + + b.Property("PictureId") + .HasColumnType("nvarchar(max)"); + + b.Property("Title") + .HasColumnType("nvarchar(max)"); + + b.HasKey("BookId"); + + b.HasIndex("AudibleProductId"); + + b.HasIndex("CategoryId"); + + b.ToTable("Books"); + }); + + modelBuilder.Entity("DataLayer.BookContributor", b => + { + b.Property("BookId") + .HasColumnType("int"); + + b.Property("ContributorId") + .HasColumnType("int"); + + b.Property("Role") + .HasColumnType("int"); + + b.Property("Order") + .HasColumnType("tinyint"); + + b.HasKey("BookId", "ContributorId", "Role"); + + b.HasIndex("BookId"); + + b.HasIndex("ContributorId"); + + b.ToTable("BookContributor"); + }); + + modelBuilder.Entity("DataLayer.Category", b => + { + b.Property("CategoryId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b.Property("AudibleCategoryId") + .HasColumnType("nvarchar(450)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.Property("ParentCategoryCategoryId") + .HasColumnType("int"); + + b.HasKey("CategoryId"); + + b.HasIndex("AudibleCategoryId"); + + b.HasIndex("ParentCategoryCategoryId"); + + b.ToTable("Categories"); + + b.HasData( + new + { + CategoryId = -1, + AudibleCategoryId = "", + Name = "" + }); + }); + + modelBuilder.Entity("DataLayer.Contributor", b => + { + b.Property("ContributorId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b.Property("AudibleAuthorId") + .HasColumnType("nvarchar(max)"); + + b.Property("Name") + .HasColumnType("nvarchar(450)"); + + b.HasKey("ContributorId"); + + b.HasIndex("Name"); + + b.ToTable("Contributors"); + }); + + modelBuilder.Entity("DataLayer.LibraryBook", b => + { + b.Property("BookId") + .HasColumnType("int"); + + b.Property("DateAdded") + .HasColumnType("datetime2"); + + b.HasKey("BookId"); + + b.ToTable("Library"); + }); + + modelBuilder.Entity("DataLayer.Series", b => + { + b.Property("SeriesId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b.Property("AudibleSeriesId") + .HasColumnType("nvarchar(450)"); + + b.Property("Name") + .HasColumnType("nvarchar(max)"); + + b.HasKey("SeriesId"); + + b.HasIndex("AudibleSeriesId"); + + b.ToTable("Series"); + }); + + modelBuilder.Entity("DataLayer.SeriesBook", b => + { + b.Property("SeriesId") + .HasColumnType("int"); + + b.Property("BookId") + .HasColumnType("int"); + + b.Property("Index") + .HasColumnType("real"); + + b.HasKey("SeriesId", "BookId"); + + b.HasIndex("BookId"); + + b.HasIndex("SeriesId"); + + b.ToTable("SeriesBook"); + }); + + modelBuilder.Entity("DataLayer.Book", b => + { + b.HasOne("DataLayer.Category", "Category") + .WithMany() + .HasForeignKey("CategoryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.OwnsOne("DataLayer.Rating", "Rating", b1 => + { + b1.Property("BookId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b1.Property("OverallRating") + .HasColumnType("real"); + + b1.Property("PerformanceRating") + .HasColumnType("real"); + + b1.Property("StoryRating") + .HasColumnType("real"); + + b1.HasKey("BookId"); + + b1.ToTable("Books"); + + b1.WithOwner() + .HasForeignKey("BookId"); + }); + + b.OwnsMany("DataLayer.Supplement", "Supplements", b1 => + { + b1.Property("SupplementId") + .ValueGeneratedOnAdd() + .HasColumnType("int") + .HasAnnotation("SqlServer:ValueGenerationStrategy", SqlServerValueGenerationStrategy.IdentityColumn); + + b1.Property("BookId") + .HasColumnType("int"); + + b1.Property("Url") + .HasColumnType("nvarchar(max)"); + + b1.HasKey("SupplementId"); + + b1.HasIndex("BookId"); + + b1.ToTable("Supplement"); + + b1.WithOwner("Book") + .HasForeignKey("BookId"); + }); + + b.OwnsOne("DataLayer.UserDefinedItem", "UserDefinedItem", b1 => + { + b1.Property("BookId") + .HasColumnType("int"); + + b1.Property("Tags") + .HasColumnType("nvarchar(max)"); + + b1.HasKey("BookId"); + + b1.ToTable("UserDefinedItem"); + + b1.WithOwner("Book") + .HasForeignKey("BookId"); + + b1.OwnsOne("DataLayer.Rating", "Rating", b2 => + { + b2.Property("UserDefinedItemBookId") + .HasColumnType("int"); + + b2.Property("OverallRating") + .HasColumnType("real"); + + b2.Property("PerformanceRating") + .HasColumnType("real"); + + b2.Property("StoryRating") + .HasColumnType("real"); + + b2.HasKey("UserDefinedItemBookId"); + + b2.ToTable("UserDefinedItem"); + + b2.WithOwner() + .HasForeignKey("UserDefinedItemBookId"); + }); + }); + }); + + modelBuilder.Entity("DataLayer.BookContributor", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithMany("ContributorsLink") + .HasForeignKey("BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DataLayer.Contributor", "Contributor") + .WithMany("BooksLink") + .HasForeignKey("ContributorId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DataLayer.Category", b => + { + b.HasOne("DataLayer.Category", "ParentCategory") + .WithMany() + .HasForeignKey("ParentCategoryCategoryId"); + }); + + modelBuilder.Entity("DataLayer.LibraryBook", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithOne() + .HasForeignKey("DataLayer.LibraryBook", "BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("DataLayer.SeriesBook", b => + { + b.HasOne("DataLayer.Book", "Book") + .WithMany("SeriesLink") + .HasForeignKey("BookId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("DataLayer.Series", "Series") + .WithMany("BooksLink") + .HasForeignKey("SeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DataLayer/Migrations/20191105183104_NoScraping.cs b/DataLayer/Migrations/20191105183104_NoScraping.cs new file mode 100644 index 00000000..e0d4cde8 --- /dev/null +++ b/DataLayer/Migrations/20191105183104_NoScraping.cs @@ -0,0 +1,82 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +namespace DataLayer.Migrations +{ + public partial class NoScraping : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_Supplement_Books_BookId", + table: "Supplement"); + + migrationBuilder.DropForeignKey( + name: "FK_UserDefinedItem_Books_BookId", + table: "UserDefinedItem"); + + migrationBuilder.DropColumn( + name: "DownloadBookLink", + table: "Library"); + + migrationBuilder.DropColumn( + name: "HasBookDetails", + table: "Books"); + + migrationBuilder.AddForeignKey( + name: "FK_Supplement_Books_BookId", + table: "Supplement", + column: "BookId", + principalTable: "Books", + principalColumn: "BookId", + onDelete: ReferentialAction.Cascade); + + migrationBuilder.AddForeignKey( + name: "FK_UserDefinedItem_Books_BookId", + table: "UserDefinedItem", + column: "BookId", + principalTable: "Books", + principalColumn: "BookId", + onDelete: ReferentialAction.Cascade); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropForeignKey( + name: "FK_Supplement_Books_BookId", + table: "Supplement"); + + migrationBuilder.DropForeignKey( + name: "FK_UserDefinedItem_Books_BookId", + table: "UserDefinedItem"); + + migrationBuilder.AddColumn( + name: "DownloadBookLink", + table: "Library", + type: "nvarchar(max)", + nullable: true); + + migrationBuilder.AddColumn( + name: "HasBookDetails", + table: "Books", + type: "bit", + nullable: false, + defaultValue: false); + + migrationBuilder.AddForeignKey( + name: "FK_Supplement_Books_BookId", + table: "Supplement", + column: "BookId", + principalTable: "Books", + principalColumn: "BookId", + onDelete: ReferentialAction.Restrict); + + migrationBuilder.AddForeignKey( + name: "FK_UserDefinedItem_Books_BookId", + table: "UserDefinedItem", + column: "BookId", + principalTable: "Books", + principalColumn: "BookId", + onDelete: ReferentialAction.Restrict); + } + } +} diff --git a/DataLayer/Migrations/LibationContextModelSnapshot.cs b/DataLayer/Migrations/LibationContextModelSnapshot.cs index 1cc7e64b..dcefe353 100644 --- a/DataLayer/Migrations/LibationContextModelSnapshot.cs +++ b/DataLayer/Migrations/LibationContextModelSnapshot.cs @@ -38,9 +38,6 @@ namespace DataLayer.Migrations b.Property("Description") .HasColumnType("nvarchar(max)"); - b.Property("HasBookDetails") - .HasColumnType("bit"); - b.Property("IsAbridged") .HasColumnType("bit"); @@ -146,9 +143,6 @@ namespace DataLayer.Migrations b.Property("DateAdded") .HasColumnType("datetime2"); - b.Property("DownloadBookLink") - .HasColumnType("nvarchar(max)"); - b.HasKey("BookId"); b.ToTable("Library"); diff --git a/DataLayer/UNTESTED/Commands/RemoveOrphans.cs b/DataLayer/UNTESTED/Commands/RemoveOrphans.cs deleted file mode 100644 index 0478580e..00000000 --- a/DataLayer/UNTESTED/Commands/RemoveOrphans.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.EntityFrameworkCore; - -namespace DataLayer -{ - public static class RemoveOrphansCommand - { - public static int RemoveOrphans(this LibationContext context) - => context.Database.ExecuteSqlRaw(@" - delete c - from Contributors c - left join BookContributor bc on c.ContributorId = bc.ContributorId - left join Books b on bc.BookId = b.BookId - where bc.ContributorId is null - "); - } -} diff --git a/DataLayer/UNTESTED/EfClasses/Book.cs b/DataLayer/UNTESTED/EfClasses/Book.cs index dc7e40c4..cb3d69eb 100644 --- a/DataLayer/UNTESTED/EfClasses/Book.cs +++ b/DataLayer/UNTESTED/EfClasses/Book.cs @@ -30,7 +30,6 @@ namespace DataLayer public string PictureId { get; set; } // book details - public bool HasBookDetails { get; private set; } public bool IsAbridged { get; private set; } public DateTime? DatePublished { get; private set; } @@ -231,8 +230,6 @@ namespace DataLayer // don't overwrite with default values IsAbridged |= isAbridged; DatePublished = datePublished ?? DatePublished; - - HasBookDetails = true; } public void UpdateCategory(Category category, DbContext context = null) diff --git a/DataLayer/UNTESTED/EfClasses/LibraryBook.cs b/DataLayer/UNTESTED/EfClasses/LibraryBook.cs index e281edb9..236c5b4d 100644 --- a/DataLayer/UNTESTED/EfClasses/LibraryBook.cs +++ b/DataLayer/UNTESTED/EfClasses/LibraryBook.cs @@ -10,18 +10,12 @@ namespace DataLayer public DateTime DateAdded { get; private set; } -/// For downloading AAX file -public string DownloadBookLink { get; private set; } - private LibraryBook() { } - public LibraryBook(Book book, DateTime dateAdded -, string downloadBookLink = null -) + public LibraryBook(Book book, DateTime dateAdded) { ArgumentValidator.EnsureNotNull(book, nameof(book)); Book = book; DateAdded = dateAdded; -DownloadBookLink = downloadBookLink; } } } diff --git a/DataLayer/UNTESTED/QueryObjects/BookQueries.cs b/DataLayer/UNTESTED/QueryObjects/BookQueries.cs index 8a9606d7..58ae2b74 100644 --- a/DataLayer/UNTESTED/QueryObjects/BookQueries.cs +++ b/DataLayer/UNTESTED/QueryObjects/BookQueries.cs @@ -8,14 +8,6 @@ namespace DataLayer { public static class BookQueries { - public static int BooksWithoutDetailsCount() - { - using var context = LibationContext.Create(); - return context - .Books - .Count(b => !b.HasBookDetails); - } - public static Book GetBook_Flat_NoTracking(string productId) { using var context = LibationContext.Create(); diff --git a/DataLayer/_HowTo- EF Core.txt b/DataLayer/_HowTo- EF Core.txt index 112d5e1e..533cf718 100644 --- a/DataLayer/_HowTo- EF Core.txt +++ b/DataLayer/_HowTo- EF Core.txt @@ -1,6 +1,5 @@ HOW TO CREATE: EF CORE PROJECT ============================== -easiest with .NET Core but there's also a work-around for .NET Standard example is for sqlite but the same works with MsSql @@ -26,7 +25,6 @@ set project "Set as StartUp Project" Tools >> Nuget Package Manager >> Package Manager Console default project: Examples\SQLite_NETCore2_0 -note: in EFCore, Enable-Migrations is no longer used. start with add-migration PM> add-migration InitialCreate PM> Update-Database diff --git a/DataLayer/_big db refactor.txt b/DataLayer/_big db refactor.txt deleted file mode 100644 index 84ff9d7b..00000000 --- a/DataLayer/_big db refactor.txt +++ /dev/null @@ -1,55 +0,0 @@ -proposed extensible schema to generalize beyond audible - -problems -0) reeks of premature optimization -- i'm currently only doing audible audiobooks. this adds several layers of abstraction for the sake of possible expansion -- there's a good chance that supporting another platform may not conform to this schema, in which case i'd have done this for nothing. genres are one likely pain point -- libation is currently single-user. hopefully the below would suffice for adding users, but if i'm wrong it might be all pain and no gain -1) very thorough == very complex -2) there are some books which would still be difficult to taxonimize -- joy of cooking. has become more of a brand -- the bible. has different versions that aren't just editions -- dictionary. authored by a publisher -3) "books" vs "editions" is a confusing problem waiting to happen - -[AIPK=auto increm PK] - -(libation) users [AIPK id, name, join date] -audible users [AIPK id, AUDIBLE-PK username] -libation audible users [PK user id, PK audible user id -- cluster PK across all FKs] -- potential danger in multi-user environment. wouldn't want one libation user getting access to a different libation user's audible info -contributors [AIPK id, name]. prev people. incl publishers -audible authors [PK/FK contributor id, AUDIBLE-PK author id] -roles [AIPK id, name]. seeded: author, narrator, publisher. could expand (eg: translator, editor) without each needing a new table -books [AIPK id, title, desc] -book contributors [FK book id, FK contributor id, FK role id, order -- cluster PK across all FKs] -- likely only authors -editions [AIPK id, FK book id, title]. could expand to include year, is first edition, is abridged -- reasons for optional different title: "Ender's Game: Special 20th Anniversary Edition", "Harry Potter and the Sorcerer's Stone" vs "Harry Potter and the Philosopher's Stone" vs "Harry Potter y la piedra filosofal", "Midnight Riot" vs "Rivers of London" -edition contributors [FK edition id, FK contributor id, FK role id, order -- cluster PK across all FKs] -- likely everything except authors. eg narrators, publisher -audiobooks [PK/FK edition id, lengthInMinutes] -- could expand to other formats by adding other similar tables. eg: print with #pages and isbn, ebook with mb -audible origins [AIPK id, name]. seeded: library. detail. json. series -audible books [PK/FK edition id, AUDIBLE-PK product id, picture id, sku, 3 ratings, audible category id, audible origin id] -- could expand to other vendors by adding other similar tables -audible user ratings [PK/FK edition id, audible user id, 3 ratings] -audible supplements [AIPK id, FK edition id, download url] -- pdfs only. although book download info could be the same format, they're substantially different and subject to change -audible book downloads [PK/FK edition id, audible user id, bookdownloadlink] -pictures [AIPK id, FK edition id, filename (xyz.jpg -- not incl path)] -audible categories [AIPK id, AUDIBLE-PK category id, name, parent]. may only nest 1 deep -(libation) library [FK libation user id, FK edition id, date added -- cluster PK across all FKs] -(libation) user defined [FK libation user id, FK edition id, tagsRaw (, notes...) -- cluster PK across all FKs] -- there's no reason to restrict tags to library items, so don't combine/link this table with library -series [AIPK id, name] -audible series [FK series id, AUDIBLE-PK series id/asin, audible origin id] -- could also include a 'name' field for what audible calls this series -series books [FK series id, FK book id (NOT edition id), index -- cluster PK across all FKs] -- "index" not "order". display this number; don't just put in this sequence -- index is float instead of int to allow for in-between books. eg 2.5 -- if only using "editions" (ie: getting rid of the "books" table), to show 2 editions as the same book in a series, give them the same index -(libation) user shelves [AIPK id, FK libation user id, name, desc] -- custom shelf. similar to library but very different in philosophy. likely different in evolving details -(libation) shelf books [AIPK id, FK user shelf id, date added, order] -- technically, it's no violation to list a book more than once so use AIPK diff --git a/DataLayer/_schema and patterns.txt b/DataLayer/_schema and patterns.txt deleted file mode 100644 index c83abea8..00000000 --- a/DataLayer/_schema and patterns.txt +++ /dev/null @@ -1,76 +0,0 @@ -ignore for now: -authorProperties [PK/FK contributor id, AUDIBLE-PK author id] - notes in Contributor.cs for later refactoring - -c# enum only, not their own tables: -roles [AIPK id, name]. seeded: author, narrator, publisher. could expand (eg: translator, editor) without each needing a new table -origins [AIPK id, name]. seeded: library. detail. json. series - - --- begin SCHEMA --------------------------------------------------------------------------------------------------------------------- -any audible keys should be indexed - -SCHEMA -====== -contributors [AIPK id, name]. people and publishers -books [AIPK id, AUDIBLE-PK product id, title, desc, lengthInMinutes, picture id, 3 ratings, category id, origin id] -- product instances. each edition and version is discrete: unique and disconnected from different editions of the same book -- on book re-import - update: - update book origin and series origin with the new source type - overwrite simple fields - invoke complex contributor updates - details page gets - un/abridged - release date - language - publisher - series info incl name - categories - if new == series: ignore. do update series info. do not update book info - else if old == json: update (incl if new == json) - else if old == library && new == detail: update - else: ignore -book contributors [FK book id, FK contributor id, FK role id, order -- cluster PK across all FKs] -supplements [AIPK id, FK book id, download url] -categories [AIPK id, AUDIBLE-PK category id, name, parent]. may only nest 1 deep -user defined [PK/FK book id, 3 ratings, tagsRaw] -series [AIPK id, AUDIBLE-PK series id/asin, name, origin id] -series books [FK series id, FK book id, index -- cluster PK across all FKs] -- "index" not "order". display this number; don't just put in this sequence -- index is float instead of int to allow for in-between books. eg 2.5 -- to show 2 editions as the same book in a series, give them the same index -- re-import using series page, there will need to be a re-eval of import logic -library [PK/FK book id, date added, bookdownloadlink] --- end SCHEMA --------------------------------------------------------------------------------------------------------------------- - --- begin SIMPLIFIED DDD --------------------------------------------------------------------------------------------------------------------- -combine domain and persistence (C(r)UD). no repository pattern. encapsulated in domain objects; direct calls to EF Core -https://www.thereformedprogrammer.net/creating-domain-driven-design-entity-classes-with-entity-framework-core/ - // pattern for x-to-many - public void AddReview(int numStars, DbContext context = null) - { - if (_reviews != null) _reviews.Add(new Review(numStars)); - else if (context == null) throw new Exception("need context"); - else if (context.Entry(this).IsKeySet) context.Add(new Review(numStars, BookId)); - else throw new Exception("Could not add"); - } - - // pattern for optional one-to-one - MyPropClass MyProps { get; private set; } - public void AddMyProps(string s, int i, DbContext context = null) - { - // avoid a trip to the db - if (MyProps != null) { MyProps.Update(s, i); return; } - if (BookId == 0) { MyProps = new MyPropClass(s, i); return; } - if (context == null) throw new Exception("need context"); - // per Jon P Smith, this single trip to db loads the property if there is one - // note: .Reference() is for single object references. for collections use .Collection() - context.Entry(this).Reference(s => s.MyProps).Load(); - if (MyProps != null) MyProps.Update(s, i); - else MyProps = new MyPropClass(s, i); - } - -repository reads are 'query object'-like extension methods -https://www.thereformedprogrammer.net/is-the-repository-pattern-useful-with-entity-framework-core/#1-query-objects-a-way-to-isolate-and-hide-database-read-code --- and SIMPLIFIED DDD --------------------------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/DtoImporterService/LibraryImporter.cs b/DtoImporterService/LibraryImporter.cs index 776c3184..2259bccd 100644 --- a/DtoImporterService/LibraryImporter.cs +++ b/DtoImporterService/LibraryImporter.cs @@ -28,10 +28,7 @@ namespace DtoImporterService { var libraryBook = new LibraryBook( context.Books.Local.Single(b => b.AudibleProductId == newItem.ProductId), - newItem.DateAdded -// needed for scraping -//,FileManager.FileUtility.RestoreDeclawed(newLibraryDTO.DownloadBookLink) - ); + newItem.DateAdded); context.Library.Add(libraryBook); } diff --git a/ScrapingDomainServices/ScrapingDomainServices.csproj b/FileLiberator/FileLiberator.csproj similarity index 62% rename from ScrapingDomainServices/ScrapingDomainServices.csproj rename to FileLiberator/FileLiberator.csproj index d6878cbf..2060cd34 100644 --- a/ScrapingDomainServices/ScrapingDomainServices.csproj +++ b/FileLiberator/FileLiberator.csproj @@ -5,10 +5,11 @@ + - - + + diff --git a/ScrapingDomainServices/UNTESTED/BackupBook.cs b/FileLiberator/UNTESTED/BackupBook.cs similarity index 98% rename from ScrapingDomainServices/UNTESTED/BackupBook.cs rename to FileLiberator/UNTESTED/BackupBook.cs index c43e2032..eb2cc7c4 100644 --- a/ScrapingDomainServices/UNTESTED/BackupBook.cs +++ b/FileLiberator/UNTESTED/BackupBook.cs @@ -4,7 +4,7 @@ using DataLayer; using Dinah.Core.ErrorHandling; using FileManager; -namespace ScrapingDomainServices +namespace FileLiberator { /// /// Download DRM book and decrypt audiobook files. diff --git a/ScrapingDomainServices/UNTESTED/DecryptBook.cs b/FileLiberator/UNTESTED/DecryptBook.cs similarity index 99% rename from ScrapingDomainServices/UNTESTED/DecryptBook.cs rename to FileLiberator/UNTESTED/DecryptBook.cs index 73fa2d36..50533c89 100644 --- a/ScrapingDomainServices/UNTESTED/DecryptBook.cs +++ b/FileLiberator/UNTESTED/DecryptBook.cs @@ -9,7 +9,7 @@ using Dinah.Core; using Dinah.Core.ErrorHandling; using FileManager; -namespace ScrapingDomainServices +namespace FileLiberator { /// /// Download DRM book and decrypt audiobook files. @@ -39,7 +39,7 @@ namespace ScrapingDomainServices => await validateAsync_ConfigureAwaitFalse(libraryBook.Book.AudibleProductId).ConfigureAwait(false); private async Task validateAsync_ConfigureAwaitFalse(string productId) => await AudibleFileStorage.AAX.ExistsAsync(productId) - && !(await AudibleFileStorage.Audio.ExistsAsync(productId)); + && !await AudibleFileStorage.Audio.ExistsAsync(productId); // do NOT use ConfigureAwait(false) on ProcessUnregistered() // often does a lot with forms in the UI context diff --git a/ScrapingDomainServices/UNTESTED/DownloadBook.cs b/FileLiberator/UNTESTED/DownloadBook.cs similarity index 68% rename from ScrapingDomainServices/UNTESTED/DownloadBook.cs rename to FileLiberator/UNTESTED/DownloadBook.cs index 23359477..08bcaa0f 100644 --- a/ScrapingDomainServices/UNTESTED/DownloadBook.cs +++ b/FileLiberator/UNTESTED/DownloadBook.cs @@ -5,7 +5,7 @@ using FileManager; using DataLayer; using Dinah.Core.ErrorHandling; -namespace ScrapingDomainServices +namespace FileLiberator { /// /// Download DRM book and decrypt audiobook files. @@ -38,10 +38,8 @@ namespace ScrapingDomainServices // in cases where title includes '&', just use everything before the '&' and ignore the rest //// var adhTitle = product.Title.Split('&')[0] -// legacy/scraping method -//await performDownloadAsync(libraryBook, tempAaxFilename); -// new/api method -tempAaxFilename = await performApiDownloadAsync(libraryBook, tempAaxFilename); + // new/api method + tempAaxFilename = await performApiDownloadAsync(libraryBook, tempAaxFilename); // move var aaxFilename = FileUtility.GetValidFilename( @@ -54,33 +52,12 @@ tempAaxFilename = await performApiDownloadAsync(libraryBook, tempAaxFilename); var statusHandler = new StatusHandler(); var isDownloaded = await AudibleFileStorage.AAX.ExistsAsync(libraryBook.Book.AudibleProductId); if (isDownloaded) - DoStatusUpdate($"Downloaded: {aaxFilename}"); + Invoke_StatusUpdate($"Downloaded: {aaxFilename}"); else statusHandler.AddError("Downloaded AAX file cannot be found"); return statusHandler; } - // GetWebClientAsync: - // wires up webClient events - // [DownloadProgressChanged, DownloadFileCompleted, DownloadDataCompleted, DownloadStringCompleted] - // to DownloadableBase events - // DownloadProgressChanged, DownloadCompleted - // fires DownloadBegin event - // method begins async file download - private async Task performDownloadAsync(LibraryBook libraryBook, string tempAaxFilename) - { - var aaxDownloadLink = libraryBook.DownloadBookLink - .Replace("/admhelper", "") - .Replace("&DownloadType=Now", "") - + "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download"; - var uri = new Uri(aaxDownloadLink); - - using var webClient = await GetWebClientAsync(tempAaxFilename); - // for book downloads only: pretend to be the audible download manager. from inAudible: - webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601"; - await webClient.DownloadFileTaskAsync(uri, tempAaxFilename); - } - private async Task performApiDownloadAsync(LibraryBook libraryBook, string tempAaxFilename) { var api = await AudibleApi.EzApiCreator.GetApiAsync(AudibleApiStorage.IdentityTokensFile); diff --git a/FileLiberator/UNTESTED/DownloadPdf.cs b/FileLiberator/UNTESTED/DownloadPdf.cs new file mode 100644 index 00000000..c275da54 --- /dev/null +++ b/FileLiberator/UNTESTED/DownloadPdf.cs @@ -0,0 +1,105 @@ +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using DataLayer; +using Dinah.Core.ErrorHandling; +using FileManager; + +namespace FileLiberator +{ + public class DownloadPdf : DownloadableBase + { + static DownloadPdf() + { + // https://stackoverflow.com/a/15483698 + ServicePointManager.ServerCertificateValidationCallback = delegate { return true; }; + } + + public override async Task ValidateAsync(LibraryBook libraryBook) + { + var product = libraryBook.Book; + + if (!product.Supplements.Any()) + return false; + + return !await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId); + } + + public override async Task ProcessItemAsync(LibraryBook libraryBook) + { + var product = libraryBook.Book; + + if (product == null) + return new StatusHandler { "Book not found" }; + + var urls = product.Supplements.Select(d => d.Url).ToList(); + if (urls.Count == 0) + return new StatusHandler { "PDF download url not found" }; + + // sanity check + if (urls.Count > 1) + throw new Exception("Multiple PDF downloads are not currently supported. typically indicates an error"); + + var url = urls.Single(); + + var destinationDir = await getDestinationDirectory(product.AudibleProductId); + if (destinationDir == null) + return new StatusHandler { "Destination directory not found for PDF download" }; + + var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url)); + + using var webClient = GetWebClient(destinationFilename); + await webClient.DownloadFileTaskAsync(url, destinationFilename); + + var statusHandler = new StatusHandler(); + var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId); + if (!exists) + statusHandler.AddError("Downloaded PDF cannot be found"); + return statusHandler; + } + + private async Task getDestinationDirectory(string productId) + { + // if audio file exists, get it's dir + var audioFile = await AudibleFileStorage.Audio.GetAsync(productId); + if (audioFile != null) + return Path.GetDirectoryName(audioFile); + + // else return base Book dir + return AudibleFileStorage.PDF.StorageDirectory; + } + + // other user agents from my chrome. from: https://www.whoishostingthis.com/tools/user-agent/ + private static string[] userAgents { get; } = new[] + { + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36", + }; + private WebClient GetWebClient(string downloadMessage) + { + var webClient = new WebClient(); + + var userAgentIndex = new Random().Next(0, userAgents.Length); // upper bound is exclusive + webClient.Headers["User-Agent"] = userAgents[userAgentIndex]; + webClient.Headers["Referer"] = "https://google.com"; + webClient.Headers["Upgrade-Insecure-Requests"] = "1"; + webClient.Headers["DNT"] = "1"; + webClient.Headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"; + webClient.Headers["Accept-Language"] = "en-US,en;q=0.9"; + + webClient.DownloadProgressChanged += (s, e) => Invoke_DownloadProgressChanged(s, new Dinah.Core.Net.Http.DownloadProgress { BytesReceived = e.BytesReceived, ProgressPercentage = e.ProgressPercentage, TotalBytesToReceive = e.TotalBytesToReceive }); + webClient.DownloadFileCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}"); + webClient.DownloadDataCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}"); + webClient.DownloadStringCompleted += (s, e) => Invoke_DownloadCompleted(s, $"Completed: {downloadMessage}"); + + Invoke_DownloadBegin(downloadMessage); + + return webClient; + } + } +} diff --git a/FileLiberator/UNTESTED/DownloadableBase.cs b/FileLiberator/UNTESTED/DownloadableBase.cs new file mode 100644 index 00000000..02fd75f4 --- /dev/null +++ b/FileLiberator/UNTESTED/DownloadableBase.cs @@ -0,0 +1,45 @@ +using System; +using System.Threading.Tasks; +using DataLayer; +using Dinah.Core.ErrorHandling; + +namespace FileLiberator +{ + public abstract class DownloadableBase : IDownloadable + { + public event EventHandler Begin; + public event EventHandler Completed; + + public event EventHandler StatusUpdate; + public event EventHandler DownloadBegin; + public event EventHandler DownloadProgressChanged; + public event EventHandler DownloadCompleted; + protected void Invoke_StatusUpdate(string message) => StatusUpdate?.Invoke(this, message); + protected void Invoke_DownloadBegin(string downloadMessage) => DownloadBegin?.Invoke(this, downloadMessage); + protected void Invoke_DownloadProgressChanged(object sender, Dinah.Core.Net.Http.DownloadProgress progress) => DownloadProgressChanged?.Invoke(sender, progress); + protected void Invoke_DownloadCompleted(object sender, string str) => DownloadCompleted?.Invoke(sender, str); + + + public abstract Task ValidateAsync(LibraryBook libraryBook); + + public abstract Task ProcessItemAsync(LibraryBook libraryBook); + + // do NOT use ConfigureAwait(false) on ProcessUnregistered() + // often does a lot with forms in the UI context + public async Task ProcessAsync(LibraryBook libraryBook) + { + var displayMessage = $"[{libraryBook.Book.AudibleProductId}] {libraryBook.Book.Title}"; + + Begin?.Invoke(this, displayMessage); + + try + { + return await ProcessItemAsync(libraryBook); + } + finally + { + Completed?.Invoke(this, displayMessage); + } + } + } +} diff --git a/ScrapingDomainServices/UNTESTED/IDecryptable.cs b/FileLiberator/UNTESTED/IDecryptable.cs similarity index 93% rename from ScrapingDomainServices/UNTESTED/IDecryptable.cs rename to FileLiberator/UNTESTED/IDecryptable.cs index 1517b14d..14a75812 100644 --- a/ScrapingDomainServices/UNTESTED/IDecryptable.cs +++ b/FileLiberator/UNTESTED/IDecryptable.cs @@ -1,6 +1,6 @@ using System; -namespace ScrapingDomainServices +namespace FileLiberator { public interface IDecryptable : IProcessable { diff --git a/ScrapingDomainServices/UNTESTED/IDownloadable.cs b/FileLiberator/UNTESTED/IDownloadable.cs similarity index 89% rename from ScrapingDomainServices/UNTESTED/IDownloadable.cs rename to FileLiberator/UNTESTED/IDownloadable.cs index 7a41bc12..22b51464 100644 --- a/ScrapingDomainServices/UNTESTED/IDownloadable.cs +++ b/FileLiberator/UNTESTED/IDownloadable.cs @@ -1,6 +1,6 @@ using System; -namespace ScrapingDomainServices +namespace FileLiberator { public interface IDownloadable : IProcessable { diff --git a/ScrapingDomainServices/UNTESTED/IProcessable.cs b/FileLiberator/UNTESTED/IProcessable.cs similarity index 95% rename from ScrapingDomainServices/UNTESTED/IProcessable.cs rename to FileLiberator/UNTESTED/IProcessable.cs index 32a96b76..006cc800 100644 --- a/ScrapingDomainServices/UNTESTED/IProcessable.cs +++ b/FileLiberator/UNTESTED/IProcessable.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using DataLayer; using Dinah.Core.ErrorHandling; -namespace ScrapingDomainServices +namespace FileLiberator { public interface IProcessable { diff --git a/ScrapingDomainServices/UNTESTED/IProcessableExt.cs b/FileLiberator/UNTESTED/IProcessableExt.cs similarity index 68% rename from ScrapingDomainServices/UNTESTED/IProcessableExt.cs rename to FileLiberator/UNTESTED/IProcessableExt.cs index ce6519af..931358dd 100644 --- a/ScrapingDomainServices/UNTESTED/IProcessableExt.cs +++ b/FileLiberator/UNTESTED/IProcessableExt.cs @@ -1,10 +1,9 @@ using System; using System.Threading.Tasks; using DataLayer; -using Dinah.Core.Collections.Generic; using Dinah.Core.ErrorHandling; -namespace ScrapingDomainServices +namespace FileLiberator { public static class IProcessableExt { @@ -30,9 +29,6 @@ namespace ScrapingDomainServices return status; } - // i'd love to turn this into Task> - // since enumeration is a blocking operation, this won't be possible until - // 2019's C# 8 async streams, aka async enumerables, aka async iterators: https://blogs.msdn.microsoft.com/dotnet/2018/11/12/building-c-8-0/ public static async Task GetNextValidAsync(this IProcessable processable) { var libraryBooks = LibraryQueries.GetLibrary_Flat_NoTracking(); @@ -43,12 +39,5 @@ namespace ScrapingDomainServices return null; } - - public static async Task ProcessValidateLibraryBookAsync(this IProcessable processable, LibraryBook libraryBook) - { - if (!await processable.ValidateAsync(libraryBook)) - return new StatusHandler { "Validation failed" }; - return await processable.ProcessAsync(libraryBook); - } } } diff --git a/FileManager/UNTESTED/AudibleApiStorage.cs b/FileManager/UNTESTED/AudibleApiStorage.cs index f0941310..83859a44 100644 --- a/FileManager/UNTESTED/AudibleApiStorage.cs +++ b/FileManager/UNTESTED/AudibleApiStorage.cs @@ -4,6 +4,7 @@ namespace FileManager { public static class AudibleApiStorage { + // not customizable. don't move to config public static string IdentityTokensFile => Path.Combine(Configuration.Instance.LibationFiles, "IdentityTokens.json"); } } diff --git a/FileManager/UNTESTED/FileUtility.cs b/FileManager/UNTESTED/FileUtility.cs index 331edda0..7ad987f1 100644 --- a/FileManager/UNTESTED/FileUtility.cs +++ b/FileManager/UNTESTED/FileUtility.cs @@ -27,13 +27,6 @@ namespace FileManager return File.Exists(path); } - /// acceptable inputs: - /// example.txt - /// C:\Users\username\Desktop\example.txt - /// Returns full name and path of unused filename. including (#) - public static string GetValidFilename(string proposedPath) - => GetValidFilename(Path.GetDirectoryName(proposedPath), Path.GetFileNameWithoutExtension(proposedPath), Path.GetExtension(proposedPath)); - public static string GetValidFilename(string dirFullPath, string filename, string extension, params string[] metadataSuffixes) { if (string.IsNullOrWhiteSpace(dirFullPath)) @@ -78,21 +71,6 @@ namespace FileManager return property; } - public static string Declaw(string str) - => str - .Replace(" str - ?.Replace(" new string(title .Where(c => (char.IsLetterOrDigit(c))) diff --git a/FileManager/UNTESTED/WebpageStorage.cs b/FileManager/UNTESTED/WebpageStorage.cs deleted file mode 100644 index ab4c5e6e..00000000 --- a/FileManager/UNTESTED/WebpageStorage.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace FileManager -{ - public static class WebpageStorage - { - // not customizable. don't move to config - private static string PagesDirectory { get; } - = new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("Pages").FullName; - private static string BookDetailsDirectory { get; } - = new DirectoryInfo(PagesDirectory).CreateSubdirectory("Book Details").FullName; - - public static string GetLibraryBatchName() => "Library_" + DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); - public static string SavePageToBatch(string contents, string batchName, string extension) - { - var batch_dir = Path.Combine(PagesDirectory, batchName); - - Directory.CreateDirectory(batch_dir); - - var file = Path.Combine(batch_dir, batchName + '.' + extension.Trim('.')); - var filename = FileUtility.GetValidFilename(file); - File.WriteAllText(filename, contents); - - return filename; - } - - public static List GetJsonFiles(DirectoryInfo libDir) - => libDir == null - ? new List() - : Directory - .EnumerateFiles(libDir.FullName, "*.json") - .Select(f => new FileInfo(f)) - .ToList(); - - public static DirectoryInfo GetMostRecentLibraryDir() - { - var dir = Directory - .EnumerateDirectories(PagesDirectory, "Library_*") - .OrderBy(a => a) - .LastOrDefault(); - if (string.IsNullOrWhiteSpace(dir)) - return null; - return new DirectoryInfo(dir); - } - - public static FileInfo GetBookDetailHtmFileInfo(string productId) - { - var path = Path.Combine(BookDetailsDirectory, $"BookDetail-{productId}.htm"); - return new FileInfo(path); - } - - public static FileInfo GetBookDetailJsonFileInfo(string productId) - { - var path = Path.Combine(BookDetailsDirectory, $"BookDetail-{productId}.json"); - return new FileInfo(path); - } - - public static FileInfo SaveBookDetailsToHtm(string productId, string contents) - { - var fi = GetBookDetailHtmFileInfo(productId); - File.WriteAllText(fi.FullName, contents); - return fi; - } - } -} diff --git a/InternalUtilities/InternalUtilities.csproj b/InternalUtilities/InternalUtilities.csproj index 312da9e6..531ac0e6 100644 --- a/InternalUtilities/InternalUtilities.csproj +++ b/InternalUtilities/InternalUtilities.csproj @@ -6,7 +6,7 @@ - + diff --git a/InternalUtilities/UNTESTED/AudibleApiActions.cs b/InternalUtilities/UNTESTED/AudibleApiActions.cs index 2c256262..4e7f28fc 100644 --- a/InternalUtilities/UNTESTED/AudibleApiActions.cs +++ b/InternalUtilities/UNTESTED/AudibleApiActions.cs @@ -33,7 +33,7 @@ namespace InternalUtilities var items = await AudibleApiExtensions.GetAllLibraryItemsAsync(api); // remove episode parents - items.RemoveAll(i => i.Episodes); + items.RemoveAll(i => i.IsEpisodes); #region // episode handling. doesn't quite work // // add individual/children episodes diff --git a/InternalUtilities/UNTESTED/DataConverter.cs b/InternalUtilities/UNTESTED/DataConverter.cs deleted file mode 100644 index c6f8f10a..00000000 --- a/InternalUtilities/UNTESTED/DataConverter.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System.IO; -using AudibleDotCom; -using FileManager; -using Newtonsoft.Json; - -namespace InternalUtilities -{ - public static partial class DataConverter - { - // also need: htm file => PageSource - - public static AudiblePageSource HtmFile_2_AudiblePageSource(string htmFilepath) - { - var htmContentsDeclawed = File.ReadAllText(htmFilepath); - var htmContents = FileUtility.RestoreDeclawed(htmContentsDeclawed); - return AudiblePageSource.Deserialize(htmContents); - } - - public static FileInfo Value_2_JsonFile(object value, string jsonFilepath) - { - var json = JsonConvert.SerializeObject(value, Formatting.Indented); - - File.WriteAllText(jsonFilepath, json); - - return new FileInfo(jsonFilepath); - } - - /// AudiblePageSource => declawed htm file - /// path of htm file - public static FileInfo AudiblePageSource_2_HtmFile_Batch(AudiblePageSource audiblePageSource, string batchName) - { - var source = audiblePageSource.Declawed().Serialized(); - var htmFile = WebpageStorage.SavePageToBatch(source, batchName, "htm"); - return new FileInfo(htmFile); - } - - /// AudiblePageSource => declawed htm file - /// path of htm file - public static FileInfo AudiblePageSource_2_HtmFile_Product(AudiblePageSource audiblePageSource) - { - if (audiblePageSource.AudiblePage == AudiblePageType.ProductDetails) - { - var source = audiblePageSource.Declawed().Serialized(); - var htmFile = WebpageStorage.SaveBookDetailsToHtm(audiblePageSource.PageId, source); - return htmFile; - } - - throw new System.NotImplementedException(); - } - } -} diff --git a/Libation.sln b/Libation.sln index f136b8be..1ce073c3 100644 --- a/Libation.sln +++ b/Libation.sln @@ -27,15 +27,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FileManager", "FileManager\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataLayer", "DataLayer\DataLayer.csproj", "{59A10DF3-63EC-43F1-A3BF-4000CFA118D2}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleDotCom", "AudibleDotCom\AudibleDotCom.csproj", "{4ABB61D3-4959-4F09-883A-9EDC8CE473FB}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Scraping", "Scraping\Scraping.csproj", "{C2C89551-44FD-41E4-80D3-69AF8CE3F174}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleDotComAutomation", "AudibleDotComAutomation\AudibleDotComAutomation.csproj", "{4CDE10DD-60EC-4CCA-99D1-75224A201C89}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CookieMonster", "CookieMonster\CookieMonster.csproj", "{7BD02E29-3430-4D06-88D2-5CECEE9ABD01}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ScrapingDomainServices", "ScrapingDomainServices\ScrapingDomainServices.csproj", "{393B5B27-D15C-4F77-9457-FA14BA8F3C73}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FileLiberator", "FileLiberator\FileLiberator.csproj", "{393B5B27-D15C-4F77-9457-FA14BA8F3C73}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "InternalUtilities", "InternalUtilities\InternalUtilities.csproj", "{06882742-27A6-4347-97D9-56162CEC9C11}" EndProject @@ -107,22 +99,6 @@ Global {59A10DF3-63EC-43F1-A3BF-4000CFA118D2}.Debug|Any CPU.Build.0 = Debug|Any CPU {59A10DF3-63EC-43F1-A3BF-4000CFA118D2}.Release|Any CPU.ActiveCfg = Release|Any CPU {59A10DF3-63EC-43F1-A3BF-4000CFA118D2}.Release|Any CPU.Build.0 = Release|Any CPU - {4ABB61D3-4959-4F09-883A-9EDC8CE473FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4ABB61D3-4959-4F09-883A-9EDC8CE473FB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4ABB61D3-4959-4F09-883A-9EDC8CE473FB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4ABB61D3-4959-4F09-883A-9EDC8CE473FB}.Release|Any CPU.Build.0 = Release|Any CPU - {C2C89551-44FD-41E4-80D3-69AF8CE3F174}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C2C89551-44FD-41E4-80D3-69AF8CE3F174}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C2C89551-44FD-41E4-80D3-69AF8CE3F174}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C2C89551-44FD-41E4-80D3-69AF8CE3F174}.Release|Any CPU.Build.0 = Release|Any CPU - {4CDE10DD-60EC-4CCA-99D1-75224A201C89}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4CDE10DD-60EC-4CCA-99D1-75224A201C89}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4CDE10DD-60EC-4CCA-99D1-75224A201C89}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4CDE10DD-60EC-4CCA-99D1-75224A201C89}.Release|Any CPU.Build.0 = Release|Any CPU - {7BD02E29-3430-4D06-88D2-5CECEE9ABD01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7BD02E29-3430-4D06-88D2-5CECEE9ABD01}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7BD02E29-3430-4D06-88D2-5CECEE9ABD01}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7BD02E29-3430-4D06-88D2-5CECEE9ABD01}.Release|Any CPU.Build.0 = Release|Any CPU {393B5B27-D15C-4F77-9457-FA14BA8F3C73}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {393B5B27-D15C-4F77-9457-FA14BA8F3C73}.Debug|Any CPU.Build.0 = Debug|Any CPU {393B5B27-D15C-4F77-9457-FA14BA8F3C73}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -227,10 +203,6 @@ Global {8BD8E012-F44F-4EE2-A234-D66C14D5FE4B} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} {1AE65B61-9C05-4C80-ABFF-48F16E22FDF1} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} {59A10DF3-63EC-43F1-A3BF-4000CFA118D2} = {751093DD-5DBA-463E-ADBE-E05FAFB6983E} - {4ABB61D3-4959-4F09-883A-9EDC8CE473FB} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} - {C2C89551-44FD-41E4-80D3-69AF8CE3F174} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} - {4CDE10DD-60EC-4CCA-99D1-75224A201C89} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} - {7BD02E29-3430-4D06-88D2-5CECEE9ABD01} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05} {393B5B27-D15C-4F77-9457-FA14BA8F3C73} = {41CDCC73-9B81-49DD-9570-C54406E852AF} {06882742-27A6-4347-97D9-56162CEC9C11} = {F0CBB7A7-D3FB-41FF-8F47-CF3F6A592249} {2E1F5DB4-40CC-4804-A893-5DCE0193E598} = {41CDCC73-9B81-49DD-9570-C54406E852AF} diff --git a/LibationSearchEngine/UNTESTED/SearchEngine.cs b/LibationSearchEngine/UNTESTED/SearchEngine.cs index 3be8c0ff..ad41ae39 100644 --- a/LibationSearchEngine/UNTESTED/SearchEngine.cs +++ b/LibationSearchEngine/UNTESTED/SearchEngine.cs @@ -231,8 +231,8 @@ namespace LibationSearchEngine return doc; } - public async Task UpdateBookAsync(string productId) => await Task.Run(() => updateBook(productId)); - private void updateBook(string productId) + /// Long running. Use await Task.Run(() => UpdateBook(productId)) + public void UpdateBook(string productId) { var libraryBook = LibraryQueries.GetLibraryBook_Flat_NoTracking(productId); var term = new Term(_ID_, productId); diff --git a/LibationWinForm/LibationWinForm.csproj b/LibationWinForm/LibationWinForm.csproj index d48e56bf..194f5e60 100644 --- a/LibationWinForm/LibationWinForm.csproj +++ b/LibationWinForm/LibationWinForm.csproj @@ -8,13 +8,9 @@ - - - - - + diff --git a/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.Designer.cs b/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.Designer.cs deleted file mode 100644 index 72d94470..00000000 --- a/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.Designer.cs +++ /dev/null @@ -1,129 +0,0 @@ -namespace LibationWinForm.BookLiberation -{ - partial class NoLongerAvailableForm - { - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Windows Form Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - this.label1 = new System.Windows.Forms.Label(); - this.textBox1 = new System.Windows.Forms.TextBox(); - this.missingBtn = new System.Windows.Forms.Button(); - this.abortBtn = new System.Windows.Forms.Button(); - this.label2 = new System.Windows.Forms.Label(); - this.label3 = new System.Windows.Forms.Label(); - this.SuspendLayout(); - // - // label1 - // - this.label1.AutoSize = true; - this.label1.Location = new System.Drawing.Point(12, 9); - this.label1.Name = "label1"; - this.label1.Size = new System.Drawing.Size(174, 39); - this.label1.TabIndex = 0; - this.label1.Text = "Book details download failed.\r\n{0} may be no longer available.\r\nVerify the book i" + - "s still available here"; - // - // textBox1 - // - this.textBox1.Location = new System.Drawing.Point(15, 51); - this.textBox1.Name = "textBox1"; - this.textBox1.ReadOnly = true; - this.textBox1.Size = new System.Drawing.Size(384, 20); - this.textBox1.TabIndex = 1; - // - // missingBtn - // - this.missingBtn.Location = new System.Drawing.Point(324, 77); - this.missingBtn.Name = "missingBtn"; - this.missingBtn.Size = new System.Drawing.Size(75, 23); - this.missingBtn.TabIndex = 3; - this.missingBtn.Text = "Missing"; - this.missingBtn.UseVisualStyleBackColor = true; - this.missingBtn.Click += new System.EventHandler(this.missingBtn_Click); - // - // abortBtn - // - this.abortBtn.Location = new System.Drawing.Point(324, 126); - this.abortBtn.Name = "abortBtn"; - this.abortBtn.Size = new System.Drawing.Size(75, 23); - this.abortBtn.TabIndex = 5; - this.abortBtn.Text = "Abort"; - this.abortBtn.UseVisualStyleBackColor = true; - this.abortBtn.Click += new System.EventHandler(this.abortBtn_Click); - // - // label2 - // - this.label2.AutoSize = true; - this.label2.Location = new System.Drawing.Point(12, 74); - this.label2.Name = "label2"; - this.label2.Size = new System.Drawing.Size(306, 26); - this.label2.TabIndex = 2; - this.label2.Text = "If the book is not available, click here to mark it as missing\r\nNo further book d" + - "etails download will be attempted for this book"; - // - // label3 - // - this.label3.AutoSize = true; - this.label3.Location = new System.Drawing.Point(12, 123); - this.label3.Name = "label3"; - this.label3.Size = new System.Drawing.Size(204, 26); - this.label3.TabIndex = 4; - this.label3.Text = "If the book is actually available, click here\r\nto abort and try again later"; - // - // NoLongerAvailableForm - // - this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); - this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.ClientSize = new System.Drawing.Size(411, 161); - this.Controls.Add(this.label3); - this.Controls.Add(this.label2); - this.Controls.Add(this.abortBtn); - this.Controls.Add(this.missingBtn); - this.Controls.Add(this.textBox1); - this.Controls.Add(this.label1); - this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedDialog; - this.MaximizeBox = false; - this.MinimizeBox = false; - this.Name = "NoLongerAvailableForm"; - this.ShowIcon = false; - this.ShowInTaskbar = false; - this.StartPosition = System.Windows.Forms.FormStartPosition.CenterParent; - this.Text = "No Longer Available"; - this.ResumeLayout(false); - this.PerformLayout(); - - } - - #endregion - - private System.Windows.Forms.Label label1; - private System.Windows.Forms.TextBox textBox1; - private System.Windows.Forms.Button missingBtn; - private System.Windows.Forms.Button abortBtn; - private System.Windows.Forms.Label label2; - private System.Windows.Forms.Label label3; - } -} \ No newline at end of file diff --git a/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.cs b/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.cs deleted file mode 100644 index 826cdbf4..00000000 --- a/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using System.Windows.Forms; -using ScrapingDomainServices; - -namespace LibationWinForm.BookLiberation -{ - public partial class NoLongerAvailableForm : Form - { - public ScrapeBookDetails.NoLongerAvailableEnum EnumResult { get; private set; } - - public NoLongerAvailableForm(string title, string url) : this() - { - this.Text += ": " + title; - this.label1.Text = string.Format(this.label1.Text, title); - this.textBox1.Text = url; - } - public NoLongerAvailableForm() => InitializeComponent(); - - private void missingBtn_Click(object sender, EventArgs e) => complete(ScrapeBookDetails.NoLongerAvailableEnum.MarkAsMissing); - private void abortBtn_Click(object sender, EventArgs e) => complete(ScrapeBookDetails.NoLongerAvailableEnum.Abort); - - private void complete(ScrapeBookDetails.NoLongerAvailableEnum nlaEnum) - { - EnumResult = nlaEnum; - Close(); - } - } -} diff --git a/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.resx b/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.resx deleted file mode 100644 index 1af7de15..00000000 --- a/LibationWinForm/UNTESTED/BookLiberation/NoLongerAvailableForm.resx +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - \ No newline at end of file diff --git a/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs b/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs index aea0ef76..f5dbe7b5 100644 --- a/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs +++ b/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs @@ -3,7 +3,8 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using DataLayer; -using ScrapingDomainServices; +using Dinah.Core.ErrorHandling; +using FileLiberator; namespace LibationWinForm.BookLiberation { @@ -24,11 +25,18 @@ namespace LibationWinForm.BookLiberation var backupBook = new BackupBook(); backupBook.Download.Completed += SetBackupCountsAsync; backupBook.Decrypt.Completed += SetBackupCountsAsync; - await backupBook.ProcessValidateLibraryBookAsync(libraryBook); - } + await ProcessValidateLibraryBookAsync(backupBook, libraryBook); + } - // Download First Book (Download encrypted/DRM file) - async Task DownloadFirstBookAsync() + static async Task ProcessValidateLibraryBookAsync(IProcessable processable, LibraryBook libraryBook) + { + if (!await processable.ValidateAsync(libraryBook)) + return new StatusHandler { "Validation failed" }; + return await processable.ProcessAsync(libraryBook); + } + + // Download First Book (Download encrypted/DRM file) + async Task DownloadFirstBookAsync() { var downloadBook = ProcessorAutomationController.GetWiredUpDownloadBook(); downloadBook.Completed += SetBackupCountsAsync; diff --git a/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.cs b/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.cs index 4f31c189..b9c10752 100644 --- a/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.cs +++ b/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.cs @@ -1,6 +1,6 @@ using System; using System.Threading.Tasks; -using ScrapingDomainServices; +using FileLiberator; namespace LibationWinForm.BookLiberation { @@ -39,21 +39,6 @@ namespace LibationWinForm.BookLiberation downloadPdf.Begin += (_, __) => wireUpDownloadable(downloadPdf); return downloadPdf; } - public static ScrapeBookDetails GetWiredUpScrapeBookDetails() - { - var scrapeBookDetails = new ScrapeBookDetails(); - scrapeBookDetails.Begin += (_, __) => wireUpDownloadable(scrapeBookDetails); - - scrapeBookDetails.NoLongerAvailableAction = noLongerAvailableUI; - - return scrapeBookDetails; - } - static ScrapeBookDetails.NoLongerAvailableEnum noLongerAvailableUI(string title, string url) - { - var nla = new NoLongerAvailableForm(title, url); - nla.ShowDialog(); - return nla.EnumResult; - } // subscribed to Begin event because a new form should be created+processed+closed on each iteration private static void wireUpDownloadable(IDownloadable downloadable) diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IRunnableDialog.cs b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IRunnableDialog.cs index 066f261f..49cb173a 100644 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IRunnableDialog.cs +++ b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IRunnableDialog.cs @@ -7,7 +7,7 @@ using System.Windows.Forms; namespace LibationWinForm { - public interface IRunnableDialog : IValidatable + public interface IRunnableDialog { IButtonControl AcceptButton { get; set; } Control.ControlCollection Controls { get; } diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IRunnableDialogExt.cs b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IRunnableDialogExt.cs index 514604ed..8f87f28c 100644 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IRunnableDialogExt.cs +++ b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IRunnableDialogExt.cs @@ -36,26 +36,6 @@ namespace LibationWinForm public static async Task Run(this IRunnableDialog dialog) { - // validate children - // OfType() -- skips items which aren't of the required type - // Cast() -- throws an exception - var errorStrings = dialog - // get children - .Controls - .GetControlListRecursive() - .OfType() - // and self - .Append(dialog) - // validate. get errors - .Select(c => c.StringBasedValidate()) - // ignore successes - .Where(e => e != null); - if (errorStrings.Any()) - { - MessageBox.Show(errorStrings.Aggregate((a, b) => a + "\r\n" + b)); - return; - } - // get top level controls only. If Enabled, disable and push on stack var disabledStack = disable(dialog); diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IValidatable.cs b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IValidatable.cs deleted file mode 100644 index 8fb2f51f..00000000 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/IValidatable.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace LibationWinForm -{ - public interface IValidatable - { - // forms has a framework for ValidateChildren and ErrorProvider.s - // i don't feel like setting it up right now. doing this instead - string StringBasedValidate(); - } -} diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.Designer.cs b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.Designer.cs deleted file mode 100644 index 0718ff5e..00000000 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.Designer.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace LibationWinForm -{ - partial class ScanLibraryDialog - { - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Windows Form Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - this.websiteProcessorControl1 = new LibationWinForm.WebsiteProcessorControl(); - this.BeginScanBtn = new System.Windows.Forms.Button(); - this.SuspendLayout(); - // - // websiteProcessorControl1 - // - this.websiteProcessorControl1.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) - | System.Windows.Forms.AnchorStyles.Left) - | System.Windows.Forms.AnchorStyles.Right))); - this.websiteProcessorControl1.Location = new System.Drawing.Point(12, 12); - this.websiteProcessorControl1.Name = "websiteProcessorControl1"; - this.websiteProcessorControl1.Size = new System.Drawing.Size(324, 137); - this.websiteProcessorControl1.TabIndex = 0; - // - // BeginScanBtn - // - this.BeginScanBtn.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Bottom | System.Windows.Forms.AnchorStyles.Left) - | System.Windows.Forms.AnchorStyles.Right))); - this.BeginScanBtn.Location = new System.Drawing.Point(12, 155); - this.BeginScanBtn.Name = "BeginScanBtn"; - this.BeginScanBtn.Size = new System.Drawing.Size(324, 23); - this.BeginScanBtn.TabIndex = 1; - this.BeginScanBtn.Text = "BEGIN SCAN"; - this.BeginScanBtn.UseVisualStyleBackColor = true; - // - // ScanLibraryDialog - // - this.AcceptButton = this.BeginScanBtn; - this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); - this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.ClientSize = new System.Drawing.Size(348, 190); - this.Controls.Add(this.BeginScanBtn); - this.Controls.Add(this.websiteProcessorControl1); - this.Name = "ScanLibraryDialog"; - this.ShowIcon = false; - this.ShowInTaskbar = false; - this.StartPosition = System.Windows.Forms.FormStartPosition.CenterParent; - this.Text = "Scan Library"; - this.ResumeLayout(false); - - } - - #endregion - - private WebsiteProcessorControl websiteProcessorControl1; - private System.Windows.Forms.Button BeginScanBtn; - } -} \ No newline at end of file diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs deleted file mode 100644 index 10350e13..00000000 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs +++ /dev/null @@ -1,41 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; -using System.Windows.Forms; -using Dinah.Core; -using ScrapingDomainServices; - -namespace LibationWinForm -{ - public partial class ScanLibraryDialog : Form, IIndexLibraryDialog - { - public ScanLibraryDialog() - { - InitializeComponent(); - } - - public string StringBasedValidate() => null; - - List successMessages { get; } = new List(); - public string SuccessMessage => string.Join("\r\n", successMessages); - - public int NewBooksAdded { get; private set; } - public int TotalBooksProcessed { get; private set; } - - public async Task DoMainWorkAsync() - { - using var pageRetriever = websiteProcessorControl1.GetPageRetriever(); - var jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false); - - successMessages.Add($"Downloaded {"library page".PluralizeWithCount(jsonFilepaths.Count)}"); - - (TotalBooksProcessed, NewBooksAdded) = await Indexer - .IndexLibraryAsync(jsonFilepaths) - .ConfigureAwait(false); - - successMessages.Add($"Total processed: {TotalBooksProcessed}"); - successMessages.Add($"New: {NewBooksAdded}"); - } - } -} diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.resx b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.resx deleted file mode 100644 index 1af7de15..00000000 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.resx +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - \ No newline at end of file diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.Designer.cs b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.Designer.cs deleted file mode 100644 index 2641d1c1..00000000 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.Designer.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace LibationWinForm -{ - partial class WebsiteProcessorControl - { - /// - /// Required designer variable. - /// - private System.ComponentModel.IContainer components = null; - - /// - /// Clean up any resources being used. - /// - /// true if managed resources should be disposed; otherwise, false. - protected override void Dispose(bool disposing) - { - if (disposing && (components != null)) - { - components.Dispose(); - } - base.Dispose(disposing); - } - - #region Component Designer generated code - - /// - /// Required method for Designer support - do not modify - /// the contents of this method with the code editor. - /// - private void InitializeComponent() - { - this.AuthGb = new System.Windows.Forms.GroupBox(); - this.AuthRb_Browserless = new System.Windows.Forms.RadioButton(); - this.AuthRb_UseCanonicalChrome = new System.Windows.Forms.RadioButton(); - this.label3 = new System.Windows.Forms.Label(); - this.AuthRb_ManualLogin = new System.Windows.Forms.RadioButton(); - this.label2 = new System.Windows.Forms.Label(); - this.PasswordTb = new System.Windows.Forms.TextBox(); - this.UsernameTb = new System.Windows.Forms.TextBox(); - this.AuthGb.SuspendLayout(); - this.SuspendLayout(); - // - // AuthGb - // - this.AuthGb.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) - | System.Windows.Forms.AnchorStyles.Left) - | System.Windows.Forms.AnchorStyles.Right))); - this.AuthGb.Controls.Add(this.AuthRb_Browserless); - this.AuthGb.Controls.Add(this.AuthRb_UseCanonicalChrome); - this.AuthGb.Controls.Add(this.label3); - this.AuthGb.Controls.Add(this.AuthRb_ManualLogin); - this.AuthGb.Controls.Add(this.label2); - this.AuthGb.Controls.Add(this.PasswordTb); - this.AuthGb.Controls.Add(this.UsernameTb); - this.AuthGb.Location = new System.Drawing.Point(0, 0); - this.AuthGb.Name = "AuthGb"; - this.AuthGb.Size = new System.Drawing.Size(324, 137); - this.AuthGb.TabIndex = 1; - this.AuthGb.TabStop = false; - this.AuthGb.Text = "Authentication"; - // - // AuthRb_Browserless - // - this.AuthRb_Browserless.AutoSize = true; - this.AuthRb_Browserless.Checked = true; - this.AuthRb_Browserless.Location = new System.Drawing.Point(6, 19); - this.AuthRb_Browserless.Name = "AuthRb_Browserless"; - this.AuthRb_Browserless.Size = new System.Drawing.Size(143, 17); - this.AuthRb_Browserless.TabIndex = 0; - this.AuthRb_Browserless.TabStop = true; - this.AuthRb_Browserless.Text = "Browserless with cookies"; - this.AuthRb_Browserless.UseVisualStyleBackColor = true; - // - // AuthRb_UseCanonicalChrome - // - this.AuthRb_UseCanonicalChrome.AutoSize = true; - this.AuthRb_UseCanonicalChrome.Location = new System.Drawing.Point(6, 114); - this.AuthRb_UseCanonicalChrome.Name = "AuthRb_UseCanonicalChrome"; - this.AuthRb_UseCanonicalChrome.Size = new System.Drawing.Size(216, 17); - this.AuthRb_UseCanonicalChrome.TabIndex = 6; - this.AuthRb_UseCanonicalChrome.Text = "Use Canonical Chrome. SEE WARNING"; - this.AuthRb_UseCanonicalChrome.UseVisualStyleBackColor = true; - this.AuthRb_UseCanonicalChrome.CheckedChanged += new System.EventHandler(this.AuthRb_UseCanonicalChrome_CheckedChanged); - // - // label3 - // - this.label3.AutoSize = true; - this.label3.Location = new System.Drawing.Point(27, 91); - this.label3.Name = "label3"; - this.label3.Size = new System.Drawing.Size(53, 13); - this.label3.TabIndex = 4; - this.label3.Text = "Password"; - // - // AuthRb_ManualLogin - // - this.AuthRb_ManualLogin.AutoSize = true; - this.AuthRb_ManualLogin.Location = new System.Drawing.Point(6, 42); - this.AuthRb_ManualLogin.Name = "AuthRb_ManualLogin"; - this.AuthRb_ManualLogin.Size = new System.Drawing.Size(89, 17); - this.AuthRb_ManualLogin.TabIndex = 1; - this.AuthRb_ManualLogin.Text = "Manual Login"; - this.AuthRb_ManualLogin.UseVisualStyleBackColor = true; - // - // label2 - // - this.label2.AutoSize = true; - this.label2.Location = new System.Drawing.Point(27, 65); - this.label2.Name = "label2"; - this.label2.Size = new System.Drawing.Size(85, 13); - this.label2.TabIndex = 2; - this.label2.Text = "Username/Email"; - // - // PasswordTb - // - this.PasswordTb.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) - | System.Windows.Forms.AnchorStyles.Right))); - this.PasswordTb.Location = new System.Drawing.Point(118, 88); - this.PasswordTb.Name = "PasswordTb"; - this.PasswordTb.PasswordChar = '*'; - this.PasswordTb.Size = new System.Drawing.Size(200, 20); - this.PasswordTb.TabIndex = 5; - this.PasswordTb.TextChanged += new System.EventHandler(this.UserIsEnteringLoginInfo); - this.PasswordTb.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.UsernamePasswordTb_KeyPress); - this.PasswordTb.MouseUp += new System.Windows.Forms.MouseEventHandler(this.UserIsEnteringLoginInfo); - // - // UsernameTb - // - this.UsernameTb.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) - | System.Windows.Forms.AnchorStyles.Right))); - this.UsernameTb.Location = new System.Drawing.Point(118, 62); - this.UsernameTb.Name = "UsernameTb"; - this.UsernameTb.Size = new System.Drawing.Size(200, 20); - this.UsernameTb.TabIndex = 3; - this.UsernameTb.TextChanged += new System.EventHandler(this.UserIsEnteringLoginInfo); - this.UsernameTb.KeyPress += new System.Windows.Forms.KeyPressEventHandler(this.UsernamePasswordTb_KeyPress); - this.UsernameTb.MouseUp += new System.Windows.Forms.MouseEventHandler(this.UserIsEnteringLoginInfo); - // - // WebsiteProcessorControl - // - this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); - this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - this.Controls.Add(this.AuthGb); - this.Name = "WebsiteProcessorControl"; - this.Size = new System.Drawing.Size(324, 137); - this.AuthGb.ResumeLayout(false); - this.AuthGb.PerformLayout(); - this.ResumeLayout(false); - - } - - #endregion - - private System.Windows.Forms.GroupBox AuthGb; - private System.Windows.Forms.RadioButton AuthRb_UseCanonicalChrome; - private System.Windows.Forms.Label label3; - private System.Windows.Forms.RadioButton AuthRb_ManualLogin; - private System.Windows.Forms.Label label2; - private System.Windows.Forms.TextBox PasswordTb; - private System.Windows.Forms.TextBox UsernameTb; - private System.Windows.Forms.RadioButton AuthRb_Browserless; - } -} diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.cs b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.cs deleted file mode 100644 index 493826d0..00000000 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Windows.Forms; -using AudibleDotComAutomation; - -namespace LibationWinForm -{ - public partial class WebsiteProcessorControl : UserControl, IValidatable - { - public event EventHandler KeyPressSubmit; - - public WebsiteProcessorControl() - { - InitializeComponent(); - } - - public IPageRetriever GetPageRetriever() - => AuthRb_UseCanonicalChrome.Checked ? new UserDataSeleniumRetriever() - : AuthRb_Browserless.Checked ? (IPageRetriever)new BrowserlessRetriever() - : new ManualLoginSeleniumRetriever(UsernameTb.Text, PasswordTb.Text); - - public string StringBasedValidate() - { - if (AuthRb_ManualLogin.Checked && (string.IsNullOrWhiteSpace(UsernameTb.Text) || string.IsNullOrWhiteSpace(PasswordTb.Text))) - return "must fill in username and password"; - - return null; - } - - private void UsernamePasswordTb_KeyPress(object sender, KeyPressEventArgs e) - { - if (e.KeyChar == (char)Keys.Return) - { - KeyPressSubmit?.Invoke(sender, e); - // call your method for action on enter - e.Handled = true; // suppress default handling - } - } - - private void UserIsEnteringLoginInfo(object sender, EventArgs e) => AuthRb_ManualLogin.Checked = true; - - private void AuthRb_UseCanonicalChrome_CheckedChanged(object sender, EventArgs e) - { - if (AuthRb_UseCanonicalChrome.Checked) - MessageBox.Show(@"A canonical version of Chrome will be used including User Data, cookies. etc. Selenium chromedriver won't launch URL if another Chrome instance is open"); - } - } -} diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.resx b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.resx deleted file mode 100644 index 1af7de15..00000000 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/WebsiteProcessorControl.resx +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/microsoft-resx - - - 2.0 - - - System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - \ No newline at end of file diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexLibraryDialog.cs b/LibationWinForm/UNTESTED/Dialogs/IndexLibraryDialog.cs index 6eb19d7b..9c173522 100644 --- a/LibationWinForm/UNTESTED/Dialogs/IndexLibraryDialog.cs +++ b/LibationWinForm/UNTESTED/Dialogs/IndexLibraryDialog.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using System.Windows.Forms; -using ApplicationService; +using ApplicationServices; namespace LibationWinForm { @@ -22,8 +22,6 @@ namespace LibationWinForm this.Shown += (_, __) => AcceptButton.PerformClick(); } - public string StringBasedValidate() => null; - List successMessages { get; } = new List(); public string SuccessMessage => string.Join("\r\n", successMessages); @@ -33,8 +31,8 @@ namespace LibationWinForm public async Task DoMainWorkAsync() { var callback = new Login.WinformResponder(); - var refresher = new LibraryIndexer(); - (TotalBooksProcessed, NewBooksAdded) = await refresher.IndexAsync(callback); + var indexer = new LibraryIndexer(); + (TotalBooksProcessed, NewBooksAdded) = await indexer.IndexAsync(callback); successMessages.Add($"Total processed: {TotalBooksProcessed}"); successMessages.Add($"New: {NewBooksAdded}"); diff --git a/LibationWinForm/UNTESTED/Form1.Designer.cs b/LibationWinForm/UNTESTED/Form1.Designer.cs index 59f30977..d045f583 100644 --- a/LibationWinForm/UNTESTED/Form1.Designer.cs +++ b/LibationWinForm/UNTESTED/Form1.Designer.cs @@ -34,10 +34,8 @@ this.filterBtn = new System.Windows.Forms.Button(); this.filterSearchTb = new System.Windows.Forms.TextBox(); this.menuStrip1 = new System.Windows.Forms.MenuStrip(); - this.indexToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.importToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.scanLibraryToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); - this.reimportMostRecentLibraryScanToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); - this.beginImportingBookDetailsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.liberateToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.beginBookBackupsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.beginPdfBackupsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); @@ -100,7 +98,7 @@ // menuStrip1 // this.menuStrip1.Items.AddRange(new System.Windows.Forms.ToolStripItem[] { - this.indexToolStripMenuItem, + this.importToolStripMenuItem, this.liberateToolStripMenuItem, this.quickFiltersToolStripMenuItem, this.settingsToolStripMenuItem}); @@ -110,38 +108,21 @@ this.menuStrip1.TabIndex = 0; this.menuStrip1.Text = "menuStrip1"; // - // indexToolStripMenuItem + // importToolStripMenuItem // - this.indexToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { - this.scanLibraryToolStripMenuItem, - this.reimportMostRecentLibraryScanToolStripMenuItem, - this.beginImportingBookDetailsToolStripMenuItem}); - this.indexToolStripMenuItem.Name = "indexToolStripMenuItem"; - this.indexToolStripMenuItem.Size = new System.Drawing.Size(47, 20); - this.indexToolStripMenuItem.Text = "&Index"; - this.indexToolStripMenuItem.DropDownOpening += new System.EventHandler(this.indexToolStripMenuItem_DropDownOpening); + this.importToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { + this.scanLibraryToolStripMenuItem}); + this.importToolStripMenuItem.Name = "importToolStripMenuItem"; + this.importToolStripMenuItem.Size = new System.Drawing.Size(47, 20); + this.importToolStripMenuItem.Text = "&Import"; // // scanLibraryToolStripMenuItem // this.scanLibraryToolStripMenuItem.Name = "scanLibraryToolStripMenuItem"; this.scanLibraryToolStripMenuItem.Size = new System.Drawing.Size(277, 22); - this.scanLibraryToolStripMenuItem.Text = "Scan &Library..."; + this.scanLibraryToolStripMenuItem.Text = "Scan &Library"; this.scanLibraryToolStripMenuItem.Click += new System.EventHandler(this.scanLibraryToolStripMenuItem_Click); // - // reimportMostRecentLibraryScanToolStripMenuItem - // - this.reimportMostRecentLibraryScanToolStripMenuItem.Name = "reimportMostRecentLibraryScanToolStripMenuItem"; - this.reimportMostRecentLibraryScanToolStripMenuItem.Size = new System.Drawing.Size(277, 22); - this.reimportMostRecentLibraryScanToolStripMenuItem.Text = "Re-&import most recent library scan: {0}"; - this.reimportMostRecentLibraryScanToolStripMenuItem.Click += new System.EventHandler(this.reimportMostRecentLibraryScanToolStripMenuItem_Click); - // - // beginImportingBookDetailsToolStripMenuItem - // - this.beginImportingBookDetailsToolStripMenuItem.Name = "beginImportingBookDetailsToolStripMenuItem"; - this.beginImportingBookDetailsToolStripMenuItem.Size = new System.Drawing.Size(277, 22); - this.beginImportingBookDetailsToolStripMenuItem.Text = "Begin importing book details: {0}"; - this.beginImportingBookDetailsToolStripMenuItem.Click += new System.EventHandler(this.beginImportingBookDetailsToolStripMenuItem_Click); - // // liberateToolStripMenuItem // this.liberateToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { @@ -277,7 +258,7 @@ #endregion private System.Windows.Forms.Panel gridPanel; private System.Windows.Forms.MenuStrip menuStrip1; - private System.Windows.Forms.ToolStripMenuItem indexToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem importToolStripMenuItem; private System.Windows.Forms.StatusStrip statusStrip1; private System.Windows.Forms.ToolStripStatusLabel springLbl; private System.Windows.Forms.ToolStripStatusLabel visibleCountLbl; @@ -291,8 +272,6 @@ private System.Windows.Forms.Button filterHelpBtn; private System.Windows.Forms.ToolStripMenuItem settingsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem scanLibraryToolStripMenuItem; - private System.Windows.Forms.ToolStripMenuItem reimportMostRecentLibraryScanToolStripMenuItem; - private System.Windows.Forms.ToolStripMenuItem beginImportingBookDetailsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem quickFiltersToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem firstFilterIsDefaultToolStripMenuItem; private System.Windows.Forms.Button addFilterBtn; diff --git a/LibationWinForm/UNTESTED/Form1.cs b/LibationWinForm/UNTESTED/Form1.cs index 6feb9ceb..a8e649ad 100644 --- a/LibationWinForm/UNTESTED/Form1.cs +++ b/LibationWinForm/UNTESTED/Form1.cs @@ -8,7 +8,6 @@ using Dinah.Core; using Dinah.Core.Collections.Generic; using Dinah.Core.Windows.Forms; using FileManager; -using ScrapingDomainServices; namespace LibationWinForm { @@ -21,9 +20,6 @@ namespace LibationWinForm private string pdfsCountsLbl_Format { get; } private string visibleCountLbl_Format { get; } - private string reimportMostRecentLibraryScanToolStripMenuItem_format { get; } - private string beginImportingBookDetailsToolStripMenuItem_format { get; } - private string beginBookBackupsToolStripMenuItem_format { get; } private string beginPdfBackupsToolStripMenuItem_format { get; } @@ -36,9 +32,6 @@ namespace LibationWinForm pdfsCountsLbl_Format = pdfsCountsLbl.Text; visibleCountLbl_Format = visibleCountLbl.Text; - reimportMostRecentLibraryScanToolStripMenuItem_format = reimportMostRecentLibraryScanToolStripMenuItem.Text; - beginImportingBookDetailsToolStripMenuItem_format = beginImportingBookDetailsToolStripMenuItem.Text; - beginBookBackupsToolStripMenuItem_format = beginBookBackupsToolStripMenuItem.Text; beginPdfBackupsToolStripMenuItem_format = beginPdfBackupsToolStripMenuItem.Text; } @@ -258,150 +251,22 @@ namespace LibationWinForm doFilter(); } } - #endregion - - #region index menu - // - // IMPORTANT - // - // IRunnableDialog.Run() extension method contains work flow - // - #region // example code: chaining multiple dialogs - public class MyDialog1 : IRunnableDialog - { - public IEnumerable Files; - - public IButtonControl AcceptButton { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - public Control.ControlCollection Controls => throw new NotImplementedException(); - public string SuccessMessage => throw new NotImplementedException(); - public DialogResult DialogResult { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - public void Close() => throw new NotImplementedException(); - public Task DoMainWorkAsync() => throw new NotImplementedException(); - public DialogResult ShowDialog() => throw new NotImplementedException(); - public string StringBasedValidate() => throw new NotImplementedException(); - } - public class MyDialog2 : Form, IIndexLibraryDialog - { - public MyDialog2(IEnumerable files) { } - Button BeginFileImportBtn = new Button(); - - public void Begin() => BeginFileImportBtn.PerformClick(); - - public int TotalBooksProcessed => throw new NotImplementedException(); - public int NewBooksAdded => throw new NotImplementedException(); - public string SuccessMessage => throw new NotImplementedException(); - public Task DoMainWorkAsync() => throw new NotImplementedException(); - public string StringBasedValidate() => throw new NotImplementedException(); - } - private async void downloadPagesToFile(object sender, EventArgs e) - { - var dialog1 = new MyDialog1(); - if (dialog1.RunDialog() != DialogResult.OK || !dialog1.Files.Any()) - return; - - if (MessageBox.Show("Index from these files?", "Index?", MessageBoxButtons.YesNo) == DialogResult.Yes) - { - var dialog2 = new MyDialog2(dialog1.Files); - dialog2.Shown += (_, __) => dialog2.Begin(); - await indexDialog(dialog2); - } - } - #endregion - - private void indexToolStripMenuItem_DropDownOpening(object sender, EventArgs e) - { - #region label: Re-import most recent library scan - { - var libDir = WebpageStorage.GetMostRecentLibraryDir(); - if (libDir == null) - { - reimportMostRecentLibraryScanToolStripMenuItem.Enabled = false; - reimportMostRecentLibraryScanToolStripMenuItem.Text = string.Format(reimportMostRecentLibraryScanToolStripMenuItem_format, "No previous scans"); - } - else - { - reimportMostRecentLibraryScanToolStripMenuItem.Enabled = true; - - var now = DateTime.Now; - var span = now - libDir.CreationTime; - var ago - // less than 1 min - = (int)span.TotalSeconds < 60 ? $"{(int)span.TotalSeconds} sec ago" - // less than 1 hr - : (int)span.TotalMinutes < 60 ? $"{(int)span.TotalMinutes} min ago" - // today. eg: 4:25 PM - : now.Date == libDir.CreationTime.Date ? libDir.CreationTime.ToString("h:mm tt") - // else date and time - : libDir.CreationTime.ToString("MM/dd/yyyy h:mm tt"); - reimportMostRecentLibraryScanToolStripMenuItem.Text = string.Format(reimportMostRecentLibraryScanToolStripMenuItem_format, ago); - } - } - #endregion - - #region label: Begin importing book details - { - var noDetails = BookQueries.BooksWithoutDetailsCount(); - if (noDetails == 0) - { - beginImportingBookDetailsToolStripMenuItem.Enabled = false; - beginImportingBookDetailsToolStripMenuItem.Text = string.Format(beginImportingBookDetailsToolStripMenuItem_format, "No books without details"); - } - else - { - beginImportingBookDetailsToolStripMenuItem.Enabled = true; - beginImportingBookDetailsToolStripMenuItem.Text = string.Format(beginImportingBookDetailsToolStripMenuItem_format, $"{noDetails} remaining"); - } - } - #endregion - } + #endregion + #region index menu private async void scanLibraryToolStripMenuItem_Click(object sender, EventArgs e) { -// legacy/scraping method -//await indexDialog(new ScanLibraryDialog()); -// new/api method -await indexDialog(new IndexLibraryDialog()); - } + var dialog = new IndexLibraryDialog(); - private async void reimportMostRecentLibraryScanToolStripMenuItem_Click(object sender, EventArgs e) - { - // DO NOT ConfigureAwait(false) - // this would result in index() => reloadGrid() => setGrid() => "gridPanel.Controls.Remove(currProductsGrid);" - // throwing 'Cross-thread operation not valid: Control 'ProductsGrid' accessed from a thread other than the thread it was created on.' - var (TotalBooksProcessed, NewBooksAdded) = await Indexer.IndexLibraryAsync(WebpageStorage.GetMostRecentLibraryDir()); + if (dialog.RunDialog().In(DialogResult.Abort, DialogResult.Cancel, DialogResult.None)) + return; - MessageBox.Show($"Total processed: {TotalBooksProcessed}\r\nNew: {NewBooksAdded}"); - - await indexComplete(TotalBooksProcessed, NewBooksAdded); - } - - private async Task indexDialog(IIndexLibraryDialog dialog) - { - if (!dialog.RunDialog().In(DialogResult.Abort, DialogResult.Cancel, DialogResult.None)) - await indexComplete(dialog.TotalBooksProcessed, dialog.NewBooksAdded); - } - private async Task indexComplete(int totalBooksProcessed, int newBooksAdded) - { // update backup counts if we have new library items - if (newBooksAdded > 0) + if (dialog.NewBooksAdded > 0) await setBackupCountsAsync(); - // skip reload if: - // - no grid is loaded - // - none indexed - if (currProductsGrid == null || totalBooksProcessed == 0) - return; - - reloadGrid(); - } - - private void updateGridRow(object _, string productId) => currProductsGrid?.UpdateRow(productId); - - private async void beginImportingBookDetailsToolStripMenuItem_Click(object sender, EventArgs e) - { - var scrapeBookDetails = BookLiberation.ProcessorAutomationController.GetWiredUpScrapeBookDetails(); - scrapeBookDetails.BookSuccessfullyImported += updateGridRow; - await BookLiberation.ProcessorAutomationController.RunAutomaticDownload(scrapeBookDetails); + if (dialog.TotalBooksProcessed > 0) + reloadGrid(); } #endregion diff --git a/LibationWinForm/UNTESTED/ProductsGrid.cs b/LibationWinForm/UNTESTED/ProductsGrid.cs index 634dea2d..243abf74 100644 --- a/LibationWinForm/UNTESTED/ProductsGrid.cs +++ b/LibationWinForm/UNTESTED/ProductsGrid.cs @@ -208,7 +208,7 @@ namespace LibationWinForm { book.UserDefinedItem.Tags = newTags; - var qtyChanges = ScrapingDomainServices.Indexer.IndexChangedTags(book); + var qtyChanges = ApplicationServices.TagUpdater.IndexChangedTags(book); return qtyChanges; } diff --git a/REFERENCE.txt b/REFERENCE.txt index 970816d9..2e01e406 100644 --- a/REFERENCE.txt +++ b/REFERENCE.txt @@ -1,5 +1,7 @@ --- begin LEGACY CODE --------------------------------------------------------------------------------------------------------------------- --- end LEGACY CODE --------------------------------------------------------------------------------------------------------------------- +-- begin VERSIONING --------------------------------------------------------------------------------------------------------------------- +https://github.com/rmcrackan/Libation/releases +v3.0 : This version is fully powered by the Audible API. Legacy scraping code is still present but is commented out. All future check-ins are not guaranteed to have any scraping code +-- end VERSIONING --------------------------------------------------------------------------------------------------------------------- -- begin AUDIBLE DETAILS --------------------------------------------------------------------------------------------------------------------- alternate book id (eg BK_RAND_006061) is called 'sku' , 'sku_lite' , 'prod_id' , 'product_id' in different parts of the site diff --git a/Scraping/Scraping.csproj b/Scraping/Scraping.csproj deleted file mode 100644 index 49e55777..00000000 --- a/Scraping/Scraping.csproj +++ /dev/null @@ -1,11 +0,0 @@ - - - - netstandard2.1 - - - - - - - diff --git a/Scraping/UNTESTED/AudibleScraper.cs b/Scraping/UNTESTED/AudibleScraper.cs deleted file mode 100644 index 3ea53311..00000000 --- a/Scraping/UNTESTED/AudibleScraper.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using AudibleDotCom; -using Dinah.Core; -using DTOs; -using Scraping.BookDetail; -using Scraping.Library; - -namespace Scraping -{ - public static class AudibleScraper - { - public static List ScrapeLibrarySources(params AudiblePageSource[] pageSources) - { - if (pageSources == null || !pageSources.Any()) - return new List(); - - if (pageSources.Select(ps => ps.AudiblePage).Distinct().Single() != AudiblePageType.Library) - throw new Exception("only Library items allowed"); - - return pageSources.SelectMany(s => scrapeLibraryPageSource(s)).ToList(); - } - private static List scrapeLibraryPageSource(AudiblePageSource pageSource) - => new LibraryScraper(pageSource) - .ScrapeCurrentPage() - // ScrapeCurrentPage() is long running. do not taunt delayed execution - .ToList(); - - public static BookDetailDTO ScrapeBookDetailsSource(AudiblePageSource pageSource) - { - ArgumentValidator.EnsureNotNull(pageSource, nameof(pageSource)); - - if (pageSource.AudiblePage != AudiblePageType.ProductDetails) - throw new Exception("only Product Details items allowed"); - - try - { - return new BookDetailScraper(pageSource).ScrapePage(); - } - catch (Exception ex) - { - throw; - } - } - } -} diff --git a/Scraping/UNTESTED/BookDetail/BookDetailScraper.cs b/Scraping/UNTESTED/BookDetail/BookDetailScraper.cs deleted file mode 100644 index ae4d5a93..00000000 --- a/Scraping/UNTESTED/BookDetail/BookDetailScraper.cs +++ /dev/null @@ -1,175 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using AudibleDotCom; -using Dinah.Core; -using DTOs; -using Newtonsoft.Json.Linq; -using Scraping.Selectors; - -namespace Scraping.BookDetail -{ - static class NewtonsoftExt - { - public static string GetDecodedTokenString(this JToken jToken) => System.Net.WebUtility.HtmlDecode(((string)jToken).Trim()); - } - internal class BookDetailScraper - { - private AudiblePageSource source { get; } - private WebElement docRoot { get; } - - public BookDetailScraper(AudiblePageSource pageSource) - { - source = pageSource; - - var doc = new HtmlAgilityPack.HtmlDocument(); - doc.LoadHtml(source.Source); - docRoot = new WebElement(doc.DocumentNode); - } - - static RuleFamilyBD ruleFamily { get; } = new RuleFamilyBD - { - RowsLocator = By.XPath("/*"), - Rules = new RuleSetBD - { - parseJson, - parseSeries - } - }; - - public BookDetailDTO ScrapePage() - { - //debug//var sw = System.Diagnostics.Stopwatch.StartNew(); - - var returnBookDetailDto = new BookDetailDTO { ProductId = source.PageId }; - - var wholePage = ruleFamily.GetRows(docRoot).Single(); - ruleFamily.Rules.Run(wholePage, returnBookDetailDto); - - //debug//sw.Stop(); var ms = sw.ElapsedMilliseconds; - - return returnBookDetailDto; - } - - static void parseJson(WebElement row, BookDetailDTO productItem) - { - // structured data is in the 2nd of the 3 json embedded sections