From 1b6c577044791c924b93ede4fa9947de5bc9ba15 Mon Sep 17 00:00:00 2001 From: Robert McRackan Date: Tue, 8 Oct 2019 08:54:12 -0400 Subject: [PATCH] Convert to new Core 3.0 using declarations --- AaxDecrypter/UNTESTED/AaxToM4bConverter.cs | 6 +- AaxDecrypter/UNTESTED/Tags.cs | 46 +++--- .../Page Retrievers/BrowserlessRetriever.cs | 40 ++--- CookieMonster/UNTESTED/Browsers/Chrome.cs | 79 +++++----- CookieMonster/UNTESTED/Browsers/FireFox.cs | 34 ++--- .../UNTESTED/QueryObjects/BookQueries.cs | 22 +-- .../UNTESTED/QueryObjects/LibraryQueries.cs | 26 ++-- DataLayer/UNTESTED/_scratch pad/ScratchPad.cs | 12 +- DomainServices/UNTESTED/DownloadBook.cs | 10 +- DomainServices/UNTESTED/DownloadPdf.cs | 4 +- DomainServices/UNTESTED/Indexer.cs | 68 ++++----- DomainServices/UNTESTED/ScrapeBookDetails.cs | 54 ++++--- FileManager/UNTESTED/PictureStorage.cs | 58 ++++---- LibationSearchEngine/UNTESTED/SearchEngine.cs | 139 ++++++++---------- .../ProcessorAutomationController.Examples.cs | 12 +- .../Dialogs/IndexDialogs/ScanLibraryDialog.cs | 5 +- REFERENCE.txt | 14 +- _DB_NOTES.txt | 2 +- _Demos/ffmpeg decrypt/Form1.cs | 70 ++++----- __TODO.txt | 4 +- 20 files changed, 324 insertions(+), 381 deletions(-) diff --git a/AaxDecrypter/UNTESTED/AaxToM4bConverter.cs b/AaxDecrypter/UNTESTED/AaxToM4bConverter.cs index 39458e9d..5ca7ee8b 100644 --- a/AaxDecrypter/UNTESTED/AaxToM4bConverter.cs +++ b/AaxDecrypter/UNTESTED/AaxToM4bConverter.cs @@ -117,9 +117,9 @@ namespace AaxDecrypter private void saveCover(string aaxFile) { - using (var file = TagLib.File.Create(aaxFile, "audio/mp4", TagLib.ReadStyle.Average)) - this.coverBytes = file.Tag.Pictures[0].Data.Data; - } + using var file = TagLib.File.Create(aaxFile, "audio/mp4", TagLib.ReadStyle.Average); + this.coverBytes = file.Tag.Pictures[0].Data.Data; + } private void printPrelim() { diff --git a/AaxDecrypter/UNTESTED/Tags.cs b/AaxDecrypter/UNTESTED/Tags.cs index be2f4023..9eb10bbf 100644 --- a/AaxDecrypter/UNTESTED/Tags.cs +++ b/AaxDecrypter/UNTESTED/Tags.cs @@ -25,35 +25,31 @@ namespace AaxDecrypter public Tags(string file) { - using (TagLib.File tagLibFile = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average)) - { - this.title = tagLibFile.Tag.Title.Replace(" (Unabridged)", ""); - this.album = tagLibFile.Tag.Album.Replace(" (Unabridged)", ""); - this.author = tagLibFile.Tag.FirstPerformer; - this.year = tagLibFile.Tag.Year.ToString(); - this.comments = tagLibFile.Tag.Comment; - this.duration = tagLibFile.Properties.Duration; - this.genre = tagLibFile.Tag.FirstGenre; + using TagLib.File tagLibFile = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average); + this.title = tagLibFile.Tag.Title.Replace(" (Unabridged)", ""); + this.album = tagLibFile.Tag.Album.Replace(" (Unabridged)", ""); + this.author = tagLibFile.Tag.FirstPerformer; + this.year = tagLibFile.Tag.Year.ToString(); + this.comments = tagLibFile.Tag.Comment; + this.duration = tagLibFile.Properties.Duration; + this.genre = tagLibFile.Tag.FirstGenre; - var tag = tagLibFile.GetTag(TagTypes.Apple, true); - this.publisher = tag.Publisher; - this.narrator = string.IsNullOrWhiteSpace(tagLibFile.Tag.FirstComposer) ? tag.Narrator : tagLibFile.Tag.FirstComposer; - this.comments = !string.IsNullOrWhiteSpace(tag.LongDescription) ? tag.LongDescription : tag.Description; - this.id = tag.AudibleCDEK; - } - } + var tag = tagLibFile.GetTag(TagTypes.Apple, true); + this.publisher = tag.Publisher; + this.narrator = string.IsNullOrWhiteSpace(tagLibFile.Tag.FirstComposer) ? tag.Narrator : tagLibFile.Tag.FirstComposer; + this.comments = !string.IsNullOrWhiteSpace(tag.LongDescription) ? tag.LongDescription : tag.Description; + this.id = tag.AudibleCDEK; + } public void AddAppleTags(string file) { - using (var file1 = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average)) - { - var tag = (AppleTag)file1.GetTag(TagTypes.Apple, true); - tag.Publisher = this.publisher; - tag.LongDescription = this.comments; - tag.Description = this.comments; - file1.Save(); - } - } + using var file1 = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average); + var tag = (AppleTag)file1.GetTag(TagTypes.Apple, true); + tag.Publisher = this.publisher; + tag.LongDescription = this.comments; + tag.Description = this.comments; + file1.Save(); + } public string GenerateFfmpegTags() { diff --git a/AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs b/AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs index 9224dd66..abc2eeca 100644 --- a/AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs +++ b/AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs @@ -94,27 +94,27 @@ var pageSource = new AudiblePageSource(AudiblePageType.Library, html, null); private static async Task getLibraryPageAsync(CookieContainer cookies, int pageNum) { - #region // POST example (from 2017 ajax) - // var destination = "https://www.audible.com/lib-ajax"; - // var webRequest = (HttpWebRequest)WebRequest.Create(destination); - // webRequest.Method = "POST"; - // webRequest.Accept = "*/*"; - // webRequest.AllowAutoRedirect = false; - // webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)"; - // webRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8"; - // webRequest.Credentials = null; - // - // webRequest.CookieContainer = new CookieContainer(); - // webRequest.CookieContainer.Add(cookies.GetCookies(new Uri(destination))); - // - // var postData = $"progType=all&timeFilter=all&itemsPerPage={itemsPerPage}&searchTerm=&searchType=&sortColumn=&sortType=down&page={pageNum}&mode=normal&subId=&subTitle="; - // var data = Encoding.UTF8.GetBytes(postData); - // webRequest.ContentLength = data.Length; - // using (var dataStream = webRequest.GetRequestStream()) - // dataStream.Write(data, 0, data.Length); - #endregion + #region // POST example (from 2017 ajax) + // var destination = "https://www.audible.com/lib-ajax"; + // var webRequest = (HttpWebRequest)WebRequest.Create(destination); + // webRequest.Method = "POST"; + // webRequest.Accept = "*/*"; + // webRequest.AllowAutoRedirect = false; + // webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)"; + // webRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8"; + // webRequest.Credentials = null; + // + // webRequest.CookieContainer = new CookieContainer(); + // webRequest.CookieContainer.Add(cookies.GetCookies(new Uri(destination))); + // + // var postData = $"progType=all&timeFilter=all&itemsPerPage={itemsPerPage}&searchTerm=&searchType=&sortColumn=&sortType=down&page={pageNum}&mode=normal&subId=&subTitle="; + // var data = Encoding.UTF8.GetBytes(postData); + // webRequest.ContentLength = data.Length; + // using var dataStream = webRequest.GetRequestStream(); + // dataStream.Write(data, 0, data.Length); + #endregion -var destination = "https://" + $"www.audible.com/lib?purchaseDateFilter=all&programFilter=all&sortBy=PURCHASE_DATE.dsc&page={pageNum}"; + var destination = "https://" + $"www.audible.com/lib?purchaseDateFilter=all&programFilter=all&sortBy=PURCHASE_DATE.dsc&page={pageNum}"; var webRequest = (HttpWebRequest)WebRequest.Create(destination); webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)"; diff --git a/CookieMonster/UNTESTED/Browsers/Chrome.cs b/CookieMonster/UNTESTED/Browsers/Chrome.cs index fb0034c3..8a77f2c7 100644 --- a/CookieMonster/UNTESTED/Browsers/Chrome.cs +++ b/CookieMonster/UNTESTED/Browsers/Chrome.cs @@ -10,53 +10,50 @@ namespace CookieMonster { internal class Chrome : IBrowser { - public async Task> GetAllCookiesAsync() - { - var col = new List(); + public async Task> GetAllCookiesAsync() + { + var col = new List(); - var strPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"Google\Chrome\User Data\Default\Cookies"); - if (!FileUtility.FileExists(strPath)) - return col; + var strPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"Google\Chrome\User Data\Default\Cookies"); + if (!FileUtility.FileExists(strPath)) + return col; - // - // IF WE GET AN ERROR HERE - // then add a reference to sqlite core in the project which is ultimately calling this. - // a project which directly references CookieMonster doesn't need to also ref sqlite. - // however, for any further number of abstractions, the project needs to directly ref sqlite. - // eg: this will not work unless the winforms proj adds sqlite to ref.s: - // LibationWinForm > AudibleDotComAutomation > CookieMonster - // - using (var conn = new SQLiteConnection("Data Source=" + strPath + ";pooling=false")) - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = "SELECT host_key, name, value, encrypted_value, last_access_utc, expires_utc FROM cookies;"; + // + // IF WE GET AN ERROR HERE + // then add a reference to sqlite core in the project which is ultimately calling this. + // a project which directly references CookieMonster doesn't need to also ref sqlite. + // however, for any further number of abstractions, the project needs to directly ref sqlite. + // eg: this will not work unless the winforms proj adds sqlite to ref.s: + // LibationWinForm > AudibleDotComAutomation > CookieMonster + // + using var conn = new SQLiteConnection("Data Source=" + strPath + ";pooling=false"); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT host_key, name, value, encrypted_value, last_access_utc, expires_utc FROM cookies;"; - conn.Open(); - using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - { - while (reader.Read()) - { - var host_key = reader.GetString(0); - var name = reader.GetString(1); - var value = reader.GetString(2); - var last_access_utc = reader.GetInt64(4); - var expires_utc = reader.GetInt64(5); + conn.Open(); + using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (reader.Read()) + { + var host_key = reader.GetString(0); + var name = reader.GetString(1); + var value = reader.GetString(2); + var last_access_utc = reader.GetInt64(4); + var expires_utc = reader.GetInt64(5); - // https://stackoverflow.com/a/25874366 - if (string.IsNullOrWhiteSpace(value)) - { - var encrypted_value = (byte[])reader[3]; - var decodedData = System.Security.Cryptography.ProtectedData.Unprotect(encrypted_value, null, System.Security.Cryptography.DataProtectionScope.CurrentUser); - value = Encoding.ASCII.GetString(decodedData); - } + // https://stackoverflow.com/a/25874366 + if (string.IsNullOrWhiteSpace(value)) + { + var encrypted_value = (byte[])reader[3]; + var decodedData = System.Security.Cryptography.ProtectedData.Unprotect(encrypted_value, null, System.Security.Cryptography.DataProtectionScope.CurrentUser); + value = Encoding.ASCII.GetString(decodedData); + } - col.Add(new CookieValue { Browser = "chrome", Domain = host_key, Name = name, Value = value, LastAccess = chromeTimeToDateTimeUtc(last_access_utc), Expires = chromeTimeToDateTimeUtc(expires_utc) }); - } - } - } + col.Add(new CookieValue { Browser = "chrome", Domain = host_key, Name = name, Value = value, LastAccess = chromeTimeToDateTimeUtc(last_access_utc), Expires = chromeTimeToDateTimeUtc(expires_utc) }); + } - return col; - } + + return col; + } // Chrome uses 1601-01-01 00:00:00 UTC as the epoch (ie the starting point for the millisecond time counter). // this is the same as "FILETIME" in Win32 except FILETIME uses 100ns ticks instead of ms. diff --git a/CookieMonster/UNTESTED/Browsers/FireFox.cs b/CookieMonster/UNTESTED/Browsers/FireFox.cs index 086f9426..6be33c6e 100644 --- a/CookieMonster/UNTESTED/Browsers/FireFox.cs +++ b/CookieMonster/UNTESTED/Browsers/FireFox.cs @@ -28,27 +28,23 @@ namespace CookieMonster File.Copy(strPath, strTemp, true); - // Now open the temporary cookie jar and extract Value from the cookie if we find it. - using (var conn = new SQLiteConnection("Data Source=" + strTemp + ";pooling=false")) - using (var cmd = conn.CreateCommand()) - { - cmd.CommandText = "SELECT host, name, value, lastAccessed, expiry FROM moz_cookies; "; + // Now open the temporary cookie jar and extract Value from the cookie if we find it. + using var conn = new SQLiteConnection("Data Source=" + strTemp + ";pooling=false"); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT host, name, value, lastAccessed, expiry FROM moz_cookies; "; - conn.Open(); - using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) - { - while (reader.Read()) - { - var host_key = reader.GetString(0); - var name = reader.GetString(1); - var value = reader.GetString(2); - var lastAccessed = reader.GetInt32(3); - var expiry = reader.GetInt32(4); + conn.Open(); + using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + while (reader.Read()) + { + var host_key = reader.GetString(0); + var name = reader.GetString(1); + var value = reader.GetString(2); + var lastAccessed = reader.GetInt32(3); + var expiry = reader.GetInt32(4); - col.Add(new CookieValue { Browser = "firefox", Domain = host_key, Name = name, Value = value, LastAccess = lastAccessedToDateTime(lastAccessed), Expires = expiryToDateTime(expiry) }); - } - } - } + col.Add(new CookieValue { Browser = "firefox", Domain = host_key, Name = name, Value = value, LastAccess = lastAccessedToDateTime(lastAccessed), Expires = expiryToDateTime(expiry) }); + } if (FileUtility.FileExists(strTemp)) File.Delete(strTemp); diff --git a/DataLayer/UNTESTED/QueryObjects/BookQueries.cs b/DataLayer/UNTESTED/QueryObjects/BookQueries.cs index 9a4919b5..e986b267 100644 --- a/DataLayer/UNTESTED/QueryObjects/BookQueries.cs +++ b/DataLayer/UNTESTED/QueryObjects/BookQueries.cs @@ -10,20 +10,20 @@ namespace DataLayer { public static int BooksWithoutDetailsCount() { - using (var context = LibationContext.Create()) - return context - .Books - .Count(b => !b.HasBookDetails); - } + using var context = LibationContext.Create(); + return context +.Books +.Count(b => !b.HasBookDetails); + } public static Book GetBook_Flat_NoTracking(string productId) { - using (var context = LibationContext.Create()) - return context - .Books - .AsNoTracking() - .GetBook(productId); - } + using var context = LibationContext.Create(); + return context +.Books +.AsNoTracking() +.GetBook(productId); + } public static Book GetBook(this IQueryable books, string productId) => books diff --git a/DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs b/DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs index a0e6cde6..f3a6aa22 100644 --- a/DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs +++ b/DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs @@ -8,22 +8,22 @@ namespace DataLayer { public static List GetLibrary_Flat_NoTracking() { - using (var context = LibationContext.Create()) - return context - .Library - .AsNoTracking() - .GetLibrary() - .ToList(); - } + using var context = LibationContext.Create(); + return context +.Library +.AsNoTracking() +.GetLibrary() +.ToList(); + } public static LibraryBook GetLibraryBook_Flat_NoTracking(string productId) { - using (var context = LibationContext.Create()) - return context - .Library - .AsNoTracking() - .GetLibraryBook(productId); - } + using var context = LibationContext.Create(); + return context +.Library +.AsNoTracking() +.GetLibraryBook(productId); + } /// This is still IQueryable. YOU MUST CALL ToList() YOURSELF public static IQueryable GetLibrary(this IQueryable library) diff --git a/DataLayer/UNTESTED/_scratch pad/ScratchPad.cs b/DataLayer/UNTESTED/_scratch pad/ScratchPad.cs index eee3f895..05fae390 100644 --- a/DataLayer/UNTESTED/_scratch pad/ScratchPad.cs +++ b/DataLayer/UNTESTED/_scratch pad/ScratchPad.cs @@ -13,13 +13,11 @@ namespace _scratch_pad // var user = new Student() { Name = "Dinah Cheshire" }; // var udi = new UserDef { UserDefId = 1, TagsRaw = "my,tags" }; - // using (var context = new MyTestContextDesignTimeDbContextFactory().Create()) - // { - // context.Add(user); - // //context.Add(udi); - // context.Update(udi); - // context.SaveChanges(); - // } + // using var context = new MyTestContextDesignTimeDbContextFactory().Create(); + // context.Add(user); + // //context.Add(udi); + // context.Update(udi); + // context.SaveChanges(); // Console.WriteLine($"Student was saved in the database with id: {user.Id}"); // } diff --git a/DomainServices/UNTESTED/DownloadBook.cs b/DomainServices/UNTESTED/DownloadBook.cs index 2b4bc275..09c43b50 100644 --- a/DomainServices/UNTESTED/DownloadBook.cs +++ b/DomainServices/UNTESTED/DownloadBook.cs @@ -43,13 +43,11 @@ namespace DomainServices .Replace("&DownloadType=Now", "") + "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download"; var uri = new Uri(aaxDownloadLink); - using (var webClient = await GetWebClient(tempAaxFilename)) - { - // for book downloads only: pretend to be the audible download manager. from inAudible: - webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601"; - await webClient.DownloadFileTaskAsync(uri, tempAaxFilename); - } + using var webClient = await GetWebClient(tempAaxFilename); + // for book downloads only: pretend to be the audible download manager. from inAudible: + webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601"; + await webClient.DownloadFileTaskAsync(uri, tempAaxFilename); // move var aaxFilename = FileUtility.GetValidFilename( diff --git a/DomainServices/UNTESTED/DownloadPdf.cs b/DomainServices/UNTESTED/DownloadPdf.cs index 1ff5b521..550bc67b 100644 --- a/DomainServices/UNTESTED/DownloadPdf.cs +++ b/DomainServices/UNTESTED/DownloadPdf.cs @@ -43,8 +43,8 @@ namespace DomainServices var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url)); - using (var webClient = await GetWebClient(destinationFilename)) - await webClient.DownloadFileTaskAsync(url, destinationFilename); + using var webClient = await GetWebClient(destinationFilename); + await webClient.DownloadFileTaskAsync(url, destinationFilename); var statusHandler = new StatusHandler(); var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId); diff --git a/DomainServices/UNTESTED/Indexer.cs b/DomainServices/UNTESTED/Indexer.cs index a4802b26..ee22a177 100644 --- a/DomainServices/UNTESTED/Indexer.cs +++ b/DomainServices/UNTESTED/Indexer.cs @@ -55,28 +55,25 @@ namespace DomainServices productItems = filterAndValidate(productItems); - int newEntries; - using (var context = LibationContext.Create()) - { - var dtoImporter = new DtoImporter(context); + using var context = LibationContext.Create(); + var dtoImporter = new DtoImporter(context); - #region // benchmarks. re-importing a library with 500 books, all with book details json files - /* - dtoImporter.ReplaceLibrary 1.2 seconds - SaveChanges() 3.4 - ReloadBookDetails() 1.3 - SaveChanges() 1.4 - */ - #endregion - // LONG RUNNING - newEntries = await Task.Run(() => dtoImporter.ReplaceLibrary(productItems)); - await context.SaveChangesAsync(); + #region // benchmarks. re-importing a library with 500 books, all with book details json files + /* + dtoImporter.ReplaceLibrary 1.2 seconds + SaveChanges() 3.4 + ReloadBookDetails() 1.3 + SaveChanges() 1.4 + */ + #endregion + // LONG RUNNING + var newEntries = await Task.Run(() => dtoImporter.ReplaceLibrary(productItems)); + await context.SaveChangesAsync(); - // must be broken out. see notes in dtoImporter.ReplaceLibrary() - // LONG RUNNING - await Task.Run(() => dtoImporter.ReloadBookDetails(productItems)); - await context.SaveChangesAsync(); - } + // must be broken out. see notes in dtoImporter.ReplaceLibrary() + // LONG RUNNING + await Task.Run(() => dtoImporter.ReloadBookDetails(productItems)); + await context.SaveChangesAsync(); await postIndexActionAsync?.Invoke(); @@ -111,13 +108,10 @@ namespace DomainServices #region update book tags public static int IndexChangedTags(Book book) { - // update disconnected entity - int qtyChanges; - using (var context = LibationContext.Create()) - { - context.Update(book); - qtyChanges = context.SaveChanges(); - } + // update disconnected entity + using var context = LibationContext.Create(); + context.Update(book); + var qtyChanges = context.SaveChanges(); // this part is tags-specific if (qtyChanges > 0) @@ -151,19 +145,15 @@ namespace DomainServices validate(bookDetailDTO); - using (var context = LibationContext.Create()) - { - var dtoImporter = new DtoImporter(context); - // LONG RUNNING - await Task.Run(() => dtoImporter.UpdateBookDetails(bookDetailDTO)); - context.SaveChanges(); + using var context = LibationContext.Create(); + var dtoImporter = new DtoImporter(context); + // LONG RUNNING + await Task.Run(() => dtoImporter.UpdateBookDetails(bookDetailDTO)); + context.SaveChanges(); - // after saving, delete orphan contributors - var count = context.RemoveOrphans(); - if (count > 0) - { - } - } + // after saving, delete orphan contributors + var count = context.RemoveOrphans(); + if (count > 0) { } // don't think there's a to-do here await postIndexActionAsync?.Invoke(); } diff --git a/DomainServices/UNTESTED/ScrapeBookDetails.cs b/DomainServices/UNTESTED/ScrapeBookDetails.cs index 7b005e6b..51ffb987 100644 --- a/DomainServices/UNTESTED/ScrapeBookDetails.cs +++ b/DomainServices/UNTESTED/ScrapeBookDetails.cs @@ -72,37 +72,35 @@ namespace DomainServices // download htm string source; var url = AudiblePage.Product.GetUrl(productId); - using (var webClient = await GetWebClient($"Getting Book Details for {libraryBook.Book.Title}")) - { - try - { - source = await webClient.DownloadStringTaskAsync(url); - var detailsAudiblePageSource = new AudiblePageSource(AudiblePageType.ProductDetails, source, productId); + using var webClient = await GetWebClient($"Getting Book Details for {libraryBook.Book.Title}"); + try + { + source = await webClient.DownloadStringTaskAsync(url); + var detailsAudiblePageSource = new AudiblePageSource(AudiblePageType.ProductDetails, source, productId); - // good habit to persist htm before attempting to parse it. this way, if there's a parse error, we can test errors on a local copy - DataConverter.AudiblePageSource_2_HtmFile_Product(detailsAudiblePageSource); + // good habit to persist htm before attempting to parse it. this way, if there's a parse error, we can test errors on a local copy + DataConverter.AudiblePageSource_2_HtmFile_Product(detailsAudiblePageSource); - bookDetailDTO = AudibleScraper.ScrapeBookDetailsSource(detailsAudiblePageSource); - } - catch (System.Net.WebException webEx) - { - // cannot continue if NoLongerAvailableAction is null, - // else we'll be right back here next loop (and infinitely) with no failure condition - if (webEx.Status != System.Net.WebExceptionStatus.ConnectionClosed || NoLongerAvailableAction == null) - throw; + bookDetailDTO = AudibleScraper.ScrapeBookDetailsSource(detailsAudiblePageSource); + } + catch (System.Net.WebException webEx) + { + // cannot continue if NoLongerAvailableAction is null, + // else we'll be right back here next loop (and infinitely) with no failure condition + if (webEx.Status != System.Net.WebExceptionStatus.ConnectionClosed || NoLongerAvailableAction == null) + throw; - var nlaEnum = NoLongerAvailableAction.Invoke( - libraryBook.Book.Title, - AudiblePage.Product.GetUrl(libraryBook.Book.AudibleProductId)); - if (nlaEnum == NoLongerAvailableEnum.Abort) - return new StatusHandler { "Cannot scrape book details. Aborting." }; - else if (nlaEnum == NoLongerAvailableEnum.MarkAsMissing) - bookDetailDTO = new BookDetailDTO { ProductId = productId }; - else - throw; - } - } - } + var nlaEnum = NoLongerAvailableAction.Invoke( + libraryBook.Book.Title, + AudiblePage.Product.GetUrl(libraryBook.Book.AudibleProductId)); + if (nlaEnum == NoLongerAvailableEnum.Abort) + return new StatusHandler { "Cannot scrape book details. Aborting." }; + else if (nlaEnum == NoLongerAvailableEnum.MarkAsMissing) + bookDetailDTO = new BookDetailDTO { ProductId = productId }; + else + throw; + } + } DataConverter.Value_2_JsonFile(bookDetailDTO, jsonFileInfo.FullName); } diff --git a/FileManager/UNTESTED/PictureStorage.cs b/FileManager/UNTESTED/PictureStorage.cs index d4572f69..efdd6bab 100644 --- a/FileManager/UNTESTED/PictureStorage.cs +++ b/FileManager/UNTESTED/PictureStorage.cs @@ -39,39 +39,37 @@ namespace FileManager { try { - using (var webClient = new System.Net.WebClient()) - { - // download any that don't exist - { - if (!FileUtility.FileExists(path80)) - { - var bytes = webClient.DownloadData( - "https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL80_.jpg"); - File.WriteAllBytes(path80, bytes); - } - } + using var webClient = new System.Net.WebClient(); + // download any that don't exist + { + if (!FileUtility.FileExists(path80)) + { + var bytes = webClient.DownloadData( + "https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL80_.jpg"); + File.WriteAllBytes(path80, bytes); + } + } - { - if (!FileUtility.FileExists(path300)) - { - var bytes = webClient.DownloadData( - "https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL300_.jpg"); - File.WriteAllBytes(path300, bytes); - } - } + { + if (!FileUtility.FileExists(path300)) + { + var bytes = webClient.DownloadData( + "https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL300_.jpg"); + File.WriteAllBytes(path300, bytes); + } + } - { - if (!FileUtility.FileExists(path500)) - { - var bytes = webClient.DownloadData( - "https://m.media-amazon.com/images/I/" + pictureId + "._SL500_.jpg"); - File.WriteAllBytes(path500, bytes); - } - } + { + if (!FileUtility.FileExists(path500)) + { + var bytes = webClient.DownloadData( + "https://m.media-amazon.com/images/I/" + pictureId + "._SL500_.jpg"); + File.WriteAllBytes(path500, bytes); + } + } - break; - } - } + break; + } catch { retry++; } } while (retry < 3); diff --git a/LibationSearchEngine/UNTESTED/SearchEngine.cs b/LibationSearchEngine/UNTESTED/SearchEngine.cs index de81cf7c..3be8c0ff 100644 --- a/LibationSearchEngine/UNTESTED/SearchEngine.cs +++ b/LibationSearchEngine/UNTESTED/SearchEngine.cs @@ -177,26 +177,19 @@ namespace LibationSearchEngine log(); - // location of index/create the index - using (var index = getIndex()) - { - var exists = IndexReader.IndexExists(index); - var createNewIndex = overwrite || !exists; + // location of index/create the index + using var index = getIndex(); + var exists = IndexReader.IndexExists(index); + var createNewIndex = overwrite || !exists; - // analyzer for tokenizing text. same analyzer should be used for indexing and searching - using (var analyzer = new StandardAnalyzer(Version)) - using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED)) - { - foreach (var libraryBook in library) - { - var doc = createBookIndexDocument(libraryBook); - ixWriter.AddDocument(doc); - } - - // don't optimize. deprecated: ixWriter.Optimize(); - // ixWriter.Commit(); not needed if we're about to dispose of writer anyway. could be needed within the using() block - } - } + // analyzer for tokenizing text. same analyzer should be used for indexing and searching + using var analyzer = new StandardAnalyzer(Version); + using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED); + foreach (var libraryBook in library) + { + var doc = createBookIndexDocument(libraryBook); + ixWriter.AddDocument(doc); + } log(); } @@ -247,47 +240,41 @@ namespace LibationSearchEngine var document = createBookIndexDocument(libraryBook); var createNewIndex = false; - using (var index = getIndex()) - using (var analyzer = new StandardAnalyzer(Version)) - using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED)) - { - ixWriter.DeleteDocuments(term); - ixWriter.AddDocument(document); - } - } + using var index = getIndex(); + using var analyzer = new StandardAnalyzer(Version); + using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED); + ixWriter.DeleteDocuments(term); + ixWriter.AddDocument(document); + } public void UpdateTags(string productId, string tags) { var productTerm = new Term(_ID_, productId); - using (var index = getIndex()) - { - Document document; + using var index = getIndex(); - // get existing document - using (var searcher = new IndexSearcher(index)) - { - var query = new TermQuery(productTerm); - var docs = searcher.Search(query, 1); - var scoreDoc = docs.ScoreDocs.SingleOrDefault(); - if (scoreDoc == null) - throw new Exception("document not found"); - document = searcher.Doc(scoreDoc.Doc); - } + // get existing document + using var searcher = new IndexSearcher(index); + var query = new TermQuery(productTerm); + var docs = searcher.Search(query, 1); + var scoreDoc = docs.ScoreDocs.SingleOrDefault(); + if (scoreDoc == null) + throw new Exception("document not found"); + var document = searcher.Doc(scoreDoc.Doc); - // update document entry with new tags - // fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new - // REMEMBER: all fields, including 'tags' are case-specific - document.RemoveField(TAGS); - document.AddAnalyzed(TAGS, tags); - // update index - var createNewIndex = false; - using (var analyzer = new StandardAnalyzer(Version)) - using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED)) - ixWriter.UpdateDocument(productTerm, document, analyzer); - } - } + // update document entry with new tags + // fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new + // REMEMBER: all fields, including 'tags' are case-specific + document.RemoveField(TAGS); + document.AddAnalyzed(TAGS, tags); + + // update index + var createNewIndex = false; + using var analyzer = new StandardAnalyzer(Version); + using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED); + ixWriter.UpdateDocument(productTerm, document, analyzer); + } public SearchResultSet Search(string searchString) { @@ -381,35 +368,33 @@ namespace LibationSearchEngine var defaultField = ALL; - using (var index = getIndex()) - using (var searcher = new IndexSearcher(index)) - using (var analyzer = new StandardAnalyzer(Version)) - { - var query = analyzer.GetQuery(defaultField, searchString); + using var index = getIndex(); + using var searcher = new IndexSearcher(index); + using var analyzer = new StandardAnalyzer(Version); + var query = analyzer.GetQuery(defaultField, searchString); - // lucene doesn't allow only negations. eg this returns nothing: - // -tags:hidden - // work arounds: https://kb.ucla.edu/articles/pure-negation-query-in-lucene - // HOWEVER, doing this to any other type of query can cause EVERYTHING to be a match unless "Occur" is carefully set - // this should really check that all leaf nodes are MUST_NOT - if (query is BooleanQuery boolQuery) - { - var occurs = getOccurs_recurs(boolQuery); - if (occurs.Any() && occurs.All(o => o == Occur.MUST_NOT)) - boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST); - } + // lucene doesn't allow only negations. eg this returns nothing: + // -tags:hidden + // work arounds: https://kb.ucla.edu/articles/pure-negation-query-in-lucene + // HOWEVER, doing this to any other type of query can cause EVERYTHING to be a match unless "Occur" is carefully set + // this should really check that all leaf nodes are MUST_NOT + if (query is BooleanQuery boolQuery) + { + var occurs = getOccurs_recurs(boolQuery); + if (occurs.Any() && occurs.All(o => o == Occur.MUST_NOT)) + boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST); + } - Console.WriteLine($" query: {query}"); + Console.WriteLine($" query: {query}"); - var docs = searcher - .Search(query, MaxSearchResultsToReturn) - .ScoreDocs - .Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score)) - .ToList(); - return new SearchResultSet(query.ToString(), docs); - } - } + var docs = searcher + .Search(query, MaxSearchResultsToReturn) + .ScoreDocs + .Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score)) + .ToList(); + return new SearchResultSet(query.ToString(), docs); + } private IEnumerable getOccurs_recurs(BooleanQuery query) { diff --git a/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs b/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs index 22fa7d17..cf5388bd 100644 --- a/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs +++ b/LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs @@ -11,12 +11,12 @@ namespace LibationWinForm.BookLiberation { async Task BackupBookAsync(string productId) { - LibraryBook libraryBook; - using (var context = LibationContext.Create()) - libraryBook = context - .Library - .GetLibrary() - .SingleOrDefault(lb => lb.Book.AudibleProductId == productId); + using var context = LibationContext.Create(); + + var libraryBook = context + .Library + .GetLibrary() + .SingleOrDefault(lb => lb.Book.AudibleProductId == productId); if (libraryBook == null) return; diff --git a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs index cc2fb6a0..f93110e1 100644 --- a/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs +++ b/LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs @@ -25,9 +25,8 @@ namespace LibationWinForm public async Task DoMainWorkAsync() { - List jsonFilepaths; - using (var pageRetriever = websiteProcessorControl1.GetPageRetriever()) - jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false); + using var pageRetriever = websiteProcessorControl1.GetPageRetriever(); + var jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false); successMessages.Add($"Downloaded {"library page".PluralizeWithCount(jsonFilepaths.Count)}"); diff --git a/REFERENCE.txt b/REFERENCE.txt index 7c50482c..83f6bedb 100644 --- a/REFERENCE.txt +++ b/REFERENCE.txt @@ -63,18 +63,16 @@ transaction notes ----------------- // https://msdn.microsoft.com/en-us/data/dn456843.aspx // Rollback is called by transaction Dispose(). No need to call it explicitly - using (var dbContext = new LibationContext()) - using (var dbContextTransaction = dbContext.Database.BeginTransaction()) - { - refreshAction(dbContext, productItems); - dbContext.SaveChanges(); - dbContextTransaction.Commit(); - } + using var dbContext = new LibationContext(); + using var dbContextTransaction = dbContext.Database.BeginTransaction(); + refreshAction(dbContext, productItems); + dbContext.SaveChanges(); + dbContextTransaction.Commit(); aggregate root is transactional boundary // //context.Database.CurrentTransaction //var dbTransaction = Microsoft.EntityFrameworkCore.Storage.DbContextTransactionExtensions.GetDbTransaction(context.Database.CurrentTransaction); - // // test with and without : using (TransactionScope scope = new TransactionScope()) + // // test with and without : using TransactionScope scope = new TransactionScope(); //System.Transactions.Transaction.Current.TransactionCompleted += (sender, e) => { }; // also : https://docs.microsoft.com/en-us/dotnet/api/system.transactions.transaction.enlistvolatile diff --git a/_DB_NOTES.txt b/_DB_NOTES.txt index 7dae2687..20a3e4f8 100644 --- a/_DB_NOTES.txt +++ b/_DB_NOTES.txt @@ -1,7 +1,7 @@ Logging/Debugging (EF CORE) =========================== Once you configure logging on a DbContext instance it will be enabled on all instances of that DbContext type -using (var context = new MyContext()) + using var context = new MyContext(); context.ConfigureLogging(s => System.Diagnostics.Debug.WriteLine(s)); // write to Visual Studio "Output" tab //context.ConfigureLogging(s => Console.WriteLine(s)); see comments at top of file: diff --git a/_Demos/ffmpeg decrypt/Form1.cs b/_Demos/ffmpeg decrypt/Form1.cs index 7a3fc681..aa6e1262 100644 --- a/_Demos/ffmpeg decrypt/Form1.cs +++ b/_Demos/ffmpeg decrypt/Form1.cs @@ -23,22 +23,20 @@ namespace ffmpeg_decrypt private void inpbutton_Click(object sender, EventArgs e) { - using (var ofd = new OpenFileDialog { Filter = "Audible Audio Files|*.aax", Title = "Select an Audible Audio File", FileName = "" }) + using var ofd = new OpenFileDialog { Filter = "Audible Audio Files|*.aax", Title = "Select an Audible Audio File", FileName = "" }; + if (ofd.ShowDialog() == DialogResult.OK) { - if (ofd.ShowDialog() == DialogResult.OK) - { - inputdisplay.Text = ofd.FileName; - outputdisplay.Text = Path.GetDirectoryName(ofd.FileName); - convertbutton.Enabled = true; - } + inputdisplay.Text = ofd.FileName; + outputdisplay.Text = Path.GetDirectoryName(ofd.FileName); + convertbutton.Enabled = true; } } private void outpbutton_Click(object sender, EventArgs e) { - using (var fbd = new FolderBrowserDialog()) - if (fbd.ShowDialog() == DialogResult.OK && !string.IsNullOrWhiteSpace(fbd.SelectedPath)) - outputdisplay.Text = fbd.SelectedPath; + using var fbd = new FolderBrowserDialog(); + if (fbd.ShowDialog() == DialogResult.OK && !string.IsNullOrWhiteSpace(fbd.SelectedPath)) + outputdisplay.Text = fbd.SelectedPath; } private async void convertbutton_Click(object sender, EventArgs e) @@ -87,18 +85,15 @@ namespace ffmpeg_decrypt WorkingDirectory = Directory.GetCurrentDirectory() }; - string ffprobeStderr; - using (var ffp = new Process { StartInfo = startInfo }) - { - ffp.Start(); + using var ffp = new Process { StartInfo = startInfo }; + ffp.Start(); - // checksum is in the debug info. ffprobe's debug info is written to stderr, not stdout - ffprobeStderr = ffp.StandardError.ReadToEnd(); + // checksum is in the debug info. ffprobe's debug info is written to stderr, not stdout + var ffprobeStderr = ffp.StandardError.ReadToEnd(); - await Task.Run(() => ffp.WaitForExit()); + await Task.Run(() => ffp.WaitForExit()); - ffp.Close(); - } + ffp.Close(); // example checksum line: // ... [aax] file checksum == 0c527840c4f18517157eb0b4f9d6f9317ce60cd1 @@ -135,16 +130,13 @@ namespace ffmpeg_decrypt WorkingDirectory = Directory.GetCurrentDirectory() }; - string rcrackStdout; - using (var rcr = new Process { StartInfo = startInfo }) - { - rcr.Start(); + using var rcr = new Process { StartInfo = startInfo }; + rcr.Start(); - rcrackStdout = rcr.StandardOutput.ReadToEnd(); + var rcrackStdout = rcr.StandardOutput.ReadToEnd(); - await Task.Run(() => rcr.WaitForExit()); - rcr.Close(); - } + await Task.Run(() => rcr.WaitForExit()); + rcr.Close(); // example result // 0c527840c4f18517157eb0b4f9d6f9317ce60cd1 \xbd\x89X\x09 hex:bd895809 @@ -202,15 +194,13 @@ namespace ffmpeg_decrypt WorkingDirectory = Directory.GetCurrentDirectory() }; - using (var ffm = new Process { StartInfo = startInfo, EnableRaisingEvents = true }) - { - ffm.ErrorDataReceived += (s, ea) => debugWindow.UIThread(() => debugWindow.AppendText($"DEBUG: {ea.Data}\r\n")); + using var ffm = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + ffm.ErrorDataReceived += (s, ea) => debugWindow.UIThread(() => debugWindow.AppendText($"DEBUG: {ea.Data}\r\n")); - ffm.Start(); - ffm.BeginErrorReadLine(); - await Task.Run(() => ffm.WaitForExit()); - ffm.Close(); - } + ffm.Start(); + ffm.BeginErrorReadLine(); + await Task.Run(() => ffm.WaitForExit()); + ffm.Close(); } /// extract embedded resource to file if it doesn't already exist @@ -224,11 +214,11 @@ namespace ffmpeg_decrypt // this technique works but there are easier ways: // https://stackoverflow.com/questions/13031778/how-can-i-extract-a-file-from-an-embedded-resource-and-save-it-to-disk Directory.CreateDirectory(resdir); - using (var resource = System.Reflection.Assembly.GetCallingAssembly().GetManifestResourceStream($"{nameof(ffmpeg_decrypt)}.res." + resourceName)) - using (var reader = new BinaryReader(resource)) - using (var file = new FileStream(Path.Combine(resdir, resourceName), FileMode.OpenOrCreate)) - using (var writer = new BinaryWriter(file)) - writer.Write(reader.ReadBytes((int)resource.Length)); + using var resource = System.Reflection.Assembly.GetCallingAssembly().GetManifestResourceStream($"{nameof(ffmpeg_decrypt)}.res." + resourceName); + using var reader = new BinaryReader(resource); + using var file = new FileStream(Path.Combine(resdir, resourceName), FileMode.OpenOrCreate); + using var writer = new BinaryWriter(file); + writer.Write(reader.ReadBytes((int)resource.Length)); } private void decryptConvertRb_CheckedChanged(object sender, EventArgs e) => convertGb.Enabled = convertRb.Checked; diff --git a/__TODO.txt b/__TODO.txt index a5a153c6..8a7f110a 100644 --- a/__TODO.txt +++ b/__TODO.txt @@ -11,8 +11,8 @@ incl episodes. eg: "Bill Bryson's Appliance of Science" replace all scraping with audible api public partial class ScanLibraryDialog : Form, IIndexLibraryDialog public async Task DoMainWorkAsync() - using (var pageRetriever = websiteProcessorControl1.GetPageRetriever()) - jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false); + using var pageRetriever = websiteProcessorControl1.GetPageRetriever(); + jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false); break out DTOs. currently coupled with scraping