tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Convert to new Core 3.0 using declarations

Browse Source
This commit is contained in:
Robert McRackan 2019-10-08 08:54:12 -04:00
parent b0fec23a51
commit 1b6c577044
20 changed files with 324 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
AaxDecrypter/UNTESTED/AaxToM4bConverter.cs
View File

@ -117,9 +117,9 @@ namespace AaxDecrypter
private void saveCover(string aaxFile) private void saveCover(string aaxFile)
{ {
using (var file = TagLib.File.Create(aaxFile, "audio/mp4", TagLib.ReadStyle.Average)) using var file = TagLib.File.Create(aaxFile, "audio/mp4", TagLib.ReadStyle.Average);
this.coverBytes = file.Tag.Pictures[0].Data.Data; this.coverBytes = file.Tag.Pictures[0].Data.Data;
} }
private void printPrelim() private void printPrelim()
{ {

46
AaxDecrypter/UNTESTED/Tags.cs
View File

@ -25,35 +25,31 @@ namespace AaxDecrypter
public Tags(string file) public Tags(string file)
{ {
using (TagLib.File tagLibFile = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average)) using TagLib.File tagLibFile = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average);
{ this.title = tagLibFile.Tag.Title.Replace(" (Unabridged)", "");
this.title = tagLibFile.Tag.Title.Replace(" (Unabridged)", ""); this.album = tagLibFile.Tag.Album.Replace(" (Unabridged)", "");
this.album = tagLibFile.Tag.Album.Replace(" (Unabridged)", ""); this.author = tagLibFile.Tag.FirstPerformer;
this.author = tagLibFile.Tag.FirstPerformer; this.year = tagLibFile.Tag.Year.ToString();
this.year = tagLibFile.Tag.Year.ToString(); this.comments = tagLibFile.Tag.Comment;
this.comments = tagLibFile.Tag.Comment; this.duration = tagLibFile.Properties.Duration;
this.duration = tagLibFile.Properties.Duration; this.genre = tagLibFile.Tag.FirstGenre;
this.genre = tagLibFile.Tag.FirstGenre;
var tag = tagLibFile.GetTag(TagTypes.Apple, true); var tag = tagLibFile.GetTag(TagTypes.Apple, true);
this.publisher = tag.Publisher; this.publisher = tag.Publisher;
this.narrator = string.IsNullOrWhiteSpace(tagLibFile.Tag.FirstComposer) ? tag.Narrator : tagLibFile.Tag.FirstComposer; this.narrator = string.IsNullOrWhiteSpace(tagLibFile.Tag.FirstComposer) ? tag.Narrator : tagLibFile.Tag.FirstComposer;
this.comments = !string.IsNullOrWhiteSpace(tag.LongDescription) ? tag.LongDescription : tag.Description; this.comments = !string.IsNullOrWhiteSpace(tag.LongDescription) ? tag.LongDescription : tag.Description;
this.id = tag.AudibleCDEK; this.id = tag.AudibleCDEK;
} }
}
public void AddAppleTags(string file) public void AddAppleTags(string file)
{ {
using (var file1 = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average)) using var file1 = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average);
{ var tag = (AppleTag)file1.GetTag(TagTypes.Apple, true);
var tag = (AppleTag)file1.GetTag(TagTypes.Apple, true); tag.Publisher = this.publisher;
tag.Publisher = this.publisher; tag.LongDescription = this.comments;
tag.LongDescription = this.comments; tag.Description = this.comments;
tag.Description = this.comments; file1.Save();
file1.Save(); }
}
}
public string GenerateFfmpegTags() public string GenerateFfmpegTags()
{ {

40
AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs
View File

@ -94,27 +94,27 @@ var pageSource = new AudiblePageSource(AudiblePageType.Library, html, null);
private static async Task<string> getLibraryPageAsync(CookieContainer cookies, int pageNum) private static async Task<string> getLibraryPageAsync(CookieContainer cookies, int pageNum)
{ {
#region // POST example (from 2017 ajax) #region // POST example (from 2017 ajax)
// var destination = "https://www.audible.com/lib-ajax"; // var destination = "https://www.audible.com/lib-ajax";
// var webRequest = (HttpWebRequest)WebRequest.Create(destination); // var webRequest = (HttpWebRequest)WebRequest.Create(destination);
// webRequest.Method = "POST"; // webRequest.Method = "POST";
// webRequest.Accept = "*/*"; // webRequest.Accept = "*/*";
// webRequest.AllowAutoRedirect = false; // webRequest.AllowAutoRedirect = false;
// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)"; // webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)";
// webRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8"; // webRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
// webRequest.Credentials = null; // webRequest.Credentials = null;
// //
// webRequest.CookieContainer = new CookieContainer(); // webRequest.CookieContainer = new CookieContainer();
// webRequest.CookieContainer.Add(cookies.GetCookies(new Uri(destination))); // webRequest.CookieContainer.Add(cookies.GetCookies(new Uri(destination)));
// //
// var postData = $"progType=all&timeFilter=all&itemsPerPage={itemsPerPage}&searchTerm=&searchType=&sortColumn=&sortType=down&page={pageNum}&mode=normal&subId=&subTitle="; // var postData = $"progType=all&timeFilter=all&itemsPerPage={itemsPerPage}&searchTerm=&searchType=&sortColumn=&sortType=down&page={pageNum}&mode=normal&subId=&subTitle=";
// var data = Encoding.UTF8.GetBytes(postData); // var data = Encoding.UTF8.GetBytes(postData);
// webRequest.ContentLength = data.Length; // webRequest.ContentLength = data.Length;
// using (var dataStream = webRequest.GetRequestStream()) // using var dataStream = webRequest.GetRequestStream();
// dataStream.Write(data, 0, data.Length); // dataStream.Write(data, 0, data.Length);
#endregion #endregion
var destination = "https://" + $"www.audible.com/lib?purchaseDateFilter=all&programFilter=all&sortBy=PURCHASE_DATE.dsc&page={pageNum}"; var destination = "https://" + $"www.audible.com/lib?purchaseDateFilter=all&programFilter=all&sortBy=PURCHASE_DATE.dsc&page={pageNum}";
var webRequest = (HttpWebRequest)WebRequest.Create(destination); var webRequest = (HttpWebRequest)WebRequest.Create(destination);
webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)"; webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)";

79
CookieMonster/UNTESTED/Browsers/Chrome.cs
View File

@ -10,53 +10,50 @@ namespace CookieMonster
{ {
internal class Chrome : IBrowser internal class Chrome : IBrowser
{ {
public async Task<IEnumerable<CookieValue>> GetAllCookiesAsync() public async Task<IEnumerable<CookieValue>> GetAllCookiesAsync()
{ {
var col = new List<CookieValue>(); var col = new List<CookieValue>();
var strPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"Google\Chrome\User Data\Default\Cookies"); var strPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"Google\Chrome\User Data\Default\Cookies");
if (!FileUtility.FileExists(strPath)) if (!FileUtility.FileExists(strPath))
return col; return col;
// //
// IF WE GET AN ERROR HERE // IF WE GET AN ERROR HERE
// then add a reference to sqlite core in the project which is ultimately calling this. // then add a reference to sqlite core in the project which is ultimately calling this.
// a project which directly references CookieMonster doesn't need to also ref sqlite. // a project which directly references CookieMonster doesn't need to also ref sqlite.
// however, for any further number of abstractions, the project needs to directly ref sqlite. // however, for any further number of abstractions, the project needs to directly ref sqlite.
// eg: this will not work unless the winforms proj adds sqlite to ref.s: // eg: this will not work unless the winforms proj adds sqlite to ref.s:
// LibationWinForm > AudibleDotComAutomation > CookieMonster // LibationWinForm > AudibleDotComAutomation > CookieMonster
// //
using (var conn = new SQLiteConnection("Data Source=" + strPath + ";pooling=false")) using var conn = new SQLiteConnection("Data Source=" + strPath + ";pooling=false");
using (var cmd = conn.CreateCommand()) using var cmd = conn.CreateCommand();
{ cmd.CommandText = "SELECT host_key, name, value, encrypted_value, last_access_utc, expires_utc FROM cookies;";
cmd.CommandText = "SELECT host_key, name, value, encrypted_value, last_access_utc, expires_utc FROM cookies;";
conn.Open(); conn.Open();
using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false);
{ while (reader.Read())
while (reader.Read()) {
{ var host_key = reader.GetString(0);
var host_key = reader.GetString(0); var name = reader.GetString(1);
var name = reader.GetString(1); var value = reader.GetString(2);
var value = reader.GetString(2); var last_access_utc = reader.GetInt64(4);
var last_access_utc = reader.GetInt64(4); var expires_utc = reader.GetInt64(5);
var expires_utc = reader.GetInt64(5);
// https://stackoverflow.com/a/25874366 // https://stackoverflow.com/a/25874366
if (string.IsNullOrWhiteSpace(value)) if (string.IsNullOrWhiteSpace(value))
{ {
var encrypted_value = (byte[])reader[3]; var encrypted_value = (byte[])reader[3];
var decodedData = System.Security.Cryptography.ProtectedData.Unprotect(encrypted_value, null, System.Security.Cryptography.DataProtectionScope.CurrentUser); var decodedData = System.Security.Cryptography.ProtectedData.Unprotect(encrypted_value, null, System.Security.Cryptography.DataProtectionScope.CurrentUser);
value = Encoding.ASCII.GetString(decodedData); value = Encoding.ASCII.GetString(decodedData);
} }
col.Add(new CookieValue { Browser = "chrome", Domain = host_key, Name = name, Value = value, LastAccess = chromeTimeToDateTimeUtc(last_access_utc), Expires = chromeTimeToDateTimeUtc(expires_utc) }); col.Add(new CookieValue { Browser = "chrome", Domain = host_key, Name = name, Value = value, LastAccess = chromeTimeToDateTimeUtc(last_access_utc), Expires = chromeTimeToDateTimeUtc(expires_utc) });
} }
}
}
return col;
} return col;
}
// Chrome uses 1601-01-01 00:00:00 UTC as the epoch (ie the starting point for the millisecond time counter). // Chrome uses 1601-01-01 00:00:00 UTC as the epoch (ie the starting point for the millisecond time counter).
// this is the same as "FILETIME" in Win32 except FILETIME uses 100ns ticks instead of ms. // this is the same as "FILETIME" in Win32 except FILETIME uses 100ns ticks instead of ms.

34
CookieMonster/UNTESTED/Browsers/FireFox.cs
View File

@ -28,27 +28,23 @@ namespace CookieMonster
File.Copy(strPath, strTemp, true); File.Copy(strPath, strTemp, true);
// Now open the temporary cookie jar and extract Value from the cookie if we find it. // Now open the temporary cookie jar and extract Value from the cookie if we find it.
using (var conn = new SQLiteConnection("Data Source=" + strTemp + ";pooling=false")) using var conn = new SQLiteConnection("Data Source=" + strTemp + ";pooling=false");
using (var cmd = conn.CreateCommand()) using var cmd = conn.CreateCommand();
{ cmd.CommandText = "SELECT host, name, value, lastAccessed, expiry FROM moz_cookies; ";
cmd.CommandText = "SELECT host, name, value, lastAccessed, expiry FROM moz_cookies; ";
conn.Open(); conn.Open();
using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false);
{ while (reader.Read())
while (reader.Read()) {
{ var host_key = reader.GetString(0);
var host_key = reader.GetString(0); var name = reader.GetString(1);
var name = reader.GetString(1); var value = reader.GetString(2);
var value = reader.GetString(2); var lastAccessed = reader.GetInt32(3);
var lastAccessed = reader.GetInt32(3); var expiry = reader.GetInt32(4);
var expiry = reader.GetInt32(4);
col.Add(new CookieValue { Browser = "firefox", Domain = host_key, Name = name, Value = value, LastAccess = lastAccessedToDateTime(lastAccessed), Expires = expiryToDateTime(expiry) }); col.Add(new CookieValue { Browser = "firefox", Domain = host_key, Name = name, Value = value, LastAccess = lastAccessedToDateTime(lastAccessed), Expires = expiryToDateTime(expiry) });
} }
}
}
if (FileUtility.FileExists(strTemp)) if (FileUtility.FileExists(strTemp))
File.Delete(strTemp); File.Delete(strTemp);

22
DataLayer/UNTESTED/QueryObjects/BookQueries.cs
View File

@ -10,20 +10,20 @@ namespace DataLayer
{ {
public static int BooksWithoutDetailsCount() public static int BooksWithoutDetailsCount()
{ {
using (var context = LibationContext.Create()) using var context = LibationContext.Create();
return context return context
.Books .Books
.Count(b => !b.HasBookDetails); .Count(b => !b.HasBookDetails);
} }
public static Book GetBook_Flat_NoTracking(string productId) public static Book GetBook_Flat_NoTracking(string productId)
{ {
using (var context = LibationContext.Create()) using var context = LibationContext.Create();
return context return context
.Books .Books
.AsNoTracking() .AsNoTracking()
.GetBook(productId); .GetBook(productId);
} }
public static Book GetBook(this IQueryable<Book> books, string productId) public static Book GetBook(this IQueryable<Book> books, string productId)
=> books => books

26
DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs
View File

@ -8,22 +8,22 @@ namespace DataLayer
{ {
public static List<LibraryBook> GetLibrary_Flat_NoTracking() public static List<LibraryBook> GetLibrary_Flat_NoTracking()
{ {
using (var context = LibationContext.Create()) using var context = LibationContext.Create();
return context return context
.Library .Library
.AsNoTracking() .AsNoTracking()
.GetLibrary() .GetLibrary()
.ToList(); .ToList();
} }
public static LibraryBook GetLibraryBook_Flat_NoTracking(string productId) public static LibraryBook GetLibraryBook_Flat_NoTracking(string productId)
{ {
using (var context = LibationContext.Create()) using var context = LibationContext.Create();
return context return context
.Library .Library
.AsNoTracking() .AsNoTracking()
.GetLibraryBook(productId); .GetLibraryBook(productId);
} }
/// <summary>This is still IQueryable. YOU MUST CALL ToList() YOURSELF</summary> /// <summary>This is still IQueryable. YOU MUST CALL ToList() YOURSELF</summary>
public static IQueryable<LibraryBook> GetLibrary(this IQueryable<LibraryBook> library) public static IQueryable<LibraryBook> GetLibrary(this IQueryable<LibraryBook> library)

12
DataLayer/UNTESTED/_scratch pad/ScratchPad.cs
View File

@ -13,13 +13,11 @@ namespace _scratch_pad
// var user = new Student() { Name = "Dinah Cheshire" }; // var user = new Student() { Name = "Dinah Cheshire" };
// var udi = new UserDef { UserDefId = 1, TagsRaw = "my,tags" }; // var udi = new UserDef { UserDefId = 1, TagsRaw = "my,tags" };
// using (var context = new MyTestContextDesignTimeDbContextFactory().Create()) // using var context = new MyTestContextDesignTimeDbContextFactory().Create();
// { // context.Add(user);
// context.Add(user); // //context.Add(udi);
// //context.Add(udi); // context.Update(udi);
// context.Update(udi); // context.SaveChanges();
// context.SaveChanges();
// }
// Console.WriteLine($"Student was saved in the database with id: {user.Id}"); // Console.WriteLine($"Student was saved in the database with id: {user.Id}");
// } // }

10
DomainServices/UNTESTED/DownloadBook.cs
View File

@ -43,13 +43,11 @@ namespace DomainServices
.Replace("&DownloadType=Now", "") .Replace("&DownloadType=Now", "")
+ "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download"; + "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download";
var uri = new Uri(aaxDownloadLink); var uri = new Uri(aaxDownloadLink);
using (var webClient = await GetWebClient(tempAaxFilename))
{
// for book downloads only: pretend to be the audible download manager. from inAudible:
webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601";
await webClient.DownloadFileTaskAsync(uri, tempAaxFilename); using var webClient = await GetWebClient(tempAaxFilename);
} // for book downloads only: pretend to be the audible download manager. from inAudible:
webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601";
await webClient.DownloadFileTaskAsync(uri, tempAaxFilename);
// move // move
var aaxFilename = FileUtility.GetValidFilename( var aaxFilename = FileUtility.GetValidFilename(

4
DomainServices/UNTESTED/DownloadPdf.cs
View File

@ -43,8 +43,8 @@ namespace DomainServices
var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url)); var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url));
using (var webClient = await GetWebClient(destinationFilename)) using var webClient = await GetWebClient(destinationFilename);
await webClient.DownloadFileTaskAsync(url, destinationFilename); await webClient.DownloadFileTaskAsync(url, destinationFilename);
var statusHandler = new StatusHandler(); var statusHandler = new StatusHandler();
var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId); var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);

68
DomainServices/UNTESTED/Indexer.cs
View File

@ -55,28 +55,25 @@ namespace DomainServices
productItems = filterAndValidate(productItems); productItems = filterAndValidate(productItems);
int newEntries; using var context = LibationContext.Create();
using (var context = LibationContext.Create()) var dtoImporter = new DtoImporter(context);
{
var dtoImporter = new DtoImporter(context);
#region // benchmarks. re-importing a library with 500 books, all with book details json files #region // benchmarks. re-importing a library with 500 books, all with book details json files
/* /*
dtoImporter.ReplaceLibrary 1.2 seconds dtoImporter.ReplaceLibrary 1.2 seconds
SaveChanges() 3.4 SaveChanges() 3.4
ReloadBookDetails() 1.3 ReloadBookDetails() 1.3
SaveChanges() 1.4 SaveChanges() 1.4
*/ */
#endregion #endregion
// LONG RUNNING // LONG RUNNING
newEntries = await Task.Run(() => dtoImporter.ReplaceLibrary(productItems)); var newEntries = await Task.Run(() => dtoImporter.ReplaceLibrary(productItems));
await context.SaveChangesAsync(); await context.SaveChangesAsync();
// must be broken out. see notes in dtoImporter.ReplaceLibrary() // must be broken out. see notes in dtoImporter.ReplaceLibrary()
// LONG RUNNING // LONG RUNNING
await Task.Run(() => dtoImporter.ReloadBookDetails(productItems)); await Task.Run(() => dtoImporter.ReloadBookDetails(productItems));
await context.SaveChangesAsync(); await context.SaveChangesAsync();
}
await postIndexActionAsync?.Invoke(); await postIndexActionAsync?.Invoke();
@ -111,13 +108,10 @@ namespace DomainServices
#region update book tags #region update book tags
public static int IndexChangedTags(Book book) public static int IndexChangedTags(Book book)
{ {
// update disconnected entity // update disconnected entity
int qtyChanges; using var context = LibationContext.Create();
using (var context = LibationContext.Create()) context.Update(book);
{ var qtyChanges = context.SaveChanges();
context.Update(book);
qtyChanges = context.SaveChanges();
}
// this part is tags-specific // this part is tags-specific
if (qtyChanges > 0) if (qtyChanges > 0)
@ -151,19 +145,15 @@ namespace DomainServices
validate(bookDetailDTO); validate(bookDetailDTO);
using (var context = LibationContext.Create()) using var context = LibationContext.Create();
{ var dtoImporter = new DtoImporter(context);
var dtoImporter = new DtoImporter(context); // LONG RUNNING
// LONG RUNNING await Task.Run(() => dtoImporter.UpdateBookDetails(bookDetailDTO));
await Task.Run(() => dtoImporter.UpdateBookDetails(bookDetailDTO)); context.SaveChanges();
context.SaveChanges();
// after saving, delete orphan contributors // after saving, delete orphan contributors
var count = context.RemoveOrphans(); var count = context.RemoveOrphans();
if (count > 0) if (count > 0) { } // don't think there's a to-do here
{
}
}
await postIndexActionAsync?.Invoke(); await postIndexActionAsync?.Invoke();
} }

54
DomainServices/UNTESTED/ScrapeBookDetails.cs
View File

@ -72,37 +72,35 @@ namespace DomainServices
// download htm // download htm
string source; string source;
var url = AudiblePage.Product.GetUrl(productId); var url = AudiblePage.Product.GetUrl(productId);
using (var webClient = await GetWebClient($"Getting Book Details for {libraryBook.Book.Title}")) using var webClient = await GetWebClient($"Getting Book Details for {libraryBook.Book.Title}");
{ try
try {
{ source = await webClient.DownloadStringTaskAsync(url);
source = await webClient.DownloadStringTaskAsync(url); var detailsAudiblePageSource = new AudiblePageSource(AudiblePageType.ProductDetails, source, productId);
var detailsAudiblePageSource = new AudiblePageSource(AudiblePageType.ProductDetails, source, productId);
// good habit to persist htm before attempting to parse it. this way, if there's a parse error, we can test errors on a local copy // good habit to persist htm before attempting to parse it. this way, if there's a parse error, we can test errors on a local copy
DataConverter.AudiblePageSource_2_HtmFile_Product(detailsAudiblePageSource); DataConverter.AudiblePageSource_2_HtmFile_Product(detailsAudiblePageSource);
bookDetailDTO = AudibleScraper.ScrapeBookDetailsSource(detailsAudiblePageSource); bookDetailDTO = AudibleScraper.ScrapeBookDetailsSource(detailsAudiblePageSource);
} }
catch (System.Net.WebException webEx) catch (System.Net.WebException webEx)
{ {
// cannot continue if NoLongerAvailableAction is null, // cannot continue if NoLongerAvailableAction is null,
// else we'll be right back here next loop (and infinitely) with no failure condition // else we'll be right back here next loop (and infinitely) with no failure condition
if (webEx.Status != System.Net.WebExceptionStatus.ConnectionClosed || NoLongerAvailableAction == null) if (webEx.Status != System.Net.WebExceptionStatus.ConnectionClosed || NoLongerAvailableAction == null)
throw; throw;
var nlaEnum = NoLongerAvailableAction.Invoke( var nlaEnum = NoLongerAvailableAction.Invoke(
libraryBook.Book.Title, libraryBook.Book.Title,
AudiblePage.Product.GetUrl(libraryBook.Book.AudibleProductId)); AudiblePage.Product.GetUrl(libraryBook.Book.AudibleProductId));
if (nlaEnum == NoLongerAvailableEnum.Abort) if (nlaEnum == NoLongerAvailableEnum.Abort)
return new StatusHandler { "Cannot scrape book details. Aborting." }; return new StatusHandler { "Cannot scrape book details. Aborting." };
else if (nlaEnum == NoLongerAvailableEnum.MarkAsMissing) else if (nlaEnum == NoLongerAvailableEnum.MarkAsMissing)
bookDetailDTO = new BookDetailDTO { ProductId = productId }; bookDetailDTO = new BookDetailDTO { ProductId = productId };
else else
throw; throw;
} }
} }
}
DataConverter.Value_2_JsonFile(bookDetailDTO, jsonFileInfo.FullName); DataConverter.Value_2_JsonFile(bookDetailDTO, jsonFileInfo.FullName);
} }

58
FileManager/UNTESTED/PictureStorage.cs
View File

@ -39,39 +39,37 @@ namespace FileManager
{ {
try try
{ {
using (var webClient = new System.Net.WebClient()) using var webClient = new System.Net.WebClient();
{ // download any that don't exist
// download any that don't exist {
{ if (!FileUtility.FileExists(path80))
if (!FileUtility.FileExists(path80)) {
{ var bytes = webClient.DownloadData(
var bytes = webClient.DownloadData( "https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL80_.jpg");
"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL80_.jpg"); File.WriteAllBytes(path80, bytes);
File.WriteAllBytes(path80, bytes); }
} }
}
{ {
if (!FileUtility.FileExists(path300)) if (!FileUtility.FileExists(path300))
{ {
var bytes = webClient.DownloadData( var bytes = webClient.DownloadData(
"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL300_.jpg"); "https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL300_.jpg");
File.WriteAllBytes(path300, bytes); File.WriteAllBytes(path300, bytes);
} }
} }
{ {
if (!FileUtility.FileExists(path500)) if (!FileUtility.FileExists(path500))
{ {
var bytes = webClient.DownloadData( var bytes = webClient.DownloadData(
"https://m.media-amazon.com/images/I/" + pictureId + "._SL500_.jpg"); "https://m.media-amazon.com/images/I/" + pictureId + "._SL500_.jpg");
File.WriteAllBytes(path500, bytes); File.WriteAllBytes(path500, bytes);
} }
} }
break; break;
} }
}
catch { retry++; } catch { retry++; }
} }
while (retry < 3); while (retry < 3);

139
LibationSearchEngine/UNTESTED/SearchEngine.cs
View File

@ -177,26 +177,19 @@ namespace LibationSearchEngine
log(); log();
// location of index/create the index // location of index/create the index
using (var index = getIndex()) using var index = getIndex();
{ var exists = IndexReader.IndexExists(index);
var exists = IndexReader.IndexExists(index); var createNewIndex = overwrite || !exists;
var createNewIndex = overwrite || !exists;
// analyzer for tokenizing text. same analyzer should be used for indexing and searching // analyzer for tokenizing text. same analyzer should be used for indexing and searching
using (var analyzer = new StandardAnalyzer(Version)) using var analyzer = new StandardAnalyzer(Version);
using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED)) using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
{ foreach (var libraryBook in library)
foreach (var libraryBook in library) {
{ var doc = createBookIndexDocument(libraryBook);
var doc = createBookIndexDocument(libraryBook); ixWriter.AddDocument(doc);
ixWriter.AddDocument(doc); }
}
// don't optimize. deprecated: ixWriter.Optimize();
// ixWriter.Commit(); not needed if we're about to dispose of writer anyway. could be needed within the using() block
}
}
log(); log();
} }
@ -247,47 +240,41 @@ namespace LibationSearchEngine
var document = createBookIndexDocument(libraryBook); var document = createBookIndexDocument(libraryBook);
var createNewIndex = false; var createNewIndex = false;
using (var index = getIndex()) using var index = getIndex();
using (var analyzer = new StandardAnalyzer(Version)) using var analyzer = new StandardAnalyzer(Version);
using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED)) using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
{ ixWriter.DeleteDocuments(term);
ixWriter.DeleteDocuments(term); ixWriter.AddDocument(document);
ixWriter.AddDocument(document); }
}
}
public void UpdateTags(string productId, string tags) public void UpdateTags(string productId, string tags)
{ {
var productTerm = new Term(_ID_, productId); var productTerm = new Term(_ID_, productId);
using (var index = getIndex()) using var index = getIndex();
{
Document document;
// get existing document // get existing document
using (var searcher = new IndexSearcher(index)) using var searcher = new IndexSearcher(index);
{ var query = new TermQuery(productTerm);
var query = new TermQuery(productTerm); var docs = searcher.Search(query, 1);
var docs = searcher.Search(query, 1); var scoreDoc = docs.ScoreDocs.SingleOrDefault();
var scoreDoc = docs.ScoreDocs.SingleOrDefault(); if (scoreDoc == null)
if (scoreDoc == null) throw new Exception("document not found");
throw new Exception("document not found"); var document = searcher.Doc(scoreDoc.Doc);
document = searcher.Doc(scoreDoc.Doc);
}
// update document entry with new tags
// fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new
// REMEMBER: all fields, including 'tags' are case-specific
document.RemoveField(TAGS);
document.AddAnalyzed(TAGS, tags);
// update index // update document entry with new tags
var createNewIndex = false; // fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new
using (var analyzer = new StandardAnalyzer(Version)) // REMEMBER: all fields, including 'tags' are case-specific
using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED)) document.RemoveField(TAGS);
ixWriter.UpdateDocument(productTerm, document, analyzer); document.AddAnalyzed(TAGS, tags);
}
} // update index
var createNewIndex = false;
using var analyzer = new StandardAnalyzer(Version);
using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
ixWriter.UpdateDocument(productTerm, document, analyzer);
}
public SearchResultSet Search(string searchString) public SearchResultSet Search(string searchString)
{ {
@ -381,35 +368,33 @@ namespace LibationSearchEngine
var defaultField = ALL; var defaultField = ALL;
using (var index = getIndex()) using var index = getIndex();
using (var searcher = new IndexSearcher(index)) using var searcher = new IndexSearcher(index);
using (var analyzer = new StandardAnalyzer(Version)) using var analyzer = new StandardAnalyzer(Version);
{ var query = analyzer.GetQuery(defaultField, searchString);
var query = analyzer.GetQuery(defaultField, searchString);
// lucene doesn't allow only negations. eg this returns nothing: // lucene doesn't allow only negations. eg this returns nothing:
// -tags:hidden // -tags:hidden
// work arounds: https://kb.ucla.edu/articles/pure-negation-query-in-lucene // work arounds: https://kb.ucla.edu/articles/pure-negation-query-in-lucene
// HOWEVER, doing this to any other type of query can cause EVERYTHING to be a match unless "Occur" is carefully set // HOWEVER, doing this to any other type of query can cause EVERYTHING to be a match unless "Occur" is carefully set
// this should really check that all leaf nodes are MUST_NOT // this should really check that all leaf nodes are MUST_NOT
if (query is BooleanQuery boolQuery) if (query is BooleanQuery boolQuery)
{ {
var occurs = getOccurs_recurs(boolQuery); var occurs = getOccurs_recurs(boolQuery);
if (occurs.Any() && occurs.All(o => o == Occur.MUST_NOT)) if (occurs.Any() && occurs.All(o => o == Occur.MUST_NOT))
boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST); boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST);
} }
Console.WriteLine($" query: {query}"); Console.WriteLine($" query: {query}");
var docs = searcher var docs = searcher
.Search(query, MaxSearchResultsToReturn) .Search(query, MaxSearchResultsToReturn)
.ScoreDocs .ScoreDocs
.Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score)) .Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score))
.ToList(); .ToList();
return new SearchResultSet(query.ToString(), docs); return new SearchResultSet(query.ToString(), docs);
} }
}
private IEnumerable<Occur> getOccurs_recurs(BooleanQuery query) private IEnumerable<Occur> getOccurs_recurs(BooleanQuery query)
{ {

12
LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs
View File

@ -11,12 +11,12 @@ namespace LibationWinForm.BookLiberation
{ {
async Task BackupBookAsync(string productId) async Task BackupBookAsync(string productId)
{ {
LibraryBook libraryBook; using var context = LibationContext.Create();
using (var context = LibationContext.Create())
libraryBook = context var libraryBook = context
.Library .Library
.GetLibrary() .GetLibrary()
.SingleOrDefault(lb => lb.Book.AudibleProductId == productId); .SingleOrDefault(lb => lb.Book.AudibleProductId == productId);
if (libraryBook == null) if (libraryBook == null)
return; return;

5
LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs
View File

@ -25,9 +25,8 @@ namespace LibationWinForm
public async Task DoMainWorkAsync() public async Task DoMainWorkAsync()
{ {
List<FileInfo> jsonFilepaths; using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
using (var pageRetriever = websiteProcessorControl1.GetPageRetriever()) var jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
successMessages.Add($"Downloaded {"library page".PluralizeWithCount(jsonFilepaths.Count)}"); successMessages.Add($"Downloaded {"library page".PluralizeWithCount(jsonFilepaths.Count)}");

14
REFERENCE.txt
View File

@ -63,18 +63,16 @@ transaction notes
----------------- -----------------
// https://msdn.microsoft.com/en-us/data/dn456843.aspx // https://msdn.microsoft.com/en-us/data/dn456843.aspx
// Rollback is called by transaction Dispose(). No need to call it explicitly // Rollback is called by transaction Dispose(). No need to call it explicitly
using (var dbContext = new LibationContext()) using var dbContext = new LibationContext();
using (var dbContextTransaction = dbContext.Database.BeginTransaction()) using var dbContextTransaction = dbContext.Database.BeginTransaction();
{ refreshAction(dbContext, productItems);
refreshAction(dbContext, productItems); dbContext.SaveChanges();
dbContext.SaveChanges(); dbContextTransaction.Commit();
dbContextTransaction.Commit();
}
aggregate root is transactional boundary aggregate root is transactional boundary
// //context.Database.CurrentTransaction // //context.Database.CurrentTransaction
//var dbTransaction = Microsoft.EntityFrameworkCore.Storage.DbContextTransactionExtensions.GetDbTransaction(context.Database.CurrentTransaction); //var dbTransaction = Microsoft.EntityFrameworkCore.Storage.DbContextTransactionExtensions.GetDbTransaction(context.Database.CurrentTransaction);
// // test with and without : using (TransactionScope scope = new TransactionScope()) // // test with and without : using TransactionScope scope = new TransactionScope();
//System.Transactions.Transaction.Current.TransactionCompleted += (sender, e) => { }; //System.Transactions.Transaction.Current.TransactionCompleted += (sender, e) => { };
// also : https://docs.microsoft.com/en-us/dotnet/api/system.transactions.transaction.enlistvolatile // also : https://docs.microsoft.com/en-us/dotnet/api/system.transactions.transaction.enlistvolatile

2
_DB_NOTES.txt
View File

@ -1,7 +1,7 @@
Logging/Debugging (EF CORE) Logging/Debugging (EF CORE)
=========================== ===========================
Once you configure logging on a DbContext instance it will be enabled on all instances of that DbContext type Once you configure logging on a DbContext instance it will be enabled on all instances of that DbContext type
using (var context = new MyContext()) using var context = new MyContext();
context.ConfigureLogging(s => System.Diagnostics.Debug.WriteLine(s)); // write to Visual Studio "Output" tab context.ConfigureLogging(s => System.Diagnostics.Debug.WriteLine(s)); // write to Visual Studio "Output" tab
//context.ConfigureLogging(s => Console.WriteLine(s)); //context.ConfigureLogging(s => Console.WriteLine(s));
see comments at top of file: see comments at top of file:

70
_Demos/ffmpeg decrypt/Form1.cs
View File

@ -23,22 +23,20 @@ namespace ffmpeg_decrypt
private void inpbutton_Click(object sender, EventArgs e) private void inpbutton_Click(object sender, EventArgs e)
{ {
using (var ofd = new OpenFileDialog { Filter = "Audible Audio Files|*.aax", Title = "Select an Audible Audio File", FileName = "" }) using var ofd = new OpenFileDialog { Filter = "Audible Audio Files|*.aax", Title = "Select an Audible Audio File", FileName = "" };
if (ofd.ShowDialog() == DialogResult.OK)
{ {
if (ofd.ShowDialog() == DialogResult.OK) inputdisplay.Text = ofd.FileName;
{ outputdisplay.Text = Path.GetDirectoryName(ofd.FileName);
inputdisplay.Text = ofd.FileName; convertbutton.Enabled = true;
outputdisplay.Text = Path.GetDirectoryName(ofd.FileName);
convertbutton.Enabled = true;
}
} }
} }
private void outpbutton_Click(object sender, EventArgs e) private void outpbutton_Click(object sender, EventArgs e)
{ {
using (var fbd = new FolderBrowserDialog()) using var fbd = new FolderBrowserDialog();
if (fbd.ShowDialog() == DialogResult.OK && !string.IsNullOrWhiteSpace(fbd.SelectedPath)) if (fbd.ShowDialog() == DialogResult.OK && !string.IsNullOrWhiteSpace(fbd.SelectedPath))
outputdisplay.Text = fbd.SelectedPath; outputdisplay.Text = fbd.SelectedPath;
} }
private async void convertbutton_Click(object sender, EventArgs e) private async void convertbutton_Click(object sender, EventArgs e)
@ -87,18 +85,15 @@ namespace ffmpeg_decrypt
WorkingDirectory = Directory.GetCurrentDirectory() WorkingDirectory = Directory.GetCurrentDirectory()
}; };
string ffprobeStderr; using var ffp = new Process { StartInfo = startInfo };
using (var ffp = new Process { StartInfo = startInfo }) ffp.Start();
{
ffp.Start();
// checksum is in the debug info. ffprobe's debug info is written to stderr, not stdout // checksum is in the debug info. ffprobe's debug info is written to stderr, not stdout
ffprobeStderr = ffp.StandardError.ReadToEnd(); var ffprobeStderr = ffp.StandardError.ReadToEnd();
await Task.Run(() => ffp.WaitForExit()); await Task.Run(() => ffp.WaitForExit());
ffp.Close(); ffp.Close();
}
// example checksum line: // example checksum line:
// ... [aax] file checksum == 0c527840c4f18517157eb0b4f9d6f9317ce60cd1 // ... [aax] file checksum == 0c527840c4f18517157eb0b4f9d6f9317ce60cd1
@ -135,16 +130,13 @@ namespace ffmpeg_decrypt
WorkingDirectory = Directory.GetCurrentDirectory() WorkingDirectory = Directory.GetCurrentDirectory()
}; };
string rcrackStdout; using var rcr = new Process { StartInfo = startInfo };
using (var rcr = new Process { StartInfo = startInfo }) rcr.Start();
{
rcr.Start();
rcrackStdout = rcr.StandardOutput.ReadToEnd(); var rcrackStdout = rcr.StandardOutput.ReadToEnd();
await Task.Run(() => rcr.WaitForExit()); await Task.Run(() => rcr.WaitForExit());
rcr.Close(); rcr.Close();
}
// example result // example result
// 0c527840c4f18517157eb0b4f9d6f9317ce60cd1 \xbd\x89X\x09 hex:bd895809 // 0c527840c4f18517157eb0b4f9d6f9317ce60cd1 \xbd\x89X\x09 hex:bd895809
@ -202,15 +194,13 @@ namespace ffmpeg_decrypt
WorkingDirectory = Directory.GetCurrentDirectory() WorkingDirectory = Directory.GetCurrentDirectory()
}; };
using (var ffm = new Process { StartInfo = startInfo, EnableRaisingEvents = true }) using var ffm = new Process { StartInfo = startInfo, EnableRaisingEvents = true };
{ ffm.ErrorDataReceived += (s, ea) => debugWindow.UIThread(() => debugWindow.AppendText($"DEBUG: {ea.Data}\r\n"));
ffm.ErrorDataReceived += (s, ea) => debugWindow.UIThread(() => debugWindow.AppendText($"DEBUG: {ea.Data}\r\n"));
ffm.Start(); ffm.Start();
ffm.BeginErrorReadLine(); ffm.BeginErrorReadLine();
await Task.Run(() => ffm.WaitForExit()); await Task.Run(() => ffm.WaitForExit());
ffm.Close(); ffm.Close();
}
} }
/// <summary>extract embedded resource to file if it doesn't already exist</summary> /// <summary>extract embedded resource to file if it doesn't already exist</summary>
@ -224,11 +214,11 @@ namespace ffmpeg_decrypt
// this technique works but there are easier ways: // this technique works but there are easier ways:
// https://stackoverflow.com/questions/13031778/how-can-i-extract-a-file-from-an-embedded-resource-and-save-it-to-disk // https://stackoverflow.com/questions/13031778/how-can-i-extract-a-file-from-an-embedded-resource-and-save-it-to-disk
Directory.CreateDirectory(resdir); Directory.CreateDirectory(resdir);
using (var resource = System.Reflection.Assembly.GetCallingAssembly().GetManifestResourceStream($"{nameof(ffmpeg_decrypt)}.res." + resourceName)) using var resource = System.Reflection.Assembly.GetCallingAssembly().GetManifestResourceStream($"{nameof(ffmpeg_decrypt)}.res." + resourceName);
using (var reader = new BinaryReader(resource)) using var reader = new BinaryReader(resource);
using (var file = new FileStream(Path.Combine(resdir, resourceName), FileMode.OpenOrCreate)) using var file = new FileStream(Path.Combine(resdir, resourceName), FileMode.OpenOrCreate);
using (var writer = new BinaryWriter(file)) using var writer = new BinaryWriter(file);
writer.Write(reader.ReadBytes((int)resource.Length)); writer.Write(reader.ReadBytes((int)resource.Length));
} }
private void decryptConvertRb_CheckedChanged(object sender, EventArgs e) => convertGb.Enabled = convertRb.Checked; private void decryptConvertRb_CheckedChanged(object sender, EventArgs e) => convertGb.Enabled = convertRb.Checked;

4
__TODO.txt
View File

@ -11,8 +11,8 @@ incl episodes. eg: "Bill Bryson's Appliance of Science"
replace all scraping with audible api replace all scraping with audible api
public partial class ScanLibraryDialog : Form, IIndexLibraryDialog public partial class ScanLibraryDialog : Form, IIndexLibraryDialog
public async Task DoMainWorkAsync() public async Task DoMainWorkAsync()
using (var pageRetriever = websiteProcessorControl1.GetPageRetriever()) using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false); jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
break out DTOs. currently coupled with scraping break out DTOs. currently coupled with scraping