tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Convert to new Core 3.0 using declarations

Browse Source
This commit is contained in:
Robert McRackan 2019-10-08 08:54:12 -04:00
parent b0fec23a51
commit 1b6c577044
20 changed files with 324 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
AaxDecrypter/UNTESTED/AaxToM4bConverter.cs
View File

@ -117,9 +117,9 @@ namespace AaxDecrypter
private void saveCover(string aaxFile)
{
using (var file = TagLib.File.Create(aaxFile, "audio/mp4", TagLib.ReadStyle.Average))
this.coverBytes = file.Tag.Pictures[0].Data.Data;
}
using var file = TagLib.File.Create(aaxFile, "audio/mp4", TagLib.ReadStyle.Average);
this.coverBytes = file.Tag.Pictures[0].Data.Data;
}
private void printPrelim()
{

46
AaxDecrypter/UNTESTED/Tags.cs
View File

@ -25,35 +25,31 @@ namespace AaxDecrypter
public Tags(string file)
{
using (TagLib.File tagLibFile = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average))
{
this.title = tagLibFile.Tag.Title.Replace(" (Unabridged)", "");
this.album = tagLibFile.Tag.Album.Replace(" (Unabridged)", "");
this.author = tagLibFile.Tag.FirstPerformer;
this.year = tagLibFile.Tag.Year.ToString();
this.comments = tagLibFile.Tag.Comment;
this.duration = tagLibFile.Properties.Duration;
this.genre = tagLibFile.Tag.FirstGenre;
using TagLib.File tagLibFile = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average);
this.title = tagLibFile.Tag.Title.Replace(" (Unabridged)", "");
this.album = tagLibFile.Tag.Album.Replace(" (Unabridged)", "");
this.author = tagLibFile.Tag.FirstPerformer;
this.year = tagLibFile.Tag.Year.ToString();
this.comments = tagLibFile.Tag.Comment;
this.duration = tagLibFile.Properties.Duration;
this.genre = tagLibFile.Tag.FirstGenre;
var tag = tagLibFile.GetTag(TagTypes.Apple, true);
this.publisher = tag.Publisher;
this.narrator = string.IsNullOrWhiteSpace(tagLibFile.Tag.FirstComposer) ? tag.Narrator : tagLibFile.Tag.FirstComposer;
this.comments = !string.IsNullOrWhiteSpace(tag.LongDescription) ? tag.LongDescription : tag.Description;
this.id = tag.AudibleCDEK;
}
}
var tag = tagLibFile.GetTag(TagTypes.Apple, true);
this.publisher = tag.Publisher;
this.narrator = string.IsNullOrWhiteSpace(tagLibFile.Tag.FirstComposer) ? tag.Narrator : tagLibFile.Tag.FirstComposer;
this.comments = !string.IsNullOrWhiteSpace(tag.LongDescription) ? tag.LongDescription : tag.Description;
this.id = tag.AudibleCDEK;
}
public void AddAppleTags(string file)
{
using (var file1 = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average))
{
var tag = (AppleTag)file1.GetTag(TagTypes.Apple, true);
tag.Publisher = this.publisher;
tag.LongDescription = this.comments;
tag.Description = this.comments;
file1.Save();
}
}
using var file1 = TagLib.File.Create(file, "audio/mp4", ReadStyle.Average);
var tag = (AppleTag)file1.GetTag(TagTypes.Apple, true);
tag.Publisher = this.publisher;
tag.LongDescription = this.comments;
tag.Description = this.comments;
file1.Save();
}
public string GenerateFfmpegTags()
{

40
AudibleDotComAutomation/UNTESTED/Page Retrievers/BrowserlessRetriever.cs
View File

@ -94,27 +94,27 @@ var pageSource = new AudiblePageSource(AudiblePageType.Library, html, null);
private static async Task<string> getLibraryPageAsync(CookieContainer cookies, int pageNum)
{
#region // POST example (from 2017 ajax)
// var destination = "https://www.audible.com/lib-ajax";
// var webRequest = (HttpWebRequest)WebRequest.Create(destination);
// webRequest.Method = "POST";
// webRequest.Accept = "*/*";
// webRequest.AllowAutoRedirect = false;
// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)";
// webRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
// webRequest.Credentials = null;
//
// webRequest.CookieContainer = new CookieContainer();
// webRequest.CookieContainer.Add(cookies.GetCookies(new Uri(destination)));
//
// var postData = $"progType=all&timeFilter=all&itemsPerPage={itemsPerPage}&searchTerm=&searchType=&sortColumn=&sortType=down&page={pageNum}&mode=normal&subId=&subTitle=";
// var data = Encoding.UTF8.GetBytes(postData);
// webRequest.ContentLength = data.Length;
// using (var dataStream = webRequest.GetRequestStream())
// dataStream.Write(data, 0, data.Length);
#endregion
#region // POST example (from 2017 ajax)
// var destination = "https://www.audible.com/lib-ajax";
// var webRequest = (HttpWebRequest)WebRequest.Create(destination);
// webRequest.Method = "POST";
// webRequest.Accept = "*/*";
// webRequest.AllowAutoRedirect = false;
// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)";
// webRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
// webRequest.Credentials = null;
//
// webRequest.CookieContainer = new CookieContainer();
// webRequest.CookieContainer.Add(cookies.GetCookies(new Uri(destination)));
//
// var postData = $"progType=all&timeFilter=all&itemsPerPage={itemsPerPage}&searchTerm=&searchType=&sortColumn=&sortType=down&page={pageNum}&mode=normal&subId=&subTitle=";
// var data = Encoding.UTF8.GetBytes(postData);
// webRequest.ContentLength = data.Length;
// using var dataStream = webRequest.GetRequestStream();
// dataStream.Write(data, 0, data.Length);
#endregion
var destination = "https://" + $"www.audible.com/lib?purchaseDateFilter=all&programFilter=all&sortBy=PURCHASE_DATE.dsc&page={pageNum}";
var destination = "https://" + $"www.audible.com/lib?purchaseDateFilter=all&programFilter=all&sortBy=PURCHASE_DATE.dsc&page={pageNum}";
var webRequest = (HttpWebRequest)WebRequest.Create(destination);
webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705)";

79
CookieMonster/UNTESTED/Browsers/Chrome.cs
View File

@ -10,53 +10,50 @@ namespace CookieMonster
{
internal class Chrome : IBrowser
{
public async Task<IEnumerable<CookieValue>> GetAllCookiesAsync()
{
var col = new List<CookieValue>();
public async Task<IEnumerable<CookieValue>> GetAllCookiesAsync()
{
var col = new List<CookieValue>();
var strPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"Google\Chrome\User Data\Default\Cookies");
if (!FileUtility.FileExists(strPath))
return col;
var strPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), @"Google\Chrome\User Data\Default\Cookies");
if (!FileUtility.FileExists(strPath))
return col;
//
// IF WE GET AN ERROR HERE
// then add a reference to sqlite core in the project which is ultimately calling this.
// a project which directly references CookieMonster doesn't need to also ref sqlite.
// however, for any further number of abstractions, the project needs to directly ref sqlite.
// eg: this will not work unless the winforms proj adds sqlite to ref.s:
// LibationWinForm > AudibleDotComAutomation > CookieMonster
//
using (var conn = new SQLiteConnection("Data Source=" + strPath + ";pooling=false"))
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = "SELECT host_key, name, value, encrypted_value, last_access_utc, expires_utc FROM cookies;";
//
// IF WE GET AN ERROR HERE
// then add a reference to sqlite core in the project which is ultimately calling this.
// a project which directly references CookieMonster doesn't need to also ref sqlite.
// however, for any further number of abstractions, the project needs to directly ref sqlite.
// eg: this will not work unless the winforms proj adds sqlite to ref.s:
// LibationWinForm > AudibleDotComAutomation > CookieMonster
//
using var conn = new SQLiteConnection("Data Source=" + strPath + ";pooling=false");
using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT host_key, name, value, encrypted_value, last_access_utc, expires_utc FROM cookies;";
conn.Open();
using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false))
{
while (reader.Read())
{
var host_key = reader.GetString(0);
var name = reader.GetString(1);
var value = reader.GetString(2);
var last_access_utc = reader.GetInt64(4);
var expires_utc = reader.GetInt64(5);
conn.Open();
using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false);
while (reader.Read())
{
var host_key = reader.GetString(0);
var name = reader.GetString(1);
var value = reader.GetString(2);
var last_access_utc = reader.GetInt64(4);
var expires_utc = reader.GetInt64(5);
// https://stackoverflow.com/a/25874366
if (string.IsNullOrWhiteSpace(value))
{
var encrypted_value = (byte[])reader[3];
var decodedData = System.Security.Cryptography.ProtectedData.Unprotect(encrypted_value, null, System.Security.Cryptography.DataProtectionScope.CurrentUser);
value = Encoding.ASCII.GetString(decodedData);
}
// https://stackoverflow.com/a/25874366
if (string.IsNullOrWhiteSpace(value))
{
var encrypted_value = (byte[])reader[3];
var decodedData = System.Security.Cryptography.ProtectedData.Unprotect(encrypted_value, null, System.Security.Cryptography.DataProtectionScope.CurrentUser);
value = Encoding.ASCII.GetString(decodedData);
}
col.Add(new CookieValue { Browser = "chrome", Domain = host_key, Name = name, Value = value, LastAccess = chromeTimeToDateTimeUtc(last_access_utc), Expires = chromeTimeToDateTimeUtc(expires_utc) });
}
}
}
col.Add(new CookieValue { Browser = "chrome", Domain = host_key, Name = name, Value = value, LastAccess = chromeTimeToDateTimeUtc(last_access_utc), Expires = chromeTimeToDateTimeUtc(expires_utc) });
}
return col;
}
return col;
}
// Chrome uses 1601-01-01 00:00:00 UTC as the epoch (ie the starting point for the millisecond time counter).
// this is the same as "FILETIME" in Win32 except FILETIME uses 100ns ticks instead of ms.

34
CookieMonster/UNTESTED/Browsers/FireFox.cs
View File

@ -28,27 +28,23 @@ namespace CookieMonster
File.Copy(strPath, strTemp, true);
// Now open the temporary cookie jar and extract Value from the cookie if we find it.
using (var conn = new SQLiteConnection("Data Source=" + strTemp + ";pooling=false"))
using (var cmd = conn.CreateCommand())
{
cmd.CommandText = "SELECT host, name, value, lastAccessed, expiry FROM moz_cookies; ";
// Now open the temporary cookie jar and extract Value from the cookie if we find it.
using var conn = new SQLiteConnection("Data Source=" + strTemp + ";pooling=false");
using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT host, name, value, lastAccessed, expiry FROM moz_cookies; ";
conn.Open();
using (var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false))
{
while (reader.Read())
{
var host_key = reader.GetString(0);
var name = reader.GetString(1);
var value = reader.GetString(2);
var lastAccessed = reader.GetInt32(3);
var expiry = reader.GetInt32(4);
conn.Open();
using var reader = await cmd.ExecuteReaderAsync().ConfigureAwait(false);
while (reader.Read())
{
var host_key = reader.GetString(0);
var name = reader.GetString(1);
var value = reader.GetString(2);
var lastAccessed = reader.GetInt32(3);
var expiry = reader.GetInt32(4);
col.Add(new CookieValue { Browser = "firefox", Domain = host_key, Name = name, Value = value, LastAccess = lastAccessedToDateTime(lastAccessed), Expires = expiryToDateTime(expiry) });
}
}
}
col.Add(new CookieValue { Browser = "firefox", Domain = host_key, Name = name, Value = value, LastAccess = lastAccessedToDateTime(lastAccessed), Expires = expiryToDateTime(expiry) });
}
if (FileUtility.FileExists(strTemp))
File.Delete(strTemp);

22
DataLayer/UNTESTED/QueryObjects/BookQueries.cs
View File

@ -10,20 +10,20 @@ namespace DataLayer
{
public static int BooksWithoutDetailsCount()
{
using (var context = LibationContext.Create())
return context
.Books
.Count(b => !b.HasBookDetails);
}
using var context = LibationContext.Create();
return context
.Books
.Count(b => !b.HasBookDetails);
}
public static Book GetBook_Flat_NoTracking(string productId)
{
using (var context = LibationContext.Create())
return context
.Books
.AsNoTracking()
.GetBook(productId);
}
using var context = LibationContext.Create();
return context
.Books
.AsNoTracking()
.GetBook(productId);
}
public static Book GetBook(this IQueryable<Book> books, string productId)
=> books

26
DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs
View File

@ -8,22 +8,22 @@ namespace DataLayer
{
public static List<LibraryBook> GetLibrary_Flat_NoTracking()
{
using (var context = LibationContext.Create())
return context
.Library
.AsNoTracking()
.GetLibrary()
.ToList();
}
using var context = LibationContext.Create();
return context
.Library
.AsNoTracking()
.GetLibrary()
.ToList();
}
public static LibraryBook GetLibraryBook_Flat_NoTracking(string productId)
{
using (var context = LibationContext.Create())
return context
.Library
.AsNoTracking()
.GetLibraryBook(productId);
}
using var context = LibationContext.Create();
return context
.Library
.AsNoTracking()
.GetLibraryBook(productId);
}
/// <summary>This is still IQueryable. YOU MUST CALL ToList() YOURSELF</summary>
public static IQueryable<LibraryBook> GetLibrary(this IQueryable<LibraryBook> library)

12
DataLayer/UNTESTED/_scratch pad/ScratchPad.cs
View File

@ -13,13 +13,11 @@ namespace _scratch_pad
// var user = new Student() { Name = "Dinah Cheshire" };
// var udi = new UserDef { UserDefId = 1, TagsRaw = "my,tags" };
// using (var context = new MyTestContextDesignTimeDbContextFactory().Create())
// {
// context.Add(user);
// //context.Add(udi);
// context.Update(udi);
// context.SaveChanges();
// }
// using var context = new MyTestContextDesignTimeDbContextFactory().Create();
// context.Add(user);
// //context.Add(udi);
// context.Update(udi);
// context.SaveChanges();
// Console.WriteLine($"Student was saved in the database with id: {user.Id}");
// }

10
DomainServices/UNTESTED/DownloadBook.cs
View File

@ -43,13 +43,11 @@ namespace DomainServices
.Replace("&DownloadType=Now", "")
+ "&asin=&source=audible_adm&size=&browser_type=&assemble_url=http://cds.audible.com/download";
var uri = new Uri(aaxDownloadLink);
using (var webClient = await GetWebClient(tempAaxFilename))
{
// for book downloads only: pretend to be the audible download manager. from inAudible:
webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601";
await webClient.DownloadFileTaskAsync(uri, tempAaxFilename);
}
using var webClient = await GetWebClient(tempAaxFilename);
// for book downloads only: pretend to be the audible download manager. from inAudible:
webClient.Headers["User-Agent"] = "Audible ADM 6.6.0.15;Windows Vista Service Pack 1 Build 7601";
await webClient.DownloadFileTaskAsync(uri, tempAaxFilename);
// move
var aaxFilename = FileUtility.GetValidFilename(

4
DomainServices/UNTESTED/DownloadPdf.cs
View File

@ -43,8 +43,8 @@ namespace DomainServices
var destinationFilename = Path.Combine(destinationDir, Path.GetFileName(url));
using (var webClient = await GetWebClient(destinationFilename))
await webClient.DownloadFileTaskAsync(url, destinationFilename);
using var webClient = await GetWebClient(destinationFilename);
await webClient.DownloadFileTaskAsync(url, destinationFilename);
var statusHandler = new StatusHandler();
var exists = await AudibleFileStorage.PDF.ExistsAsync(product.AudibleProductId);

68
DomainServices/UNTESTED/Indexer.cs
View File

@ -55,28 +55,25 @@ namespace DomainServices
productItems = filterAndValidate(productItems);
int newEntries;
using (var context = LibationContext.Create())
{
var dtoImporter = new DtoImporter(context);
using var context = LibationContext.Create();
var dtoImporter = new DtoImporter(context);
#region // benchmarks. re-importing a library with 500 books, all with book details json files
/*
dtoImporter.ReplaceLibrary 1.2 seconds
SaveChanges() 3.4
ReloadBookDetails() 1.3
SaveChanges() 1.4
*/
#endregion
// LONG RUNNING
newEntries = await Task.Run(() => dtoImporter.ReplaceLibrary(productItems));
await context.SaveChangesAsync();
#region // benchmarks. re-importing a library with 500 books, all with book details json files
/*
dtoImporter.ReplaceLibrary 1.2 seconds
SaveChanges() 3.4
ReloadBookDetails() 1.3
SaveChanges() 1.4
*/
#endregion
// LONG RUNNING
var newEntries = await Task.Run(() => dtoImporter.ReplaceLibrary(productItems));
await context.SaveChangesAsync();
// must be broken out. see notes in dtoImporter.ReplaceLibrary()
// LONG RUNNING
await Task.Run(() => dtoImporter.ReloadBookDetails(productItems));
await context.SaveChangesAsync();
}
// must be broken out. see notes in dtoImporter.ReplaceLibrary()
// LONG RUNNING
await Task.Run(() => dtoImporter.ReloadBookDetails(productItems));
await context.SaveChangesAsync();
await postIndexActionAsync?.Invoke();
@ -111,13 +108,10 @@ namespace DomainServices
#region update book tags
public static int IndexChangedTags(Book book)
{
// update disconnected entity
int qtyChanges;
using (var context = LibationContext.Create())
{
context.Update(book);
qtyChanges = context.SaveChanges();
}
// update disconnected entity
using var context = LibationContext.Create();
context.Update(book);
var qtyChanges = context.SaveChanges();
// this part is tags-specific
if (qtyChanges > 0)
@ -151,19 +145,15 @@ namespace DomainServices
validate(bookDetailDTO);
using (var context = LibationContext.Create())
{
var dtoImporter = new DtoImporter(context);
// LONG RUNNING
await Task.Run(() => dtoImporter.UpdateBookDetails(bookDetailDTO));
context.SaveChanges();
using var context = LibationContext.Create();
var dtoImporter = new DtoImporter(context);
// LONG RUNNING
await Task.Run(() => dtoImporter.UpdateBookDetails(bookDetailDTO));
context.SaveChanges();
// after saving, delete orphan contributors
var count = context.RemoveOrphans();
if (count > 0)
{
}
}
// after saving, delete orphan contributors
var count = context.RemoveOrphans();
if (count > 0) { } // don't think there's a to-do here
await postIndexActionAsync?.Invoke();
}

54
DomainServices/UNTESTED/ScrapeBookDetails.cs
View File

@ -72,37 +72,35 @@ namespace DomainServices
// download htm
string source;
var url = AudiblePage.Product.GetUrl(productId);
using (var webClient = await GetWebClient($"Getting Book Details for {libraryBook.Book.Title}"))
{
try
{
source = await webClient.DownloadStringTaskAsync(url);
var detailsAudiblePageSource = new AudiblePageSource(AudiblePageType.ProductDetails, source, productId);
using var webClient = await GetWebClient($"Getting Book Details for {libraryBook.Book.Title}");
try
{
source = await webClient.DownloadStringTaskAsync(url);
var detailsAudiblePageSource = new AudiblePageSource(AudiblePageType.ProductDetails, source, productId);
// good habit to persist htm before attempting to parse it. this way, if there's a parse error, we can test errors on a local copy
DataConverter.AudiblePageSource_2_HtmFile_Product(detailsAudiblePageSource);
// good habit to persist htm before attempting to parse it. this way, if there's a parse error, we can test errors on a local copy
DataConverter.AudiblePageSource_2_HtmFile_Product(detailsAudiblePageSource);
bookDetailDTO = AudibleScraper.ScrapeBookDetailsSource(detailsAudiblePageSource);
}
catch (System.Net.WebException webEx)
{
// cannot continue if NoLongerAvailableAction is null,
// else we'll be right back here next loop (and infinitely) with no failure condition
if (webEx.Status != System.Net.WebExceptionStatus.ConnectionClosed || NoLongerAvailableAction == null)
throw;
bookDetailDTO = AudibleScraper.ScrapeBookDetailsSource(detailsAudiblePageSource);
}
catch (System.Net.WebException webEx)
{
// cannot continue if NoLongerAvailableAction is null,
// else we'll be right back here next loop (and infinitely) with no failure condition
if (webEx.Status != System.Net.WebExceptionStatus.ConnectionClosed || NoLongerAvailableAction == null)
throw;
var nlaEnum = NoLongerAvailableAction.Invoke(
libraryBook.Book.Title,
AudiblePage.Product.GetUrl(libraryBook.Book.AudibleProductId));
if (nlaEnum == NoLongerAvailableEnum.Abort)
return new StatusHandler { "Cannot scrape book details. Aborting." };
else if (nlaEnum == NoLongerAvailableEnum.MarkAsMissing)
bookDetailDTO = new BookDetailDTO { ProductId = productId };
else
throw;
}
}
}
var nlaEnum = NoLongerAvailableAction.Invoke(
libraryBook.Book.Title,
AudiblePage.Product.GetUrl(libraryBook.Book.AudibleProductId));
if (nlaEnum == NoLongerAvailableEnum.Abort)
return new StatusHandler { "Cannot scrape book details. Aborting." };
else if (nlaEnum == NoLongerAvailableEnum.MarkAsMissing)
bookDetailDTO = new BookDetailDTO { ProductId = productId };
else
throw;
}
}
DataConverter.Value_2_JsonFile(bookDetailDTO, jsonFileInfo.FullName);
}

58
FileManager/UNTESTED/PictureStorage.cs
View File

@ -39,39 +39,37 @@ namespace FileManager
{
try
{
using (var webClient = new System.Net.WebClient())
{
// download any that don't exist
{
if (!FileUtility.FileExists(path80))
{
var bytes = webClient.DownloadData(
"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL80_.jpg");
File.WriteAllBytes(path80, bytes);
}
}
using var webClient = new System.Net.WebClient();
// download any that don't exist
{
if (!FileUtility.FileExists(path80))
{
var bytes = webClient.DownloadData(
"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL80_.jpg");
File.WriteAllBytes(path80, bytes);
}
}
{
if (!FileUtility.FileExists(path300))
{
var bytes = webClient.DownloadData(
"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL300_.jpg");
File.WriteAllBytes(path300, bytes);
}
}
{
if (!FileUtility.FileExists(path300))
{
var bytes = webClient.DownloadData(
"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL300_.jpg");
File.WriteAllBytes(path300, bytes);
}
}
{
if (!FileUtility.FileExists(path500))
{
var bytes = webClient.DownloadData(
"https://m.media-amazon.com/images/I/" + pictureId + "._SL500_.jpg");
File.WriteAllBytes(path500, bytes);
}
}
{
if (!FileUtility.FileExists(path500))
{
var bytes = webClient.DownloadData(
"https://m.media-amazon.com/images/I/" + pictureId + "._SL500_.jpg");
File.WriteAllBytes(path500, bytes);
}
}
break;
}
}
break;
}
catch { retry++; }
}
while (retry < 3);

139
LibationSearchEngine/UNTESTED/SearchEngine.cs
View File

@ -177,26 +177,19 @@ namespace LibationSearchEngine
log();
// location of index/create the index
using (var index = getIndex())
{
var exists = IndexReader.IndexExists(index);
var createNewIndex = overwrite || !exists;
// location of index/create the index
using var index = getIndex();
var exists = IndexReader.IndexExists(index);
var createNewIndex = overwrite || !exists;
// analyzer for tokenizing text. same analyzer should be used for indexing and searching
using (var analyzer = new StandardAnalyzer(Version))
using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED))
{
foreach (var libraryBook in library)
{
var doc = createBookIndexDocument(libraryBook);
ixWriter.AddDocument(doc);
}
// don't optimize. deprecated: ixWriter.Optimize();
// ixWriter.Commit(); not needed if we're about to dispose of writer anyway. could be needed within the using() block
}
}
// analyzer for tokenizing text. same analyzer should be used for indexing and searching
using var analyzer = new StandardAnalyzer(Version);
using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
foreach (var libraryBook in library)
{
var doc = createBookIndexDocument(libraryBook);
ixWriter.AddDocument(doc);
}
log();
}
@ -247,47 +240,41 @@ namespace LibationSearchEngine
var document = createBookIndexDocument(libraryBook);
var createNewIndex = false;
using (var index = getIndex())
using (var analyzer = new StandardAnalyzer(Version))
using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED))
{
ixWriter.DeleteDocuments(term);
ixWriter.AddDocument(document);
}
}
using var index = getIndex();
using var analyzer = new StandardAnalyzer(Version);
using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
ixWriter.DeleteDocuments(term);
ixWriter.AddDocument(document);
}
public void UpdateTags(string productId, string tags)
{
var productTerm = new Term(_ID_, productId);
using (var index = getIndex())
{
Document document;
using var index = getIndex();
// get existing document
using (var searcher = new IndexSearcher(index))
{
var query = new TermQuery(productTerm);
var docs = searcher.Search(query, 1);
var scoreDoc = docs.ScoreDocs.SingleOrDefault();
if (scoreDoc == null)
throw new Exception("document not found");
document = searcher.Doc(scoreDoc.Doc);
}
// get existing document
using var searcher = new IndexSearcher(index);
var query = new TermQuery(productTerm);
var docs = searcher.Search(query, 1);
var scoreDoc = docs.ScoreDocs.SingleOrDefault();
if (scoreDoc == null)
throw new Exception("document not found");
var document = searcher.Doc(scoreDoc.Doc);
// update document entry with new tags
// fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new
// REMEMBER: all fields, including 'tags' are case-specific
document.RemoveField(TAGS);
document.AddAnalyzed(TAGS, tags);
// update index
var createNewIndex = false;
using (var analyzer = new StandardAnalyzer(Version))
using (var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED))
ixWriter.UpdateDocument(productTerm, document, analyzer);
}
}
// update document entry with new tags
// fields are key value pairs and MULTIPLE FIELDS CAN HAVE THE SAME KEY. must remove old before adding new
// REMEMBER: all fields, including 'tags' are case-specific
document.RemoveField(TAGS);
document.AddAnalyzed(TAGS, tags);
// update index
var createNewIndex = false;
using var analyzer = new StandardAnalyzer(Version);
using var ixWriter = new IndexWriter(index, analyzer, createNewIndex, IndexWriter.MaxFieldLength.UNLIMITED);
ixWriter.UpdateDocument(productTerm, document, analyzer);
}
public SearchResultSet Search(string searchString)
{
@ -381,35 +368,33 @@ namespace LibationSearchEngine
var defaultField = ALL;
using (var index = getIndex())
using (var searcher = new IndexSearcher(index))
using (var analyzer = new StandardAnalyzer(Version))
{
var query = analyzer.GetQuery(defaultField, searchString);
using var index = getIndex();
using var searcher = new IndexSearcher(index);
using var analyzer = new StandardAnalyzer(Version);
var query = analyzer.GetQuery(defaultField, searchString);
// lucene doesn't allow only negations. eg this returns nothing:
// -tags:hidden
// work arounds: https://kb.ucla.edu/articles/pure-negation-query-in-lucene
// HOWEVER, doing this to any other type of query can cause EVERYTHING to be a match unless "Occur" is carefully set
// this should really check that all leaf nodes are MUST_NOT
if (query is BooleanQuery boolQuery)
{
var occurs = getOccurs_recurs(boolQuery);
if (occurs.Any() && occurs.All(o => o == Occur.MUST_NOT))
boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST);
}
// lucene doesn't allow only negations. eg this returns nothing:
// -tags:hidden
// work arounds: https://kb.ucla.edu/articles/pure-negation-query-in-lucene
// HOWEVER, doing this to any other type of query can cause EVERYTHING to be a match unless "Occur" is carefully set
// this should really check that all leaf nodes are MUST_NOT
if (query is BooleanQuery boolQuery)
{
var occurs = getOccurs_recurs(boolQuery);
if (occurs.Any() && occurs.All(o => o == Occur.MUST_NOT))
boolQuery.Add(new MatchAllDocsQuery(), Occur.MUST);
}
Console.WriteLine($" query: {query}");
Console.WriteLine($" query: {query}");
var docs = searcher
.Search(query, MaxSearchResultsToReturn)
.ScoreDocs
.Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score))
.ToList();
return new SearchResultSet(query.ToString(), docs);
}
}
var docs = searcher
.Search(query, MaxSearchResultsToReturn)
.ScoreDocs
.Select(ds => new ScoreDocExplicit(searcher.Doc(ds.Doc), ds.Score))
.ToList();
return new SearchResultSet(query.ToString(), docs);
}
private IEnumerable<Occur> getOccurs_recurs(BooleanQuery query)
{

12
LibationWinForm/UNTESTED/BookLiberation/ProcessorAutomationController.Examples.cs
View File

@ -11,12 +11,12 @@ namespace LibationWinForm.BookLiberation
{
async Task BackupBookAsync(string productId)
{
LibraryBook libraryBook;
using (var context = LibationContext.Create())
libraryBook = context
.Library
.GetLibrary()
.SingleOrDefault(lb => lb.Book.AudibleProductId == productId);
using var context = LibationContext.Create();
var libraryBook = context
.Library
.GetLibrary()
.SingleOrDefault(lb => lb.Book.AudibleProductId == productId);
if (libraryBook == null)
return;

5
LibationWinForm/UNTESTED/Dialogs/IndexDialogs/ScanLibraryDialog.cs
View File

@ -25,9 +25,8 @@ namespace LibationWinForm
public async Task DoMainWorkAsync()
{
List<FileInfo> jsonFilepaths;
using (var pageRetriever = websiteProcessorControl1.GetPageRetriever())
jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
var jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
successMessages.Add($"Downloaded {"library page".PluralizeWithCount(jsonFilepaths.Count)}");

14
REFERENCE.txt
View File

@ -63,18 +63,16 @@ transaction notes
-----------------
// https://msdn.microsoft.com/en-us/data/dn456843.aspx
// Rollback is called by transaction Dispose(). No need to call it explicitly
using (var dbContext = new LibationContext())
using (var dbContextTransaction = dbContext.Database.BeginTransaction())
{
refreshAction(dbContext, productItems);
dbContext.SaveChanges();
dbContextTransaction.Commit();
}
using var dbContext = new LibationContext();
using var dbContextTransaction = dbContext.Database.BeginTransaction();
refreshAction(dbContext, productItems);
dbContext.SaveChanges();
dbContextTransaction.Commit();
aggregate root is transactional boundary
// //context.Database.CurrentTransaction
//var dbTransaction = Microsoft.EntityFrameworkCore.Storage.DbContextTransactionExtensions.GetDbTransaction(context.Database.CurrentTransaction);
// // test with and without : using (TransactionScope scope = new TransactionScope())
// // test with and without : using TransactionScope scope = new TransactionScope();
//System.Transactions.Transaction.Current.TransactionCompleted += (sender, e) => { };
// also : https://docs.microsoft.com/en-us/dotnet/api/system.transactions.transaction.enlistvolatile

2
_DB_NOTES.txt
View File

@ -1,7 +1,7 @@
Logging/Debugging (EF CORE)
===========================
Once you configure logging on a DbContext instance it will be enabled on all instances of that DbContext type
using (var context = new MyContext())
using var context = new MyContext();
context.ConfigureLogging(s => System.Diagnostics.Debug.WriteLine(s)); // write to Visual Studio "Output" tab
//context.ConfigureLogging(s => Console.WriteLine(s));
see comments at top of file:

70
_Demos/ffmpeg decrypt/Form1.cs
View File

@ -23,22 +23,20 @@ namespace ffmpeg_decrypt
private void inpbutton_Click(object sender, EventArgs e)
{
using (var ofd = new OpenFileDialog { Filter = "Audible Audio Files|*.aax", Title = "Select an Audible Audio File", FileName = "" })
using var ofd = new OpenFileDialog { Filter = "Audible Audio Files|*.aax", Title = "Select an Audible Audio File", FileName = "" };
if (ofd.ShowDialog() == DialogResult.OK)
{
if (ofd.ShowDialog() == DialogResult.OK)
{
inputdisplay.Text = ofd.FileName;
outputdisplay.Text = Path.GetDirectoryName(ofd.FileName);
convertbutton.Enabled = true;
}
inputdisplay.Text = ofd.FileName;
outputdisplay.Text = Path.GetDirectoryName(ofd.FileName);
convertbutton.Enabled = true;
}
}
private void outpbutton_Click(object sender, EventArgs e)
{
using (var fbd = new FolderBrowserDialog())
if (fbd.ShowDialog() == DialogResult.OK && !string.IsNullOrWhiteSpace(fbd.SelectedPath))
outputdisplay.Text = fbd.SelectedPath;
using var fbd = new FolderBrowserDialog();
if (fbd.ShowDialog() == DialogResult.OK && !string.IsNullOrWhiteSpace(fbd.SelectedPath))
outputdisplay.Text = fbd.SelectedPath;
}
private async void convertbutton_Click(object sender, EventArgs e)
@ -87,18 +85,15 @@ namespace ffmpeg_decrypt
WorkingDirectory = Directory.GetCurrentDirectory()
};
string ffprobeStderr;
using (var ffp = new Process { StartInfo = startInfo })
{
ffp.Start();
using var ffp = new Process { StartInfo = startInfo };
ffp.Start();
// checksum is in the debug info. ffprobe's debug info is written to stderr, not stdout
ffprobeStderr = ffp.StandardError.ReadToEnd();
// checksum is in the debug info. ffprobe's debug info is written to stderr, not stdout
var ffprobeStderr = ffp.StandardError.ReadToEnd();
await Task.Run(() => ffp.WaitForExit());
await Task.Run(() => ffp.WaitForExit());
ffp.Close();
}
ffp.Close();
// example checksum line:
// ... [aax] file checksum == 0c527840c4f18517157eb0b4f9d6f9317ce60cd1
@ -135,16 +130,13 @@ namespace ffmpeg_decrypt
WorkingDirectory = Directory.GetCurrentDirectory()
};
string rcrackStdout;
using (var rcr = new Process { StartInfo = startInfo })
{
rcr.Start();
using var rcr = new Process { StartInfo = startInfo };
rcr.Start();
rcrackStdout = rcr.StandardOutput.ReadToEnd();
var rcrackStdout = rcr.StandardOutput.ReadToEnd();
await Task.Run(() => rcr.WaitForExit());
rcr.Close();
}
await Task.Run(() => rcr.WaitForExit());
rcr.Close();
// example result
// 0c527840c4f18517157eb0b4f9d6f9317ce60cd1 \xbd\x89X\x09 hex:bd895809
@ -202,15 +194,13 @@ namespace ffmpeg_decrypt
WorkingDirectory = Directory.GetCurrentDirectory()
};
using (var ffm = new Process { StartInfo = startInfo, EnableRaisingEvents = true })
{
ffm.ErrorDataReceived += (s, ea) => debugWindow.UIThread(() => debugWindow.AppendText($"DEBUG: {ea.Data}\r\n"));
using var ffm = new Process { StartInfo = startInfo, EnableRaisingEvents = true };
ffm.ErrorDataReceived += (s, ea) => debugWindow.UIThread(() => debugWindow.AppendText($"DEBUG: {ea.Data}\r\n"));
ffm.Start();
ffm.BeginErrorReadLine();
await Task.Run(() => ffm.WaitForExit());
ffm.Close();
}
ffm.Start();
ffm.BeginErrorReadLine();
await Task.Run(() => ffm.WaitForExit());
ffm.Close();
}
/// <summary>extract embedded resource to file if it doesn't already exist</summary>
@ -224,11 +214,11 @@ namespace ffmpeg_decrypt
// this technique works but there are easier ways:
// https://stackoverflow.com/questions/13031778/how-can-i-extract-a-file-from-an-embedded-resource-and-save-it-to-disk
Directory.CreateDirectory(resdir);
using (var resource = System.Reflection.Assembly.GetCallingAssembly().GetManifestResourceStream($"{nameof(ffmpeg_decrypt)}.res." + resourceName))
using (var reader = new BinaryReader(resource))
using (var file = new FileStream(Path.Combine(resdir, resourceName), FileMode.OpenOrCreate))
using (var writer = new BinaryWriter(file))
writer.Write(reader.ReadBytes((int)resource.Length));
using var resource = System.Reflection.Assembly.GetCallingAssembly().GetManifestResourceStream($"{nameof(ffmpeg_decrypt)}.res." + resourceName);
using var reader = new BinaryReader(resource);
using var file = new FileStream(Path.Combine(resdir, resourceName), FileMode.OpenOrCreate);
using var writer = new BinaryWriter(file);
writer.Write(reader.ReadBytes((int)resource.Length));
}
private void decryptConvertRb_CheckedChanged(object sender, EventArgs e) => convertGb.Enabled = convertRb.Checked;

4
__TODO.txt
View File

@ -11,8 +11,8 @@ incl episodes. eg: "Bill Bryson's Appliance of Science"
replace all scraping with audible api
public partial class ScanLibraryDialog : Form, IIndexLibraryDialog
public async Task DoMainWorkAsync()
using (var pageRetriever = websiteProcessorControl1.GetPageRetriever())
jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
using var pageRetriever = websiteProcessorControl1.GetPageRetriever();
jsonFilepaths = await DownloadLibrary.DownloadLibraryAsync(pageRetriever).ConfigureAwait(false);
break out DTOs. currently coupled with scraping