Merge pull request #526 from Mbucari/master

Add better AYCL detection and add verbose library scan logging

Source/ApplicationServices/LibraryCommands.cs

@@ -7,8 +7,11 @@ using AudibleApi;
 using AudibleUtilities;
 using DataLayer;
 using Dinah.Core;
+using Dinah.Core.Logging;
 using DtoImporterService;
+using FileManager;
 using LibationFileManager;
+using Newtonsoft.Json.Linq;
 using Serilog;
 using static DtoImporterService.PerfLogger;
 
@@ -169,13 +172,21 @@ namespace ApplicationServices
         private static async Task<List<ImportItem>> scanAccountsAsync(Func<Account, Task<ApiExtended>> apiExtendedfunc, Account[] accounts, LibraryOptions libraryOptions)
         {
             var tasks = new List<Task<List<ImportItem>>>();
-            foreach (var account in accounts)
+
+            using LogArchiver archiver
+                = Log.Logger.IsDebugEnabled()
+                ? new LogArchiver(System.IO.Path.Combine(Configuration.Instance.LibationFiles, "LibraryScans.zip"))
+                : default;
+
+			archiver?.DeleteAllButNewestN(20);
+
+			foreach (var account in accounts)
             {
                 // get APIs in serial b/c of logins. do NOT move inside of parallel (Task.WhenAll)
                 var apiExtended = await apiExtendedfunc(account);
 
                 // add scanAccountAsync as a TASK: do not await
-                tasks.Add(scanAccountAsync(apiExtended, account, libraryOptions));
+                tasks.Add(scanAccountAsync(apiExtended, account, libraryOptions, archiver));
             }
 
             // import library in parallel
@@ -184,7 +195,7 @@ namespace ApplicationServices
             return importItems;
         }
 
-        private static async Task<List<ImportItem>> scanAccountAsync(ApiExtended apiExtended, Account account, LibraryOptions libraryOptions)
+        private static async Task<List<ImportItem>> scanAccountAsync(ApiExtended apiExtended, Account account, LibraryOptions libraryOptions, LogArchiver archiver)
         {
             ArgumentValidator.EnsureNotNull(account, nameof(account));
 
@@ -197,6 +208,8 @@ namespace ApplicationServices
 
             var dtoItems = await apiExtended.GetLibraryValidatedAsync(libraryOptions, Configuration.Instance.ImportEpisodes);
 
+            archiver?.AddFile($"{DateTime.Now:u} {account.MaskedLogEntry}.json", new JObject { { "Account", account.MaskedLogEntry }, { "ScannedDateTime", DateTime.Now.ToString("u") }, {"Items", JArray.FromObject(dtoItems) } });
+
             logTime($"post scanAccountAsync {account.AccountName} qty: {dtoItems.Count}");
 
             return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = account.Locale?.Name }).ToList();

Source/AudibleUtilities/ApiExtended.cs

@@ -135,14 +135,6 @@ namespace AudibleUtilities
 			Serilog.Log.Logger.Debug("Episode scan complete. Found {count} episodes and series.", count);
 			Serilog.Log.Logger.Debug($"Completed library scan in {sw.Elapsed.TotalMilliseconds:F0} ms.");
 
-#if DEBUG
-			//// this will not work for multi accounts
-			//var library_json = "library.json";
-			//library_json = System.IO.Path.GetFullPath(library_json);
-			//if (System.IO.File.Exists(library_json))
-			//    items = AudibleApi.Common.Converter.FromJson<List<Item>>(System.IO.File.ReadAllText(library_json));
-			//System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
-#endif
 			var validators = new List<IValidator>();
 			validators.AddRange(getValidators());
 			foreach (var v in validators)

Source/DataLayer/EfClasses/Rating.cs

@@ -5,7 +5,7 @@ using Dinah.Core;
 namespace DataLayer
 {
     /// <summary>Parameterless ctor and setters should be used by EF only. Everything else should treat it as immutable</summary>
-    public class Rating : ValueObject_Static<Rating>
+    public class Rating : ValueObject_Static<Rating>, IComparable<Rating>, IComparable
     {
         public float OverallRating { get; private set; }
         public float PerformanceRating { get; private set; }
@@ -38,6 +38,16 @@ namespace DataLayer
             yield return StoryRating;
         }
 
-		public override string ToString() => $"Overall={OverallRating} Perf={PerformanceRating} Story={StoryRating}";
+        public override string ToString() => $"Overall={OverallRating} Perf={PerformanceRating} Story={StoryRating}";
-	}
+
+        public int CompareTo(Rating other)
+        {
+            var compare = OverallRating.CompareTo(other.OverallRating);
+            if (compare != 0) return compare;
+            compare = PerformanceRating.CompareTo(other.PerformanceRating);
+            if (compare != 0) return compare;
+            return StoryRating.CompareTo(other.StoryRating);
+        }
+        public int CompareTo(object obj) => obj is Rating second ? CompareTo(second) : -1;
+    }
 }

Source/DataLayer/EfClasses/UserDefinedItem.cs

@@ -122,7 +122,11 @@ namespace DataLayer
         public Rating Rating { get; private set; } = new Rating(0, 0, 0);
 
         public void UpdateRating(float overallRating, float performanceRating, float storyRating)
-            => Rating.Update(overallRating, performanceRating, storyRating);
+        {
+            var changed = Rating.OverallRating != overallRating || Rating.PerformanceRating != performanceRating || Rating.StoryRating != storyRating;
+			Rating.Update(overallRating, performanceRating, storyRating);
+            if (changed) OnItemChanged(nameof(Rating));
+		}
         #endregion
 
         #region LiberatedStatuses

Source/DtoImporterService/LibraryBookImporter.cs

@@ -70,8 +70,11 @@ namespace DtoImporterService
 				}
 			}
 
+			var scannedAccounts = importItems.Select(i => i.AccountId).Distinct().ToList();
+
 			//If an existing Book wasn't found in the import, the owning LibraryBook's Book will be null.
-			foreach (var nullBook in DbContext.LibraryBooks.AsEnumerable().Where(lb => lb.Book is null))
+			//Only change AbsentFromLastScan for LibraryBooks of accounts that were scanned.
+			foreach (var nullBook in DbContext.LibraryBooks.AsEnumerable().Where(lb => lb.Book is null && lb.Account.In(scannedAccounts)))
 				nullBook.AbsentFromLastScan = true;
 
 			//Join importItems on LibraryBooks before iterating over LibraryBooks to avoid
@@ -88,12 +91,29 @@ namespace DtoImporterService
 			return qtyNew;
 		}
 
+		/*
+		 * Subscription Plan Names:
+		 * 
+		 * US:	"SpecialBenefit"
+		 * IT:	"Rodizio"
+		 * 
+		 * Audible Plus Plan Names:
+		 * 
+		 * US:	"US Minerva"
+		 * IT:	"Audible-AYCL" 
+		 *
+		 */
+
 
 		//This SEEMS to work to detect plus titles which are no longer available.
 		//I have my doubts it won't yield false negatives, but I have more
 		//confidence that it won't yield many/any false positives.
 		private static bool isPlusTitleUnavailable(ImportItem item)
 			=> item.DtoItem.IsAyce is true
-			&& item.DtoItem.Plans?.Any(p => p.PlanName.ContainsInsensitive("Minerva") || p.PlanName.ContainsInsensitive("Free")) is not true;
+			&& item.DtoItem.Plans?.Any(p =>
+				p.PlanName.ContainsInsensitive("Minerva") ||
+				p.PlanName.ContainsInsensitive("AYCL") ||
+				p.PlanName.ContainsInsensitive("Free")
+			) is not true;
 	}
 }

Source/FileManager/LogArchiver.cs

@@ -0,0 +1,67 @@
+using Dinah.Core;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Text;
+
+namespace FileManager
+{
+	public class LogArchiver : IDisposable
+	{
+		public Encoding Encoding { get; set; }
+		public string FileName { get; }
+		private readonly ZipArchive archive;
+
+		public LogArchiver(string filename) : this(filename, Encoding.UTF8) { }
+		public LogArchiver(string filename, Encoding encoding)
+		{
+			FileName = ArgumentValidator.EnsureNotNull(filename, nameof(filename));
+			Encoding = ArgumentValidator.EnsureNotNull(encoding, nameof(encoding));
+			archive = new ZipArchive(File.Open(FileName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite), ZipArchiveMode.Update, false, Encoding);
+		}
+
+		public void DeleteOlderThan(DateTime cutoffDate)
+			=> DeleteEntries(archive.Entries.Where(e => e.LastWriteTime < cutoffDate).ToList());
+
+		public void DeleteOldestN(int quantity)
+			=> DeleteEntries(archive.Entries.OrderBy(e => e.LastWriteTime).Take(quantity).ToList());
+		
+		public void DeleteAllButNewestN(int quantity)
+			=> DeleteEntries(archive.Entries.OrderByDescending(e => e.LastWriteTime).Skip(quantity).ToList());
+		
+		private void DeleteEntries(List<ZipArchiveEntry> entries)
+		{
+			foreach (var e in entries)
+				e.Delete();
+		}
+
+		public void AddFile(string name, JObject contents, string comment = null)
+			=> AddFile(name, Encoding.GetBytes(contents.ToString(Newtonsoft.Json.Formatting.Indented)), comment);
+
+		public void AddFile(string name, string contents, string comment = null)
+			=> AddFile(name, Encoding.GetBytes(contents), comment);
+
+		private readonly object lockOob = new();
+
+		public void AddFile(string name, ReadOnlySpan<byte> contents, string comment = null)
+		{
+			ArgumentValidator.EnsureNotNull(name, nameof(name));
+
+			name = ReplacementCharacters.Barebones.ReplaceFilenameChars(name);
+
+			lock (lockOob)
+			{
+				var entry = archive.CreateEntry(name, CompressionLevel.SmallestSize);
+
+				entry.Comment = comment;
+				using var entryStream = entry.Open();
+				entryStream.Write(contents);
+			}
+		}
+
+		public void Dispose() => archive.Dispose();
+	}
+}