Fix performance issues, esp regarding saving tags

.gitignore

@@ -328,3 +328,7 @@ ASALocalRun/
 
 # MFractors (Xamarin productivity tool) working folder 
 .mfractor/
+
+
+# manually ignored files
+/__TODO.txt

ApplicationServices/UNTESTED/LibraryCommands.cs

@@ -23,14 +23,12 @@ namespace ApplicationServices
 			return (totalCount, newCount);
 		}
 
-		public static int IndexChangedTags(Book book)
+		public static int UpdateTags(this LibationContext context, Book book, string newTags)
 		{
-			// update disconnected entity
+			book.UserDefinedItem.Tags = newTags;
-			using var context = LibationContext.Create();
+
-			context.Update(book);
 			var qtyChanges = context.SaveChanges();
 
-			// this part is tags-specific
 			if (qtyChanges > 0)
 				SearchEngineCommands.UpdateBookTags(book);
 

ApplicationServices/UNTESTED/SearchEngineCommands.cs

@@ -1,5 +1,4 @@
-using System.Threading.Tasks;
+using DataLayer;
-using DataLayer;
 using LibationSearchEngine;
 
 namespace ApplicationServices

DataLayer/UNTESTED/EfClasses/Book.cs

@@ -61,12 +61,17 @@ namespace DataLayer
             string title,
             string description,
             int lengthInMinutes,
-            IEnumerable<Contributor> authors)
+			IEnumerable<Contributor> authors,
+			IEnumerable<Contributor> narrators)
         {
             // validate
             ArgumentValidator.EnsureNotNull(audibleProductId, nameof(audibleProductId));
             var productId = audibleProductId.Id;
             ArgumentValidator.EnsureNotNullOrWhiteSpace(productId, nameof(productId));
+
+			// assign as soon as possible. stuff below relies on this
+            AudibleProductId = productId;
+
             ArgumentValidator.EnsureNotNullOrWhiteSpace(title, nameof(title));
 
             // non-ef-ctor init.s
@@ -79,19 +84,13 @@ namespace DataLayer
             CategoryId = Category.GetEmpty().CategoryId;
 
             // simple assigns
-            AudibleProductId = productId;
             Title = title;
             Description = description;
             LengthInMinutes = lengthInMinutes;
 
             // assigns with biz logic
             ReplaceAuthors(authors);
-            //ReplaceNarrators(narrators);
+            ReplaceNarrators(narrators);
-
-            // import previously saved tags
-            // do this immediately. any save occurs before reloading tags will overwrite persistent tags with new blank entries; all old persisted tags will be lost
-            // if refactoring, DO NOT use "ProductId" before it's assigned to. to be safe, just use "productId"
-            UserDefinedItem = new UserDefinedItem(this) { Tags = FileManager.TagsPersistence.GetTags(productId) };
         }
 
         #region contributors, authors, narrators

DataLayer/UNTESTED/EfClasses/UserDefinedItem.cs

@@ -13,29 +13,36 @@ namespace DataLayer
 
         private UserDefinedItem() { }
         internal UserDefinedItem(Book book)
-        {
+		{
-            ArgumentValidator.EnsureNotNull(book, nameof(book));
+			ArgumentValidator.EnsureNotNull(book, nameof(book));
             Book = book;
-        }
+
+			// import previously saved tags
+			ArgumentValidator.EnsureNotNullOrWhiteSpace(book.AudibleProductId, nameof(book.AudibleProductId));
+			Tags = FileManager.TagsPersistence.GetTags(book.AudibleProductId);
+		}
 
         private string _tags = "";
         public string Tags
         {
             get => _tags;
             set => _tags = sanitize(value);
-        }
+		}
-        #region sanitize tags: space delimited. Inline/denormalized. Lower case. Alpha numeric and hyphen
+
-        // only legal chars are letters numbers underscores and separating whitespace
+		public IEnumerable<string> TagsEnumerated => Tags == "" ? new string[0] : Tags.Split(null as char[], StringSplitOptions.RemoveEmptyEntries);
-        //
+
-        // technically, the only char.s which aren't easily supported are  \  [  ]
+		#region sanitize tags: space delimited. Inline/denormalized. Lower case. Alpha numeric and hyphen
-        // however, whitelisting is far safer than blacklisting (eg: new lines, non-printable character)
+		// only legal chars are letters numbers underscores and separating whitespace
-        // it's easy to expand whitelist as needed
+		//
-        // for lucene, ToLower() isn't needed because search is case-inspecific. for here, it prevents duplicates
+		// technically, the only char.s which aren't easily supported are  \  [  ]
-        //
+		// however, whitelisting is far safer than blacklisting (eg: new lines, non-printable character)
-        // there are also other allowed but misleading characters. eg: the ^ operator defines a 'boost' score
+		// it's easy to expand whitelist as needed
-        // full list of characters which must be escaped:
+		// for lucene, ToLower() isn't needed because search is case-inspecific. for here, it prevents duplicates
-        //   + - && || ! ( ) { } [ ] ^ " ~ * ? : \
+		//
-        static Regex regex = new Regex(@"[^\w\d\s_]", RegexOptions.Compiled);
+		// there are also other allowed but misleading characters. eg: the ^ operator defines a 'boost' score
+		// full list of characters which must be escaped:
+		//   + - && || ! ( ) { } [ ] ^ " ~ * ? : \
+		static Regex regex { get; } = new Regex(@"[^\w\d\s_]", RegexOptions.Compiled);
         private static string sanitize(string input)
         {
             if (string.IsNullOrWhiteSpace(input))
@@ -63,8 +70,6 @@ namespace DataLayer
 
             return string.Join(" ", unique);
         }
-
-        public IEnumerable<string> TagsEnumerated => Tags == "" ? new string[0] : Tags.Split(null as char[], StringSplitOptions.RemoveEmptyEntries);
         #endregion
 
         // owned: not an optional one-to-one

DataLayer/UNTESTED/QueryObjects/LibraryQueries.cs

@@ -5,13 +5,19 @@ using Microsoft.EntityFrameworkCore;
 namespace DataLayer
 {
     public static class LibraryQueries
-    {
+	{
-        public static List<LibraryBook> GetLibrary_Flat_NoTracking()
+		public static List<LibraryBook> GetLibrary_Flat_WithTracking(this LibationContext context)
+			=> context
+				.Library
+				.GetLibrary()
+				.ToList();
+
+		public static List<LibraryBook> GetLibrary_Flat_NoTracking()
         {
 			using var context = LibationContext.Create();
 			return context
                 .Library
-//.AsNoTracking()
+                .AsNoTracking()
                 .GetLibrary()
                 .ToList();
 		}
@@ -21,7 +27,7 @@ namespace DataLayer
 			using var context = LibationContext.Create();
 			return context
                 .Library
-//.AsNoTracking()
+                .AsNoTracking()
 				.GetLibraryBook(productId);
 		}
 

DataLayer/UNTESTED/TagPersistenceInterceptor.cs

@@ -4,57 +4,35 @@ using System.Linq;
 using Dinah.Core.Collections.Generic;
 using Dinah.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.ChangeTracking;
 
 namespace DataLayer
 {
     internal class TagPersistenceInterceptor : IDbInterceptor
     {
-        public void Executing(DbContext context)
-        {
-            doWork__EFCore(context);
-        }
-
         public void Executed(DbContext context) { }
 
-        static void doWork__EFCore(DbContext context)
+        public void Executing(DbContext context)
-        {
+		{
-            // persist tags:
+			// persist tags:
-            var modifiedEntities = context.ChangeTracker.Entries().Where(p => p.State.In(EntityState.Modified, EntityState.Added)).ToList();
+			var modifiedEntities = context
-            var tagSets = modifiedEntities.Select(e => e.Entity as UserDefinedItem).Where(a => a != null).ToList();
+				.ChangeTracker
-            foreach (var t in tagSets)
+				.Entries()
-                FileManager.TagsPersistence.Save(t.Book.AudibleProductId, t.Tags);
+				.Where(p => p.State.In(EntityState.Modified, EntityState.Added))
-        }
+				.ToList();
 
-        #region // notes: working with proxies, esp EF 6
+			persistTags(modifiedEntities);
-        // EF 6: entities are proxied with lazy loading when collections are virtual
+		}
-        // EF Core: lazy loading is supported in 2.1 (there is a version of lazy loading with proxy-wrapping and a proxy-less version with DI) but not on by default and are not supported here
 
-        //static void doWork_EF6(DbContext context)
+		private static void persistTags(List<EntityEntry> modifiedEntities)
-        //{
+		{
-        //    var modifiedEntities = context.ChangeTracker.Entries().Where(p => p.State == EntityState.Modified).ToList();
+			var tagSets = modifiedEntities
-        //    var unproxiedEntities = modifiedEntities.Select(me => UnProxy(context, me.Entity)).ToList();
+				.Select(e => e.Entity as UserDefinedItem)
-
+				// filter by null but NOT by blank. blank is the valid way to show the absence of tags
-        //    // persist tags
+				.Where(a => a != null)
-        //    var tagSets = unproxiedEntities.Select(ue => ue as UserDefinedItem).Where(a => a != null).ToList();
+				.ToList();
-        //    foreach (var t in tagSets)
+			foreach (var t in tagSets)
-        //        FileManager.TagsPersistence.Save(t.ProductId, t.TagsRaw);
+				FileManager.TagsPersistence.Save(t.Book.AudibleProductId, t.Tags);
-        //}
+		}
-
+	}
-        //// https://stackoverflow.com/a/25774651
-        //private static T UnProxy<T>(DbContext context, T proxyObject) where T : class
-        //{
-        //    // alternative: https://docs.microsoft.com/en-us/ef/ef6/fundamentals/proxies
-        //    var proxyCreationEnabled = context.Configuration.ProxyCreationEnabled;
-        //    try
-        //    {
-        //        context.Configuration.ProxyCreationEnabled = false;
-        //        return context.Entry(proxyObject).CurrentValues.ToObject() as T;
-        //    }
-        //    finally
-        //    {
-        //        context.Configuration.ProxyCreationEnabled = proxyCreationEnabled;
-        //    }
-        //}
-        #endregion
-    }
 }

DtoImporterService/BookImporter.cs

@@ -57,38 +57,40 @@ namespace DtoImporterService
 						.Select(a => context.Contributors.Local.Single(c => a.Name == c.Name))
 						.ToList();
 
+					// if no narrators listed, author is the narrator
+					if (item.Narrators is null || !item.Narrators.Any())
+						item.Narrators = item.Authors;
+					// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
+					var narrators = item
+						.Narrators
+						.Select(n => context.Contributors.Local.Single(c => n.Name == c.Name))
+						.ToList();
+
 					book = context.Books.Add(new Book(
-						new AudibleProductId(item.ProductId), item.Title, item.Description, item.LengthInMinutes, authors))
+						new AudibleProductId(item.ProductId),
-						.Entity;
+						item.Title,
+						item.Description,
+						item.LengthInMinutes,
+						authors,
+						narrators)
+					).Entity;
+
+					var publisherName = item.Publisher;
+					if (!string.IsNullOrWhiteSpace(publisherName))
+					{
+						var publisher = context.Contributors.Local.Single(c => publisherName == c.Name);
+						book.ReplacePublisher(publisher);
+					}
 
 					qtyNew++;
 				}
 
-				// if no narrators listed, author is the narrator
-				if (item.Narrators is null || !item.Narrators.Any())
-					item.Narrators = item.Authors;
-				// nested logic is required so order of names is retained. else, contributors may appear in the order they were inserted into the db
-				var narrators = item
-					.Narrators
-					.Select(n => context.Contributors.Local.Single(c => n.Name == c.Name))
-					.ToList();
-				// not all books have narrators. these will already be using author as narrator. don't undo this
-				if (narrators.Any())
-					book.ReplaceNarrators(narrators);
-
 				// set/update book-specific info which may have changed
 				book.PictureId = item.PictureId;
 				book.UpdateProductRating(item.Product_OverallStars, item.Product_PerformanceStars, item.Product_StoryStars);
 				if (!string.IsNullOrWhiteSpace(item.SupplementUrl))
 					book.AddSupplementDownloadUrl(item.SupplementUrl);
 
-				var publisherName = item.Publisher;
-				if (!string.IsNullOrWhiteSpace(publisherName))
-				{
-					var publisher = context.Contributors.Local.Single(c => publisherName == c.Name);
-					book.ReplacePublisher(publisher);
-				}
-
 				// important to update user-specific info. this will have changed if user has rated/reviewed the book since last library import
 				book.UserDefinedItem.UpdateRating(item.MyUserRating_Overall, item.MyUserRating_Performance, item.MyUserRating_Story);
 

FileManager/FileManager.csproj

@@ -4,6 +4,10 @@
     <TargetFramework>netstandard2.1</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Polly" Version="7.1.1" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\Dinah.Core\Dinah.Core\Dinah.Core.csproj" />
   </ItemGroup>

FileManager/UNTESTED/TagsPersistence.cs

@@ -3,64 +3,57 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Newtonsoft.Json;
+using Polly;
+using Polly.Retry;
 
 namespace FileManager
 {
     /// <summary>
     /// Tags must also be stored in db for search performance. Stored in json file to survive a db reset.
-    /// json is only read when a product is first loaded
+    /// json is only read when a product is first loaded into the db
     /// json is only written to when tags are edited
     /// json access is infrequent and one-off
-    /// all other reads happen against db. No volitile storage
     /// </summary>
     public static class TagsPersistence
     {
-        public static string TagsFile => Path.Combine(Configuration.Instance.LibationFiles, "BookTags.json");
+		private static string TagsFile => Path.Combine(Configuration.Instance.LibationFiles, "BookTags.json");
 
         private static object locker { get; } = new object();
 
+		// if failed, retry only 1 time after a wait of 100 ms
+		// 1st save attempt sometimes fails with
+		//   The requested operation cannot be performed on a file with a user-mapped section open.
+		private static RetryPolicy policy { get; }
+			= Policy.Handle<Exception>()
+			.WaitAndRetry(new[] { TimeSpan.FromMilliseconds(100) });
+
         public static void Save(string productId, string tags)
-			=> System.Threading.Tasks.Task.Run(() => save_fireAndForget(productId, tags));
-
-		private static void save_fireAndForget(string productId, string tags)
 		{
+			ensureCache();
+
+			cache[productId] = tags;
+
 			lock (locker)
-			{
+				policy.Execute(() => File.WriteAllText(TagsFile, JsonConvert.SerializeObject(cache, Formatting.Indented)));
-				// get all
-				var allDictionary = retrieve();
-
-				// update/upsert tag list
-				allDictionary[productId] = tags;
-
-				// re-save:
-				//   this often fails the first time with
-				//     The requested operation cannot be performed on a file with a user-mapped section open.
-				//   2nd immediate attempt failing was rare. So I added sleep. We'll see...
-				void resave() => File.WriteAllText(TagsFile, JsonConvert.SerializeObject(allDictionary, Formatting.Indented));
-				try { resave(); }
-				catch (IOException debugEx)
-				{
-					// 1000 was always reliable but very slow. trying other values
-					var waitMs = 100;
-
-					System.Threading.Thread.Sleep(waitMs);
-					resave();
-				}
-			}
 		}
 
+		private static Dictionary<string, string> cache;
+
 		public static string GetTags(string productId)
         {
-            var dic = retrieve();
+			ensureCache();
-            return dic.ContainsKey(productId) ? dic[productId] : null;
+
+			cache.TryGetValue(productId, out string value);
+			return value;
         }
 
-        private static Dictionary<string, string> retrieve()
+		private static void ensureCache()
-        {
+		{
-            if (!FileUtility.FileExists(TagsFile))
+			if (cache is null)
-                return new Dictionary<string, string>();
+				lock (locker)
-			lock (locker)
+					cache = !FileUtility.FileExists(TagsFile)
-				return JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(TagsFile));
+						? new Dictionary<string, string>()
-        }
+						: JsonConvert.DeserializeObject<Dictionary<string, string>>(File.ReadAllText(TagsFile));
-    }
+		}
+	}
 }

InternalUtilities/InternalUtilities.csproj

@@ -4,10 +4,6 @@
     <TargetFramework>netstandard2.1</TargetFramework>
   </PropertyGroup>
 
-  <ItemGroup>
-    <PackageReference Include="Polly" Version="7.1.1" />
-  </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\..\audible api\AudibleApi\AudibleApi\AudibleApi.csproj" />
     <ProjectReference Include="..\FileManager\FileManager.csproj" />

LibationWinForm/UNTESTED/ProductsGrid.cs

@@ -27,10 +27,15 @@ namespace LibationWinForm
 		public event EventHandler<int> VisibleCountChanged;
 
 		private DataGridView dataGridView;
+		private LibationContext context;
 
-        public ProductsGrid() => InitializeComponent();
+		public ProductsGrid()
+		{
+			InitializeComponent();
+			Disposed += (_, __) => { if (context != null) context.Dispose(); };
+		}
 
-        private bool hasBeenDisplayed = false;
+		private bool hasBeenDisplayed = false;
         public void Display()
         {
             if (hasBeenDisplayed)
@@ -87,10 +92,11 @@ namespace LibationWinForm
             }
 
 
-            //
+			//
-            // transform into sorted GridEntry.s BEFORE binding
+			// transform into sorted GridEntry.s BEFORE binding
-            //
+			//
-            var lib = LibraryQueries.GetLibrary_Flat_NoTracking();
+			context = LibationContext.Create();
+			var lib = context.GetLibrary_Flat_WithTracking();
 
 			// if no data. hide all columns. return
 			if (!lib.Any())
@@ -174,8 +180,8 @@ namespace LibationWinForm
             if (editTagsForm.ShowDialog() != DialogResult.OK)
                 return;
 
-            var qtyChanges = saveChangedTags(liveGridEntry.GetBook(), editTagsForm.NewTags);
+			var qtyChanges = context.UpdateTags(liveGridEntry.GetBook(), editTagsForm.NewTags);
-            if (qtyChanges == 0)
+			if (qtyChanges == 0)
                 return;
 
             // force a re-draw, and re-apply filters
@@ -186,14 +192,6 @@ namespace LibationWinForm
             filter();
         }
 
-        private static int saveChangedTags(Book book, string newTags)
-        {
-            book.UserDefinedItem.Tags = newTags;
-
-            var qtyChanges = LibraryCommands.IndexChangedTags(book);
-            return qtyChanges;
-        }
-
         #region Cell Formatting
         private void replaceFormatted(object sender, DataGridViewCellFormattingEventArgs e)
         {
@@ -213,22 +211,6 @@ namespace LibationWinForm
         }
         #endregion
 
-        public void UpdateRow(string productId)
-        {
-            for (var r = dataGridView.RowCount - 1; r >= 0; r--)
-            {
-                var gridEntry = getGridEntry(r);
-                if (gridEntry.GetBook().AudibleProductId == productId)
-                {
-                    var libBook = LibraryQueries.GetLibraryBook_Flat_NoTracking(productId);
-                    gridEntry.REPLACE_Library_Book(libBook);
-                    dataGridView.InvalidateRow(r);
-
-                    return;
-                }
-            }
-        }
-
         #region filter
         string _filterSearchString;
         private void filter() => Filter(_filterSearchString);