Improve file cache performance and add migration

LibraryCommands.GetCounts hits the file cache hard. The previous cache implementation was linear list, so finding an entry by ID was (n). When you consider that each book may have many files, the number of cache entries could grow to many multiples of the library size.

The new cache uses a dictionary with the ID as its key, and a CacheEntry list as its value.

Source/AppScaffolding/LibationScaffolding.cs

@@ -90,6 +90,7 @@ namespace AppScaffolding
 			Migrations.migrate_to_v6_6_9(config);
             Migrations.migrate_to_v11_5_0(config);
 			Migrations.migrate_to_v11_6_5(config);
+			Migrations.migrate_to_v12_0_1(config);
 		}
 
 		/// <summary>Initialize logging. Wire-up events. Run after migration</summary>
@@ -417,6 +418,82 @@ namespace AppScaffolding
             public List<string> Filters { get; set; } = new();
         }
 
+
+		public static void migrate_to_v12_0_1(Configuration config)
+		{
+#nullable enable
+			//Migrate from version 1 file cache to the dictionary-based version 2 cache
+			const string FILENAME_V1 = "FileLocations.json";
+			const string FILENAME_V2 = "FileLocationsV2.json";
+
+			var jsonFileV1 = Path.Combine(Configuration.Instance.LibationFiles, FILENAME_V1);
+			var jsonFileV2 = Path.Combine(Configuration.Instance.LibationFiles, FILENAME_V2);
+
+			if (!File.Exists(jsonFileV2) && File.Exists(jsonFileV1))
+			{
+				try
+				{
+					//FilePathCache loads the cache in its static constructor,
+					//so perform migration without using FilePathCache.CacheEntry
+					if (JArray.Parse(File.ReadAllText(jsonFileV1)) is not JArray v1Cache || v1Cache.Count == 0)
+						return;
+
+					Dictionary<string, JArray> cache = new();
+
+					//Convert to c# objects to speed up searching by ID inside the iterator 
+					var allItems
+						= v1Cache
+						.Select(i => new
+						{
+							Id = i["Id"]?.Value<string>(),
+							Path = i["Path"]?["Path"]?.Value<string>()
+						}).Where(i => i.Id != null)
+						.ToArray();
+
+					foreach (var id in allItems.Select(i => i.Id).OfType<string>().Distinct())
+					{
+						//Use this opportunity to purge non-existent files and re-classify file types
+						//(due to *.aax files previously not being classified as FileType.AAXC)
+						var items = allItems
+							.Where(i => i.Id == id && File.Exists(i.Path))
+							.Select(i => new JObject
+							{
+								{ "Id", i.Id },
+								{ "FileType", (int)FileTypes.GetFileTypeFromPath(i.Path) },
+								{ "Path", new JObject{ { "Path", i.Path } } }
+							})
+							.ToArray();
+
+						if (items.Length == 0)
+							continue;
+
+						cache[id] = new JArray(items);
+					}
+
+					var cacheJson = new JObject { { "Dictionary", JObject.FromObject(cache) } };
+					var cacheFileText = cacheJson.ToString(Formatting.Indented);
+
+					void migrate()
+					{
+						File.WriteAllText(jsonFileV2, cacheFileText);
+						File.Delete(jsonFileV1);
+					}
+
+					try { migrate(); }
+					catch (IOException)
+					{
+						try { migrate(); }
+						catch (IOException)
+						{
+							migrate();
+						}
+					}
+				}
+				catch { /* eat */ }
+			}
+#nullable restore
+		}
+
 		public static void migrate_to_v11_6_5(Configuration config)
 		{
 			//Settings migration for unsupported sample rates (#1116)

Source/LibationFileManager/FilePathCache.cs

@@ -1,8 +1,8 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using Dinah.Core.Collections.Immutable;
 using FileManager;
 using Newtonsoft.Json;
 
@@ -13,34 +13,31 @@ namespace LibationFileManager
 	{
 		public record CacheEntry(string Id, FileType FileType, LongPath Path);
 
-		private const string FILENAME = "FileLocations.json";
+		private const string FILENAME_V2 = "FileLocationsV2.json";
 
 		public static event EventHandler<CacheEntry>? Inserted;
 		public static event EventHandler<CacheEntry>? Removed;
 
-		private static Cache<CacheEntry> cache { get; } = new Cache<CacheEntry>();
+		private static LongPath jsonFileV2 => Path.Combine(Configuration.Instance.LibationFiles, FILENAME_V2);
 
-		private static LongPath jsonFile => Path.Combine(Configuration.Instance.LibationFiles, FILENAME);
+		private static readonly FileCacheV2<CacheEntry> Cache = new();
 
 		static FilePathCache()
 		{
 			// load json into memory. if file doesn't exist, nothing to do. save() will create if needed
-			if (!File.Exists(jsonFile))
+			if (!File.Exists(jsonFileV2))
 				return;
 
 			try
 			{
-                var list = JsonConvert.DeserializeObject<List<CacheEntry>>(File.ReadAllText(jsonFile));
+				Cache = JsonConvert.DeserializeObject<FileCacheV2<CacheEntry>>(File.ReadAllText(jsonFileV2))
-                if (list is null)
+					?? throw new NullReferenceException("File exists but deserialize is null. This will never happen when file is healthy.");
-                    throw new NullReferenceException("File exists but deserialize is null. This will never happen when file is healthy.");
-
-                cache = new Cache<CacheEntry>(list);
 			}
 			catch (Exception ex)
 			{
-                Serilog.Log.Logger.Error(ex, "Error deserializing file. Wrong format. Possibly corrupt. Deleting file. {@DebugInfo}", new { jsonFile });
+				Serilog.Log.Logger.Error(ex, "Error deserializing file. Wrong format. Possibly corrupt. Deleting file. {@DebugInfo}", new { jsonFileV2 });
 				lock (locker)
-                    File.Delete(jsonFile);
+					File.Delete(jsonFileV2);
 				return;
 			}
 		}
@@ -48,42 +45,63 @@ namespace LibationFileManager
 		public static bool Exists(string id, FileType type) => GetFirstPath(id, type) is not null;
 
 		public static List<(FileType fileType, LongPath path)> GetFiles(string id)
-			=> getEntries(entry => entry.Id == id)
+		{
-			.Select(entry => (entry.FileType, entry.Path))
+			var matchingFiles = Cache.GetIdEntries(id);
-			.ToList();
+
+			bool cacheChanged = false;
+
+			//Verify all entries exist
+			for (int i = 0; i < matchingFiles.Count; i++)
+			{
+				if (!File.Exists(matchingFiles[i].Path))
+				{
+					matchingFiles.RemoveAt(i);
+					cacheChanged |= Remove(matchingFiles[i]);
+				}
+			}
+			if (cacheChanged)
+				save();
+
+			return matchingFiles.Select(e => (e.FileType, e.Path)).ToList();
+		}
 
 		public static LongPath? GetFirstPath(string id, FileType type)
-			=> getEntries(entry => entry.Id == id && entry.FileType == type)
-			?.FirstOrDefault()
-			?.Path;
-
-		private static IEnumerable<CacheEntry> getEntries(Func<CacheEntry, bool> predicate)
 		{
-			var entries = cache.Where(predicate).ToList();
+			var matchingFiles = Cache.GetIdEntries(id).Where(e => e.FileType == type).ToList();
-			if (entries is null || !entries.Any())
-				return Enumerable.Empty<CacheEntry>();
 
-			remove(entries.Where(e => !File.Exists(e.Path)).ToList());
+			bool cacheChanged = false;
-
+			try
-			return cache.Where(predicate).ToList();
+			{
+				//Verify entries exist, but return first matching 'type'
+				for (int i = 0; i < matchingFiles.Count; i++)
+				{
+					if (File.Exists(matchingFiles[i].Path))
+						return matchingFiles[i].Path;
+					else
+					{
+						matchingFiles.RemoveAt(i);
+						cacheChanged |= Remove(matchingFiles[i]);
 					}
-
-		private static void remove(List<CacheEntry> entries)
-		{
-			if (entries is null)
-				return;
-
-			lock (locker)
-			{
-				foreach (var entry in entries)
-				{
-					cache.Remove(entry);
-					Removed?.Invoke(null, entry);
 				}
+				return null;
+			}
+			finally
+			{
+				if (cacheChanged)
 					save();
 			}
 		}
 
+		private static bool Remove(CacheEntry entry)
+		{
+			if (Cache.Remove(entry.Id, entry))
+			{
+				Removed?.Invoke(null, entry);
+				return true;
+			}
+			return false;
+		}
+
 		public static void Insert(string id, string path)
 		{
 			var type = FileTypes.GetFileTypeFromPath(path);
@@ -92,7 +110,7 @@ namespace LibationFileManager
 
 		public static void Insert(CacheEntry entry)
 		{
-			cache.Add(entry);
+			Cache.Add(entry.Id, entry);
 			Inserted?.Invoke(null, entry);
 			save();
 		}
@@ -102,7 +120,7 @@ namespace LibationFileManager
 		private static void save()
 		{
 			// create json if not exists
-			static void resave() => File.WriteAllText(jsonFile, JsonConvert.SerializeObject(cache.ToList(), Formatting.Indented));
+			static void resave() => File.WriteAllText(jsonFileV2, JsonConvert.SerializeObject(Cache, Formatting.Indented));
 
 			lock (locker)
 			{
@@ -112,11 +130,41 @@ namespace LibationFileManager
 					try { resave(); }
 					catch (IOException ex)
 					{
-						Serilog.Log.Logger.Error(ex, $"Error saving {FILENAME}");
+						Serilog.Log.Logger.Error(ex, $"Error saving {FILENAME_V2}");
 						throw;
 					}
 				}
 			}
 		}
+
+		private class FileCacheV2<TEntry>
+		{
+			[JsonProperty]
+			private readonly ConcurrentDictionary<string, List<TEntry>> Dictionary = new();
+
+			public List<TEntry> GetIdEntries(string id)
+			{
+				static List<TEntry> empty() => new();
+
+				return Dictionary.TryGetValue(id, out var entries) ? entries.ToList() : empty();
+			}
+
+			public void Add(string id, TEntry entry)
+			{
+				Dictionary.AddOrUpdate(id, [entry], (id, entries) => { entries.Add(entry); return entries; });
+			}
+
+			public void AddRange(string id, IEnumerable<TEntry> entries)
+			{
+				Dictionary.AddOrUpdate(id, entries.ToList(), (id, entries) =>
+				{
+					entries.AddRange(entries);
+					return entries;
+				});
+			}
+
+			public bool Remove(string id, TEntry entry)
+				=> Dictionary.TryGetValue(id, out List<TEntry>? entries) && entries.Remove(entry);
+		}
 	}
 }

Source/LibationFileManager/FileTypes.cs

@@ -11,6 +11,7 @@ namespace LibationFileManager
 	{
 		private static Dictionary<string, FileType> dic => new()
 		{
+			["aax"] = FileType.AAXC,
 			["aaxc"] = FileType.AAXC,
 			["cue"] = FileType.Cue,
 			["pdf"] = FileType.PDF,