Improve file cache performance and add migration
LibraryCommands.GetCounts hits the file cache hard. The previous cache implementation was linear list, so finding an entry by ID was (n). When you consider that each book may have many files, the number of cache entries could grow to many multiples of the library size. The new cache uses a dictionary with the ID as its key, and a CacheEntry list as its value.
This commit is contained in:
MBucari
Michael Bucari-Tovo
parent
47c9fcb883
commit
a9375f1520
@ -90,6 +90,7 @@ namespace AppScaffolding
|
||||
Migrations.migrate_to_v6_6_9(config);
|
||||
Migrations.migrate_to_v11_5_0(config);
|
||||
Migrations.migrate_to_v11_6_5(config);
|
||||
Migrations.migrate_to_v12_0_1(config);
|
||||
}
|
||||
|
||||
/// <summary>Initialize logging. Wire-up events. Run after migration</summary>
|
||||
@ -417,6 +418,82 @@ namespace AppScaffolding
|
||||
public List<string> Filters { get; set; } = new();
|
||||
}
|
||||
|
||||
|
||||
public static void migrate_to_v12_0_1(Configuration config)
|
||||
{
|
||||
#nullable enable
|
||||
//Migrate from version 1 file cache to the dictionary-based version 2 cache
|
||||
const string FILENAME_V1 = "FileLocations.json";
|
||||
const string FILENAME_V2 = "FileLocationsV2.json";
|
||||
|
||||
var jsonFileV1 = Path.Combine(Configuration.Instance.LibationFiles, FILENAME_V1);
|
||||
var jsonFileV2 = Path.Combine(Configuration.Instance.LibationFiles, FILENAME_V2);
|
||||
|
||||
if (!File.Exists(jsonFileV2) && File.Exists(jsonFileV1))
|
||||
{
|
||||
try
|
||||
{
|
||||
//FilePathCache loads the cache in its static constructor,
|
||||
//so perform migration without using FilePathCache.CacheEntry
|
||||
if (JArray.Parse(File.ReadAllText(jsonFileV1)) is not JArray v1Cache || v1Cache.Count == 0)
|
||||
return;
|
||||
|
||||
Dictionary<string, JArray> cache = new();
|
||||
|
||||
//Convert to c# objects to speed up searching by ID inside the iterator
|
||||
var allItems
|
||||
= v1Cache
|
||||
.Select(i => new
|
||||
{
|
||||
Id = i["Id"]?.Value<string>(),
|
||||
Path = i["Path"]?["Path"]?.Value<string>()
|
||||
}).Where(i => i.Id != null)
|
||||
.ToArray();
|
||||
|
||||
foreach (var id in allItems.Select(i => i.Id).OfType<string>().Distinct())
|
||||
{
|
||||
//Use this opportunity to purge non-existent files and re-classify file types
|
||||
//(due to *.aax files previously not being classified as FileType.AAXC)
|
||||
var items = allItems
|
||||
.Where(i => i.Id == id && File.Exists(i.Path))
|
||||
.Select(i => new JObject
|
||||
{
|
||||
{ "Id", i.Id },
|
||||
{ "FileType", (int)FileTypes.GetFileTypeFromPath(i.Path) },
|
||||
{ "Path", new JObject{ { "Path", i.Path } } }
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
if (items.Length == 0)
|
||||
continue;
|
||||
|
||||
cache[id] = new JArray(items);
|
||||
}
|
||||
|
||||
var cacheJson = new JObject { { "Dictionary", JObject.FromObject(cache) } };
|
||||
var cacheFileText = cacheJson.ToString(Formatting.Indented);
|
||||
|
||||
void migrate()
|
||||
{
|
||||
File.WriteAllText(jsonFileV2, cacheFileText);
|
||||
File.Delete(jsonFileV1);
|
||||
}
|
||||
|
||||
try { migrate(); }
|
||||
catch (IOException)
|
||||
{
|
||||
try { migrate(); }
|
||||
catch (IOException)
|
||||
{
|
||||
migrate();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch { /* eat */ }
|
||||
}
|
||||
#nullable restore
|
||||
}
|
||||
|
||||
public static void migrate_to_v11_6_5(Configuration config)
|
||||
{
|
||||
//Settings migration for unsupported sample rates (#1116)
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Dinah.Core.Collections.Immutable;
|
||||
using FileManager;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
@ -13,78 +13,96 @@ namespace LibationFileManager
|
||||
{
|
||||
public record CacheEntry(string Id, FileType FileType, LongPath Path);
|
||||
|
||||
private const string FILENAME = "FileLocations.json";
|
||||
private const string FILENAME_V2 = "FileLocationsV2.json";
|
||||
|
||||
public static event EventHandler<CacheEntry>? Inserted;
|
||||
public static event EventHandler<CacheEntry>? Removed;
|
||||
|
||||
private static Cache<CacheEntry> cache { get; } = new Cache<CacheEntry>();
|
||||
private static LongPath jsonFileV2 => Path.Combine(Configuration.Instance.LibationFiles, FILENAME_V2);
|
||||
|
||||
private static LongPath jsonFile => Path.Combine(Configuration.Instance.LibationFiles, FILENAME);
|
||||
private static readonly FileCacheV2<CacheEntry> Cache = new();
|
||||
|
||||
static FilePathCache()
|
||||
{
|
||||
static FilePathCache()
|
||||
{
|
||||
// load json into memory. if file doesn't exist, nothing to do. save() will create if needed
|
||||
if (!File.Exists(jsonFile))
|
||||
if (!File.Exists(jsonFileV2))
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
var list = JsonConvert.DeserializeObject<List<CacheEntry>>(File.ReadAllText(jsonFile));
|
||||
if (list is null)
|
||||
throw new NullReferenceException("File exists but deserialize is null. This will never happen when file is healthy.");
|
||||
|
||||
cache = new Cache<CacheEntry>(list);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Logger.Error(ex, "Error deserializing file. Wrong format. Possibly corrupt. Deleting file. {@DebugInfo}", new { jsonFile });
|
||||
lock (locker)
|
||||
File.Delete(jsonFile);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
public static bool Exists(string id, FileType type) => GetFirstPath(id, type) is not null;
|
||||
|
||||
public static List<(FileType fileType, LongPath path)> GetFiles(string id)
|
||||
=> getEntries(entry => entry.Id == id)
|
||||
.Select(entry => (entry.FileType, entry.Path))
|
||||
.ToList();
|
||||
|
||||
public static LongPath? GetFirstPath(string id, FileType type)
|
||||
=> getEntries(entry => entry.Id == id && entry.FileType == type)
|
||||
?.FirstOrDefault()
|
||||
?.Path;
|
||||
|
||||
private static IEnumerable<CacheEntry> getEntries(Func<CacheEntry, bool> predicate)
|
||||
{
|
||||
var entries = cache.Where(predicate).ToList();
|
||||
if (entries is null || !entries.Any())
|
||||
return Enumerable.Empty<CacheEntry>();
|
||||
|
||||
remove(entries.Where(e => !File.Exists(e.Path)).ToList());
|
||||
|
||||
return cache.Where(predicate).ToList();
|
||||
}
|
||||
|
||||
private static void remove(List<CacheEntry> entries)
|
||||
{
|
||||
if (entries is null)
|
||||
return;
|
||||
|
||||
lock (locker)
|
||||
try
|
||||
{
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
cache.Remove(entry);
|
||||
Removed?.Invoke(null, entry);
|
||||
}
|
||||
save();
|
||||
Cache = JsonConvert.DeserializeObject<FileCacheV2<CacheEntry>>(File.ReadAllText(jsonFileV2))
|
||||
?? throw new NullReferenceException("File exists but deserialize is null. This will never happen when file is healthy.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Serilog.Log.Logger.Error(ex, "Error deserializing file. Wrong format. Possibly corrupt. Deleting file. {@DebugInfo}", new { jsonFileV2 });
|
||||
lock (locker)
|
||||
File.Delete(jsonFileV2);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
public static void Insert(string id, string path)
|
||||
public static bool Exists(string id, FileType type) => GetFirstPath(id, type) is not null;
|
||||
|
||||
public static List<(FileType fileType, LongPath path)> GetFiles(string id)
|
||||
{
|
||||
var matchingFiles = Cache.GetIdEntries(id);
|
||||
|
||||
bool cacheChanged = false;
|
||||
|
||||
//Verify all entries exist
|
||||
for (int i = 0; i < matchingFiles.Count; i++)
|
||||
{
|
||||
if (!File.Exists(matchingFiles[i].Path))
|
||||
{
|
||||
matchingFiles.RemoveAt(i);
|
||||
cacheChanged |= Remove(matchingFiles[i]);
|
||||
}
|
||||
}
|
||||
if (cacheChanged)
|
||||
save();
|
||||
|
||||
return matchingFiles.Select(e => (e.FileType, e.Path)).ToList();
|
||||
}
|
||||
|
||||
public static LongPath? GetFirstPath(string id, FileType type)
|
||||
{
|
||||
var matchingFiles = Cache.GetIdEntries(id).Where(e => e.FileType == type).ToList();
|
||||
|
||||
bool cacheChanged = false;
|
||||
try
|
||||
{
|
||||
//Verify entries exist, but return first matching 'type'
|
||||
for (int i = 0; i < matchingFiles.Count; i++)
|
||||
{
|
||||
if (File.Exists(matchingFiles[i].Path))
|
||||
return matchingFiles[i].Path;
|
||||
else
|
||||
{
|
||||
matchingFiles.RemoveAt(i);
|
||||
cacheChanged |= Remove(matchingFiles[i]);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (cacheChanged)
|
||||
save();
|
||||
}
|
||||
}
|
||||
|
||||
private static bool Remove(CacheEntry entry)
|
||||
{
|
||||
if (Cache.Remove(entry.Id, entry))
|
||||
{
|
||||
Removed?.Invoke(null, entry);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static void Insert(string id, string path)
|
||||
{
|
||||
var type = FileTypes.GetFileTypeFromPath(path);
|
||||
Insert(new CacheEntry(id, type, path));
|
||||
@ -92,7 +110,7 @@ namespace LibationFileManager
|
||||
|
||||
public static void Insert(CacheEntry entry)
|
||||
{
|
||||
cache.Add(entry);
|
||||
Cache.Add(entry.Id, entry);
|
||||
Inserted?.Invoke(null, entry);
|
||||
save();
|
||||
}
|
||||
@ -102,7 +120,7 @@ namespace LibationFileManager
|
||||
private static void save()
|
||||
{
|
||||
// create json if not exists
|
||||
static void resave() => File.WriteAllText(jsonFile, JsonConvert.SerializeObject(cache.ToList(), Formatting.Indented));
|
||||
static void resave() => File.WriteAllText(jsonFileV2, JsonConvert.SerializeObject(Cache, Formatting.Indented));
|
||||
|
||||
lock (locker)
|
||||
{
|
||||
@ -112,11 +130,41 @@ namespace LibationFileManager
|
||||
try { resave(); }
|
||||
catch (IOException ex)
|
||||
{
|
||||
Serilog.Log.Logger.Error(ex, $"Error saving {FILENAME}");
|
||||
Serilog.Log.Logger.Error(ex, $"Error saving {FILENAME_V2}");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class FileCacheV2<TEntry>
|
||||
{
|
||||
[JsonProperty]
|
||||
private readonly ConcurrentDictionary<string, List<TEntry>> Dictionary = new();
|
||||
|
||||
public List<TEntry> GetIdEntries(string id)
|
||||
{
|
||||
static List<TEntry> empty() => new();
|
||||
|
||||
return Dictionary.TryGetValue(id, out var entries) ? entries.ToList() : empty();
|
||||
}
|
||||
|
||||
public void Add(string id, TEntry entry)
|
||||
{
|
||||
Dictionary.AddOrUpdate(id, [entry], (id, entries) => { entries.Add(entry); return entries; });
|
||||
}
|
||||
|
||||
public void AddRange(string id, IEnumerable<TEntry> entries)
|
||||
{
|
||||
Dictionary.AddOrUpdate(id, entries.ToList(), (id, entries) =>
|
||||
{
|
||||
entries.AddRange(entries);
|
||||
return entries;
|
||||
});
|
||||
}
|
||||
|
||||
public bool Remove(string id, TEntry entry)
|
||||
=> Dictionary.TryGetValue(id, out List<TEntry>? entries) && entries.Remove(entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -11,6 +11,7 @@ namespace LibationFileManager
|
||||
{
|
||||
private static Dictionary<string, FileType> dic => new()
|
||||
{
|
||||
["aax"] = FileType.AAXC,
|
||||
["aaxc"] = FileType.AAXC,
|
||||
["cue"] = FileType.Cue,
|
||||
["pdf"] = FileType.PDF,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user