From f5e16673680bafe3a3f044aafcd3f56b33958892 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Tue, 27 Jul 2021 10:13:37 -0600 Subject: [PATCH 1/4] Added a background file system watcher. --- FileManager/AudibleFileStorage.cs | 29 ++++++-- FileManager/BackgroundFileSystem.cs | 107 ++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 FileManager/BackgroundFileSystem.cs diff --git a/FileManager/AudibleFileStorage.cs b/FileManager/AudibleFileStorage.cs index 448f60dd..d2e5c733 100644 --- a/FileManager/AudibleFileStorage.cs +++ b/FileManager/AudibleFileStorage.cs @@ -41,6 +41,8 @@ namespace FileManager return Directory.CreateDirectory(Configuration.Instance.Books).FullName; } } + + private static BackgroundFileSystem BookDirectoryFiles { get; } = new BackgroundFileSystem(); #endregion #region instance @@ -53,7 +55,7 @@ namespace FileManager { extensions_noDots = Extensions.Select(ext => ext.Trim('.')).ToList(); extAggr = extensions_noDots.Aggregate((a, b) => $"{a}|{b}"); - } + } /// /// Example for full books: @@ -63,16 +65,31 @@ namespace FileManager /// public bool Exists(string productId) => GetPath(productId) != null; - public string GetPath(string productId) + public string GetPath(string productId) { var cachedFile = FilePathCache.GetPath(productId, FileType); if (cachedFile != null) return cachedFile; - var firstOrNull = - Directory - .EnumerateFiles(StorageDirectory, "*.*", SearchOption.AllDirectories) - .FirstOrDefault(s => Regex.IsMatch(s, $@"{productId}.*?\.({extAggr})$", RegexOptions.IgnoreCase)); + string storageDir = StorageDirectory; + string regexPattern = $@"{productId}.*?\.({extAggr})$"; + string firstOrNull; + + if (storageDir == BooksDirectory) + { + //If user changed the BooksDirectory, reinitialize. + if (storageDir != BookDirectoryFiles.RootDirectory) + BookDirectoryFiles.Init(storageDir, "*.*", SearchOption.AllDirectories); + + firstOrNull = BookDirectoryFiles.FindFile(regexPattern, RegexOptions.IgnoreCase); + } + else + { + firstOrNull = + Directory + .EnumerateFiles(storageDir, "*.*", SearchOption.AllDirectories) + .FirstOrDefault(s => Regex.IsMatch(s, regexPattern, RegexOptions.IgnoreCase)); + } if (firstOrNull is null) return null; diff --git a/FileManager/BackgroundFileSystem.cs b/FileManager/BackgroundFileSystem.cs new file mode 100644 index 00000000..98430686 --- /dev/null +++ b/FileManager/BackgroundFileSystem.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace FileManager +{ + class BackgroundFileSystem + { + public string RootDirectory { get; private set; } + public string SearchPattern { get; private set; } + public SearchOption SearchOption { get; private set; } + + private FileSystemWatcher fileSystemWatcher { get; set; } + private BlockingCollection directoryChangesEvents { get; set; } + private Task backgroundScanner { get; set; } + private List fsCache { get; set; } + + public string FindFile(string regexPattern, RegexOptions options) => + fsCache.FirstOrDefault(s => Regex.IsMatch(s, regexPattern, options)); + + + public void Init(string rootDirectory, string searchPattern, SearchOption searchOptions) + { + RootDirectory = rootDirectory; + SearchPattern = searchPattern; + SearchOption = searchOptions; + + //Calling CompleteAdding() will cause background scanner to terminate. + directoryChangesEvents?.CompleteAdding(); + fsCache?.Clear(); + directoryChangesEvents?.Dispose(); + fileSystemWatcher?.Dispose(); + + fsCache = Directory.EnumerateFiles(RootDirectory, SearchPattern, SearchOption).ToList(); + + directoryChangesEvents = new BlockingCollection(); + fileSystemWatcher = new FileSystemWatcher(RootDirectory); + fileSystemWatcher.Created += FileSystemWatcher_Changed; + fileSystemWatcher.Deleted += FileSystemWatcher_Changed; + fileSystemWatcher.Renamed += FileSystemWatcher_Changed; + fileSystemWatcher.Error += FileSystemWatcher_Error; + fileSystemWatcher.IncludeSubdirectories = true; + fileSystemWatcher.EnableRaisingEvents = true; + + //Wait for background scanner to terminate before reinitializing. + backgroundScanner?.Wait(); + backgroundScanner = new Task(BackgroundScanner); + backgroundScanner.Start(); + } + + private void FileSystemWatcher_Error(object sender, ErrorEventArgs e) + { + Init(RootDirectory, SearchPattern, SearchOption); + } + + private void FileSystemWatcher_Changed(object sender, FileSystemEventArgs e) + { + directoryChangesEvents.Add(e); + } + + private void UpdateLocalCache(FileSystemEventArgs change) + { + if (change.ChangeType == WatcherChangeTypes.Deleted) + { + RemovePath(change.FullPath); + } + else if (change.ChangeType == WatcherChangeTypes.Created) + { + AddPath(change.FullPath); + } + else if (change.ChangeType == WatcherChangeTypes.Renamed) + { + var renameChange = change as RenamedEventArgs; + + RemovePath(renameChange.OldFullPath); + AddPath(renameChange.FullPath); + } + } + + private void RemovePath(string path) + { + var pathsToRemove = fsCache.Where(p => p.StartsWith(path)).ToArray(); + + foreach (var p in pathsToRemove) + fsCache.Remove(p); + } + + private void AddPath(string path) + { + if (File.GetAttributes(path).HasFlag(FileAttributes.Directory)) + fsCache.AddRange(Directory.EnumerateFiles(path, SearchPattern, SearchOption)); + else + fsCache.Add(path); + } + + private void BackgroundScanner() + { + while (directoryChangesEvents.TryTake(out FileSystemEventArgs change, -1)) + UpdateLocalCache(change); + } + } +} From a77405c63270040b309ea5acf623554ddf6bae75 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Tue, 27 Jul 2021 10:21:17 -0600 Subject: [PATCH 2/4] Make thread safe. --- FileManager/BackgroundFileSystem.cs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/FileManager/BackgroundFileSystem.cs b/FileManager/BackgroundFileSystem.cs index 98430686..e60bc4cb 100644 --- a/FileManager/BackgroundFileSystem.cs +++ b/FileManager/BackgroundFileSystem.cs @@ -20,9 +20,13 @@ namespace FileManager private Task backgroundScanner { get; set; } private List fsCache { get; set; } - public string FindFile(string regexPattern, RegexOptions options) => - fsCache.FirstOrDefault(s => Regex.IsMatch(s, regexPattern, options)); - + public string FindFile(string regexPattern, RegexOptions options) + { + lock (fsCache) + { + return fsCache.FirstOrDefault(s => Regex.IsMatch(s, regexPattern, options)); + } + } public void Init(string rootDirectory, string searchPattern, SearchOption searchOptions) { @@ -63,6 +67,13 @@ namespace FileManager directoryChangesEvents.Add(e); } + #region Background Thread + private void BackgroundScanner() + { + while (directoryChangesEvents.TryTake(out FileSystemEventArgs change, -1)) + UpdateLocalCache(change); + } + private void UpdateLocalCache(FileSystemEventArgs change) { if (change.ChangeType == WatcherChangeTypes.Deleted) @@ -98,10 +109,6 @@ namespace FileManager fsCache.Add(path); } - private void BackgroundScanner() - { - while (directoryChangesEvents.TryTake(out FileSystemEventArgs change, -1)) - UpdateLocalCache(change); - } + #endregion } } From 406aea6eadebf2540bb17d0244c14ed973e46b68 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Tue, 27 Jul 2021 10:23:34 -0600 Subject: [PATCH 3/4] More thread safety. --- FileManager/BackgroundFileSystem.cs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/FileManager/BackgroundFileSystem.cs b/FileManager/BackgroundFileSystem.cs index e60bc4cb..66bb76f7 100644 --- a/FileManager/BackgroundFileSystem.cs +++ b/FileManager/BackgroundFileSystem.cs @@ -95,18 +95,24 @@ namespace FileManager private void RemovePath(string path) { - var pathsToRemove = fsCache.Where(p => p.StartsWith(path)).ToArray(); + lock (fsCache) + { + var pathsToRemove = fsCache.Where(p => p.StartsWith(path)).ToArray(); - foreach (var p in pathsToRemove) - fsCache.Remove(p); + foreach (var p in pathsToRemove) + fsCache.Remove(p); + } } private void AddPath(string path) { - if (File.GetAttributes(path).HasFlag(FileAttributes.Directory)) - fsCache.AddRange(Directory.EnumerateFiles(path, SearchPattern, SearchOption)); - else - fsCache.Add(path); + lock (fsCache) + { + if (File.GetAttributes(path).HasFlag(FileAttributes.Directory)) + fsCache.AddRange(Directory.EnumerateFiles(path, SearchPattern, SearchOption)); + else + fsCache.Add(path); + } } #endregion From f06b04ede4eff5c46f4e239d71f9fd1903dbabe2 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Tue, 27 Jul 2021 10:26:15 -0600 Subject: [PATCH 4/4] Fixed possible race condition. --- FileManager/BackgroundFileSystem.cs | 47 ++++++++++++++--------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/FileManager/BackgroundFileSystem.cs b/FileManager/BackgroundFileSystem.cs index 66bb76f7..1b828999 100644 --- a/FileManager/BackgroundFileSystem.cs +++ b/FileManager/BackgroundFileSystem.cs @@ -76,43 +76,40 @@ namespace FileManager private void UpdateLocalCache(FileSystemEventArgs change) { - if (change.ChangeType == WatcherChangeTypes.Deleted) + lock (fsCache) { - RemovePath(change.FullPath); - } - else if (change.ChangeType == WatcherChangeTypes.Created) - { - AddPath(change.FullPath); - } - else if (change.ChangeType == WatcherChangeTypes.Renamed) - { - var renameChange = change as RenamedEventArgs; + if (change.ChangeType == WatcherChangeTypes.Deleted) + { + RemovePath(change.FullPath); + } + else if (change.ChangeType == WatcherChangeTypes.Created) + { + AddPath(change.FullPath); + } + else if (change.ChangeType == WatcherChangeTypes.Renamed) + { + var renameChange = change as RenamedEventArgs; - RemovePath(renameChange.OldFullPath); - AddPath(renameChange.FullPath); + RemovePath(renameChange.OldFullPath); + AddPath(renameChange.FullPath); + } } } private void RemovePath(string path) { - lock (fsCache) - { - var pathsToRemove = fsCache.Where(p => p.StartsWith(path)).ToArray(); + var pathsToRemove = fsCache.Where(p => p.StartsWith(path)).ToArray(); - foreach (var p in pathsToRemove) - fsCache.Remove(p); - } + foreach (var p in pathsToRemove) + fsCache.Remove(p); } private void AddPath(string path) { - lock (fsCache) - { - if (File.GetAttributes(path).HasFlag(FileAttributes.Directory)) - fsCache.AddRange(Directory.EnumerateFiles(path, SearchPattern, SearchOption)); - else - fsCache.Add(path); - } + if (File.GetAttributes(path).HasFlag(FileAttributes.Directory)) + fsCache.AddRange(Directory.EnumerateFiles(path, SearchPattern, SearchOption)); + else + fsCache.Add(path); } #endregion