From a3844a35356ab91e9bd610faca4c72d8e20c73b9 Mon Sep 17 00:00:00 2001 From: Michael Bucari-Tovo Date: Fri, 17 Jun 2022 23:23:09 -0600 Subject: [PATCH] Add long path support --- Source/FileLiberator/ConvertToMp3.cs | 2 +- Source/FileManager/BackgroundFileSystem.cs | 20 ++-- Source/FileManager/FileNamingTemplate.cs | 6 +- Source/FileManager/FileUtility.cs | 50 +++++----- Source/FileManager/LongPath.cs | 92 +++++++++++++++++++ .../LibationFileManager/AudibleFileStorage.cs | 17 ++-- Source/LibationFileManager/Configuration.cs | 4 +- Source/LibationFileManager/FilePathCache.cs | 11 ++- Source/LibationFileManager/Templates.cs | 4 +- .../Dialogs/MessageBoxAlertAdminDialog.cs | 5 +- .../Dialogs/SettingsDialog.cs | 4 +- Source/LibationWinForms/Form1.ProcessQueue.cs | 2 +- 12 files changed, 160 insertions(+), 57 deletions(-) create mode 100644 Source/FileManager/LongPath.cs diff --git a/Source/FileLiberator/ConvertToMp3.cs b/Source/FileLiberator/ConvertToMp3.cs index 6cf99936..92b2a3f2 100644 --- a/Source/FileLiberator/ConvertToMp3.cs +++ b/Source/FileLiberator/ConvertToMp3.cs @@ -27,7 +27,7 @@ namespace FileLiberator public static bool ValidateMp3(LibraryBook libraryBook) { var path = AudibleFileStorage.Audio.GetPath(libraryBook.Book.AudibleProductId); - return path?.ToLower()?.EndsWith(".m4b") == true && !File.Exists(Mp3FileName(path)); + return path?.ToString()?.ToLower()?.EndsWith(".m4b") == true && !File.Exists(Mp3FileName(path)); } public override bool Validate(LibraryBook libraryBook) => ValidateMp3(libraryBook); diff --git a/Source/FileManager/BackgroundFileSystem.cs b/Source/FileManager/BackgroundFileSystem.cs index e4b2b10b..c3503e2d 100644 --- a/Source/FileManager/BackgroundFileSystem.cs +++ b/Source/FileManager/BackgroundFileSystem.cs @@ -12,7 +12,7 @@ namespace FileManager /// public class BackgroundFileSystem { - public string RootDirectory { get; private set; } + public LongPath RootDirectory { get; private set; } public string SearchPattern { get; private set; } public SearchOption SearchOption { get; private set; } @@ -21,9 +21,9 @@ namespace FileManager private Task backgroundScanner { get; set; } private object fsCacheLocker { get; } = new(); - private List fsCache { get; } = new(); + private List fsCache { get; } = new(); - public BackgroundFileSystem(string rootDirectory, string searchPattern, SearchOption searchOptions) + public BackgroundFileSystem(LongPath rootDirectory, string searchPattern, SearchOption searchOptions) { RootDirectory = rootDirectory; SearchPattern = searchPattern; @@ -124,16 +124,18 @@ namespace FileManager } } - private void RemovePath(string path) + private void RemovePath(LongPath path) { - var pathsToRemove = fsCache.Where(p => p.StartsWith(path)).ToArray(); + path = path.LongPathName; + var pathsToRemove = fsCache.Where(p => ((string)p).StartsWith(path)).ToArray(); foreach (var p in pathsToRemove) fsCache.Remove(p); } - private void AddPath(string path) + private void AddPath(LongPath path) { + path = path.LongPathName; if (!File.Exists(path) && !Directory.Exists(path)) return; if (File.GetAttributes(path).HasFlag(FileAttributes.Directory)) @@ -141,12 +143,14 @@ namespace FileManager else AddUniqueFile(path); } - private void AddUniqueFiles(IEnumerable newFiles) + + private void AddUniqueFiles(IEnumerable newFiles) { foreach (var file in newFiles) AddUniqueFile(file); } - private void AddUniqueFile(string newFile) + + private void AddUniqueFile(LongPath newFile) { if (!fsCache.Contains(newFile)) fsCache.Add(newFile); diff --git a/Source/FileManager/FileNamingTemplate.cs b/Source/FileManager/FileNamingTemplate.cs index 1dba7e0f..f21b3b06 100644 --- a/Source/FileManager/FileNamingTemplate.cs +++ b/Source/FileManager/FileNamingTemplate.cs @@ -23,16 +23,16 @@ namespace FileManager => ParameterReplacements.Add(key, value); /// If set, truncate each parameter replacement to this many characters. Default 50 - public int? ParameterMaxSize { get; set; } = 50; + public int? ParameterMaxSize { get; set; } = null; /// Optional step 2: Replace all illegal characters with this. Default= public string IllegalCharacterReplacements { get; set; } /// Generate a valid path for this file or directory - public string GetFilePath(bool returnFirstExisting = false) + public LongPath GetFilePath(bool returnFirstExisting = false) { var filename = Template; - + foreach (var r in ParameterReplacements) filename = filename.Replace($"<{formatKey(r.Key)}>", formatValue(r.Value)); diff --git a/Source/FileManager/FileUtility.cs b/Source/FileManager/FileUtility.cs index 61d44847..9a3c3a83 100644 --- a/Source/FileManager/FileUtility.cs +++ b/Source/FileManager/FileUtility.cs @@ -39,8 +39,6 @@ namespace FileManager return position.ToString().PadLeft(total.ToString().Length, '0'); } - private const int MAX_FILENAME_LENGTH = 255; - private const int MAX_DIRECTORY_LENGTH = 247; /// /// Ensure valid file name path: @@ -48,7 +46,7 @@ namespace FileManager ///
- ensure uniqueness ///
- enforce max file length /// - public static string GetValidFilename(string path, string illegalCharacterReplacements = "", bool returnFirstExisting = false) + public static LongPath GetValidFilename(LongPath path, string illegalCharacterReplacements = "", bool returnFirstExisting = false) { ArgumentValidator.EnsureNotNull(path, nameof(path)); @@ -57,14 +55,15 @@ namespace FileManager // ensure uniqueness and check lengths var dir = Path.GetDirectoryName(path); - dir = dir.Truncate(MAX_DIRECTORY_LENGTH); - - var filename = Path.GetFileNameWithoutExtension(path); - var fileStem = Path.Combine(dir, filename); + dir = dir?.Truncate(LongPath.MaxDirectoryLength) ?? string.Empty; var extension = Path.GetExtension(path); - var fullfilename = fileStem.Truncate(MAX_FILENAME_LENGTH - extension.Length) + extension; + var filename = Path.GetFileNameWithoutExtension(path).Truncate(LongPath.MaxFilenameLength - extension.Length); + var fileStem = Path.Combine(dir, filename); + + + var fullfilename = fileStem.Truncate(LongPath.MaxPathLength - extension.Length) + extension; fullfilename = removeInvalidWhitespace(fullfilename); @@ -72,7 +71,7 @@ namespace FileManager while (File.Exists(fullfilename) && !returnFirstExisting) { var increm = $" ({++i})"; - fullfilename = fileStem.Truncate(MAX_FILENAME_LENGTH - increm.Length - extension.Length) + increm + extension; + fullfilename = fileStem.Truncate(LongPath.MaxPathLength - increm.Length - extension.Length) + increm + extension; } return fullfilename; @@ -85,16 +84,18 @@ namespace FileManager => string.Join(illegalCharacterReplacements ?? "", str.Split(Path.GetInvalidFileNameChars())); /// Use with full path, not file name. Valid path charaters which are invalid file name characters will be retained: '\\', '/' - public static string GetSafePath(string path, string illegalCharacterReplacements = "") + public static LongPath GetSafePath(LongPath path, string illegalCharacterReplacements = "") { ArgumentValidator.EnsureNotNull(path, nameof(path)); - path = replaceInvalidChars(path, illegalCharacterReplacements); - path = standardizeSlashes(path); - path = replaceColons(path, illegalCharacterReplacements); - path = removeDoubleSlashes(path); + var pathNoPrefix = path.PathWithoutPrefix; - return path; + pathNoPrefix = replaceInvalidChars(pathNoPrefix, illegalCharacterReplacements); + pathNoPrefix = standardizeSlashes(pathNoPrefix); + pathNoPrefix = replaceColons(pathNoPrefix, illegalCharacterReplacements); + pathNoPrefix = removeDoubleSlashes(pathNoPrefix); + + return pathNoPrefix; } private static char[] invalidChars { get; } = Path.GetInvalidPathChars().Union(new[] { @@ -169,7 +170,7 @@ namespace FileManager ///
- Perform ///
- Return valid path /// - public static string SaferMoveToValidPath(string source, string destination) + public static string SaferMoveToValidPath(LongPath source, LongPath destination) { destination = GetValidFilename(destination); SaferMove(source, destination); @@ -184,7 +185,7 @@ namespace FileManager .WaitAndRetry(maxRetryAttempts, i => pauseBetweenFailures); /// Delete file. No error when source does not exist. Retry up to 3 times before throwing exception. - public static void SaferDelete(string source) + public static void SaferDelete(LongPath source) => retryPolicy.Execute(() => { try @@ -207,7 +208,7 @@ namespace FileManager }); /// Move file. No error when source does not exist. Retry up to 3 times before throwing exception. - public static void SaferMove(string source, string destination) + public static void SaferMove(LongPath source, LongPath destination) => retryPolicy.Execute(() => { try @@ -242,27 +243,32 @@ namespace FileManager /// Filename pattern match /// Search subdirectories or only top level directory for files /// List of files - public static IEnumerable SaferEnumerateFiles(string path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly) + public static IEnumerable SaferEnumerateFiles(LongPath path, string searchPattern = "*", SearchOption searchOption = SearchOption.TopDirectoryOnly) { - var foundFiles = Enumerable.Empty(); + var foundFiles = Enumerable.Empty(); if (searchOption == SearchOption.AllDirectories) { try { - IEnumerable subDirs = Directory.EnumerateDirectories(path); + var list = Directory.EnumerateDirectories(path).ToList(); + IEnumerable subDirs = Directory.EnumerateDirectories(path).Select(p => (LongPath)p); // Add files in subdirectories recursively to the list foreach (string dir in subDirs) foundFiles = foundFiles.Concat(SaferEnumerateFiles(dir, searchPattern, searchOption)); } catch (UnauthorizedAccessException) { } catch (PathTooLongException) { } + catch(Exception ex) + { + + } } try { // Add files from the current directory - foundFiles = foundFiles.Concat(Directory.EnumerateFiles(path, searchPattern)); + foundFiles = foundFiles.Concat(Directory.EnumerateFiles(path, searchPattern).Select(f => (LongPath)f)); } catch (UnauthorizedAccessException) { } diff --git a/Source/FileManager/LongPath.cs b/Source/FileManager/LongPath.cs new file mode 100644 index 00000000..18f59e8b --- /dev/null +++ b/Source/FileManager/LongPath.cs @@ -0,0 +1,92 @@ +using Newtonsoft.Json; +using System; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; + +namespace FileManager +{ + public class LongPath + { + //https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd + + public const int MaxDirectoryLength = MaxPathLength - 13; + public const int MaxPathLength = short.MaxValue; + public const int MaxFilenameLength = 255; + + private const int MAX_PATH = 260; + private const string LONG_PATH_PREFIX = "\\\\?\\"; + private static readonly StringBuilder longPathBuffer = new(MaxPathLength); + + public string Path { get; init; } + public override string ToString() => Path; + + public static implicit operator LongPath(string path) + { + if (path is null) return null; + + //File I/O functions in the Windows API convert "/" to "\" as part of converting + //the name to an NT-style name, except when using the "\\?\" prefix + path = path.Replace('/', '\\'); + + if (path.StartsWith(LONG_PATH_PREFIX)) + return new LongPath { Path = path }; + else if ((path.Length > 2 && path[1] == ':') || path.StartsWith("UNC\\")) + return new LongPath { Path = LONG_PATH_PREFIX + path }; + else if (path.StartsWith("\\\\")) + //The "\\?\" prefix can also be used with paths constructed according to the + //universal naming convention (UNC). To specify such a path using UNC, use + //the "\\?\UNC\" prefix. + return new LongPath { Path = LONG_PATH_PREFIX + "UNC\\" + path.Substring(2) }; + else + { + //These prefixes are not used as part of the path itself. They indicate that + //the path should be passed to the system with minimal modification, which + //means that you cannot use forward slashes to represent path separators, or + //a period to represent the current directory, or double dots to represent the + //parent directory. Because you cannot use the "\\?\" prefix with a relative + //path, relative paths are always limited to a total of MAX_PATH characters. + if (path.Length > MAX_PATH) + throw new System.IO.PathTooLongException(); + return new LongPath { Path = path }; + } + } + + public static implicit operator string(LongPath path) => path?.Path ?? null; + + [JsonIgnore] + public string ShortPathName + { + get + { + if (Path is null) return null; + GetShortPathName(Path, longPathBuffer, MAX_PATH); + return longPathBuffer.ToString(); + } + } + + [JsonIgnore] + public string LongPathName + { + get + { + if (Path is null) return null; + GetLongPathName(Path, longPathBuffer, MaxPathLength); + return longPathBuffer.ToString(); + } + } + + [JsonIgnore] + public string PathWithoutPrefix + => Path?.StartsWith(LONG_PATH_PREFIX) == true ? + Path.Remove(0, LONG_PATH_PREFIX.Length) : + Path; + + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] + private static extern int GetShortPathName(string path, StringBuilder shortPath, int shortPathLength); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] + private static extern int GetLongPathName(string lpszShortPath, StringBuilder lpszLongPath, int cchBuffer); + } +} diff --git a/Source/LibationFileManager/AudibleFileStorage.cs b/Source/LibationFileManager/AudibleFileStorage.cs index 26d3c304..d5c6a344 100644 --- a/Source/LibationFileManager/AudibleFileStorage.cs +++ b/Source/LibationFileManager/AudibleFileStorage.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; @@ -9,18 +8,18 @@ namespace LibationFileManager { public abstract class AudibleFileStorage { - protected abstract string GetFilePathCustom(string productId); + protected abstract LongPath GetFilePathCustom(string productId); #region static - public static string DownloadsInProgressDirectory => Directory.CreateDirectory(Path.Combine(Configuration.Instance.InProgress, "DownloadsInProgress")).FullName; - public static string DecryptInProgressDirectory => Directory.CreateDirectory(Path.Combine(Configuration.Instance.InProgress, "DecryptInProgress")).FullName; + public static LongPath DownloadsInProgressDirectory => Directory.CreateDirectory(Path.Combine(Configuration.Instance.InProgress, "DownloadsInProgress")).FullName; + public static LongPath DecryptInProgressDirectory => Directory.CreateDirectory(Path.Combine(Configuration.Instance.InProgress, "DecryptInProgress")).FullName; private static AaxcFileStorage AAXC { get; } = new AaxcFileStorage(); public static bool AaxcExists(string productId) => AAXC.Exists(productId); public static AudioFileStorage Audio { get; } = new AudioFileStorage(); - public static string BooksDirectory + public static LongPath BooksDirectory { get { @@ -43,7 +42,7 @@ namespace LibationFileManager regexTemplate = $@"{{0}}.*?\.({extAggr})$"; } - protected string GetFilePath(string productId) + protected LongPath GetFilePath(string productId) { // primary lookup var cachedFile = FilePathCache.GetFirstPath(productId, FileType); @@ -70,7 +69,7 @@ namespace LibationFileManager { internal AaxcFileStorage() : base(FileType.AAXC) { } - protected override string GetFilePathCustom(string productId) + protected override LongPath GetFilePathCustom(string productId) { var regex = GetBookSearchRegex(productId); return FileUtility @@ -88,7 +87,7 @@ namespace LibationFileManager private static BackgroundFileSystem BookDirectoryFiles { get; set; } private static object bookDirectoryFilesLocker { get; } = new(); - protected override string GetFilePathCustom(string productId) + protected override LongPath GetFilePathCustom(string productId) { // If user changed the BooksDirectory: reinitialize lock (bookDirectoryFilesLocker) @@ -101,6 +100,6 @@ namespace LibationFileManager public void Refresh() => BookDirectoryFiles.RefreshFiles(); - public string GetPath(string productId) => GetFilePath(productId); + public LongPath GetPath(string productId) => GetFilePath(productId); } } diff --git a/Source/LibationFileManager/Configuration.cs b/Source/LibationFileManager/Configuration.cs index bc4b3698..19cbd710 100644 --- a/Source/LibationFileManager/Configuration.cs +++ b/Source/LibationFileManager/Configuration.cs @@ -455,7 +455,7 @@ namespace LibationFileManager } } - private static string libationFilesPathCache; + private static string libationFilesPathCache { get; set; } private string getLibationFilesSettingFromJson() { @@ -478,7 +478,7 @@ namespace LibationFileManager catch { } // not found. write to file. read from file - var endingContents = new JObject { { LIBATION_FILES_KEY, UserProfile } }.ToString(Formatting.Indented); + var endingContents = new JObject { { LIBATION_FILES_KEY, UserProfile.ToString() } }.ToString(Formatting.Indented); if (startingContents != endingContents) { File.WriteAllText(APPSETTINGS_JSON, endingContents); diff --git a/Source/LibationFileManager/FilePathCache.cs b/Source/LibationFileManager/FilePathCache.cs index 802864b8..1f096990 100644 --- a/Source/LibationFileManager/FilePathCache.cs +++ b/Source/LibationFileManager/FilePathCache.cs @@ -3,13 +3,14 @@ using System.Collections.Generic; using System.IO; using System.Linq; using Dinah.Core.Collections.Immutable; +using FileManager; using Newtonsoft.Json; namespace LibationFileManager { public static class FilePathCache { - public record CacheEntry(string Id, FileType FileType, string Path); + public record CacheEntry(string Id, FileType FileType, LongPath Path); private const string FILENAME = "FileLocations.json"; @@ -18,7 +19,7 @@ namespace LibationFileManager private static Cache cache { get; } = new Cache(); - private static string jsonFile => Path.Combine(Configuration.Instance.LibationFiles, FILENAME); + private static LongPath jsonFile => Path.Combine(Configuration.Instance.LibationFiles, FILENAME); static FilePathCache() { @@ -44,12 +45,12 @@ namespace LibationFileManager public static bool Exists(string id, FileType type) => GetFirstPath(id, type) is not null; - public static List<(FileType fileType, string path)> GetFiles(string id) + public static List<(FileType fileType, LongPath path)> GetFiles(string id) => getEntries(entry => entry.Id == id) .Select(entry => (entry.FileType, entry.Path)) .ToList(); - public static string GetFirstPath(string id, FileType type) + public static LongPath GetFirstPath(string id, FileType type) => getEntries(entry => entry.Id == id && entry.FileType == type) ?.FirstOrDefault() ?.Path; @@ -62,7 +63,7 @@ namespace LibationFileManager remove(entries.Where(e => !File.Exists(e.Path)).ToList()); - return entries; + return cache.Where(predicate).ToList(); } private static void remove(List entries) diff --git a/Source/LibationFileManager/Templates.cs b/Source/LibationFileManager/Templates.cs index 4ce42d03..26665648 100644 --- a/Source/LibationFileManager/Templates.cs +++ b/Source/LibationFileManager/Templates.cs @@ -105,7 +105,7 @@ namespace LibationFileManager => string.IsNullOrWhiteSpace(template) ? "" : getFileNamingTemplate(libraryBookDto, template, null, null) - .GetFilePath(); + .GetFilePath().PathWithoutPrefix; private static Regex ifSeriesRegex { get; } = new Regex("(.*?)<-if series>", RegexOptions.Compiled | RegexOptions.IgnoreCase); @@ -276,7 +276,7 @@ namespace LibationFileManager fileNamingTemplate.AddParameterReplacement(TemplateTags.ChNumber0, FileUtility.GetSequenceFormatted(props.PartsPosition, props.PartsTotal)); fileNamingTemplate.AddParameterReplacement(TemplateTags.ChTitle, props.Title ?? ""); - return fileNamingTemplate.GetFilePath(); + return fileNamingTemplate.GetFilePath().PathWithoutPrefix; } #endregion } diff --git a/Source/LibationWinForms/Dialogs/MessageBoxAlertAdminDialog.cs b/Source/LibationWinForms/Dialogs/MessageBoxAlertAdminDialog.cs index 6630d747..3f0ab9e0 100644 --- a/Source/LibationWinForms/Dialogs/MessageBoxAlertAdminDialog.cs +++ b/Source/LibationWinForms/Dialogs/MessageBoxAlertAdminDialog.cs @@ -3,6 +3,7 @@ using System; using System.Linq; using System.Drawing; using System.Windows.Forms; +using FileManager; namespace LibationWinForms.Dialogs { @@ -47,7 +48,7 @@ namespace LibationWinForms.Dialogs private void logsLink_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e) { - string dir = ""; + LongPath dir = ""; try { dir = LibationFileManager.Configuration.Instance.LibationFiles; @@ -56,7 +57,7 @@ namespace LibationWinForms.Dialogs try { - Go.To.Folder(dir); + Go.To.Folder(dir.ShortPathName); } catch { diff --git a/Source/LibationWinForms/Dialogs/SettingsDialog.cs b/Source/LibationWinForms/Dialogs/SettingsDialog.cs index c7e2ded9..aa85b114 100644 --- a/Source/LibationWinForms/Dialogs/SettingsDialog.cs +++ b/Source/LibationWinForms/Dialogs/SettingsDialog.cs @@ -1,10 +1,10 @@ using System; -using System.Collections.Generic; using System.Linq; using System.IO; using System.Windows.Forms; using Dinah.Core; using LibationFileManager; +using FileManager; namespace LibationWinForms.Dialogs { @@ -124,7 +124,7 @@ namespace LibationWinForms.Dialogs chapterFileTemplateTb.Text = config.ChapterFileTemplate; } - private void logsBtn_Click(object sender, EventArgs e) => Go.To.Folder(Configuration.Instance.LibationFiles); + private void logsBtn_Click(object sender, EventArgs e) => Go.To.Folder(((LongPath)Configuration.Instance.LibationFiles).ShortPathName); private void folderTemplateBtn_Click(object sender, EventArgs e) => editTemplate(Templates.Folder, folderTemplateTb); private void fileTemplateBtn_Click(object sender, EventArgs e) => editTemplate(Templates.File, fileTemplateTb); diff --git a/Source/LibationWinForms/Form1.ProcessQueue.cs b/Source/LibationWinForms/Form1.ProcessQueue.cs index 806d43c0..22d12329 100644 --- a/Source/LibationWinForms/Form1.ProcessQueue.cs +++ b/Source/LibationWinForms/Form1.ProcessQueue.cs @@ -38,7 +38,7 @@ namespace LibationWinForms { // liberated: open explorer to file var filePath = AudibleFileStorage.Audio.GetPath(e.Book.AudibleProductId); - if (!Go.To.File(filePath)) + if (!Go.To.File(filePath?.ShortPathName)) { var suffix = string.IsNullOrWhiteSpace(filePath) ? "" : $":\r\n{filePath}"; MessageBox.Show($"File not found" + suffix);