Merge pull request #258 from Mbucari/master

Add support for unlimited library size.

Source/AaxDecrypter/NetworkFileStream.cs

@@ -217,8 +217,10 @@ namespace AaxDecrypter
 		{
 			var downloadPosition = WritePosition;
 			var nextFlush = downloadPosition + DATA_FLUSH_SZ;
-
 			var buff = new byte[DOWNLOAD_BUFF_SZ];
+
+			try
+			{
 				do
 				{
 					var bytesRead = _networkStream.Read(buff, 0, DOWNLOAD_BUFF_SZ);
@@ -250,7 +252,12 @@ namespace AaxDecrypter
 
 				if (WritePosition > ContentLength)
 					throw new WebException($"Downloaded size (0x{WritePosition:X10}) is greater than {nameof(ContentLength)} (0x{ContentLength:X10}).");
-
+			}
+			catch (Exception ex)
+			{
+				Serilog.Log.Error(ex, "An error was encountered while downloading {Uri}", Uri);
+				IsCancelled = true;
+			}
 		}
 
 		#endregion

Source/AudibleUtilities/ApiExtended.cs

@@ -118,31 +118,37 @@ namespace AudibleUtilities
 		private async Task<List<Item>> getItemsAsync(LibraryOptions libraryOptions, bool importEpisodes)
 		{
 			var items = new List<Item>();
-#if DEBUG
-//// this will not work for multi accounts
-//var library_json = "library.json";
-//library_json = System.IO.Path.GetFullPath(library_json);
-//if (System.IO.File.Exists(library_json))
-//{
-//    items = AudibleApi.Common.Converter.FromJson<List<Item>>(System.IO.File.ReadAllText(library_json));
-//}
-#endif
 
-			Serilog.Log.Logger.Debug("Begin initial library scan");
+			Serilog.Log.Logger.Debug("Begin library scan");
 
-			if (!items.Any())
+			List<Task<List<Item>>> getChildEpisodesTasks = new();
-				items = await Api.GetAllLibraryItemsAsync(libraryOptions);
 
-			Serilog.Log.Logger.Debug("Initial library scan complete. Begin episode scan");
+			int count = 0;
 
-			await manageEpisodesAsync(items, importEpisodes);
+			await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions))
+			{
+				if (item.IsEpisodes && importEpisodes)
+				{
+					//Get child episodes asynchronously and await all at the end
+					getChildEpisodesTasks.Add(getChildEpisodesAsync(item));
+				}
+				else if (!item.IsEpisodes)
+					items.Add(item);
 
-			Serilog.Log.Logger.Debug("Episode scan complete");
+				count++;
+			}
+
+			Serilog.Log.Logger.Debug("Library scan complete. Found {count} books. Waiting on episode scans to complete", count);
+
+			//await and add all episides from all parents
+			foreach (var epList in await Task.WhenAll(getChildEpisodesTasks))
+				items.AddRange(epList);
+
+			Serilog.Log.Logger.Debug("Scan complete");
 
 #if DEBUG
 //System.IO.File.WriteAllText(library_json, AudibleApi.Common.Converter.ToJson(items));
 #endif
-
 			var validators = new List<IValidator>();
 			validators.AddRange(getValidators());
 			foreach (var v in validators)
@@ -156,55 +162,17 @@ namespace AudibleUtilities
 		}
 
 		#region episodes and podcasts
-		private async Task manageEpisodesAsync(List<Item> items, bool importEpisodes)
-		{
-			// add podcasts and episodes to list. If fail, don't let it de-rail the rest of the import
-			try
-			{
-				// get parents
-				var parents = items.Where(i => i.IsEpisodes).ToList();
-#if DEBUG
-//var parentsDebug = parents.Select(i => i.ToJson()).Aggregate((a, b) => $"{a}\r\n\r\n{b}");
-//System.IO.File.WriteAllText("parents.json", parentsDebug);
-#endif
 		
-				if (!parents.Any())
+		private async Task<List<Item>> getChildEpisodesAsync(Item parent)
-					return;
-
-				Serilog.Log.Logger.Information($"{parents.Count} series of shows/podcasts found");
-
-				// remove episode parents. even if the following stuff fails, these will still be removed from the collection
-				items.RemoveAll(i => i.IsEpisodes);
-
-				if (importEpisodes)
 		{
-					// add children
+			Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent);
-					var children = await getEpisodesAsync(parents);
-					Serilog.Log.Logger.Information($"{children.Count} episodes of shows/podcasts found");
-					items.AddRange(children);
-				}
-			}
-			catch (Exception ex)
-			{
-				Serilog.Log.Logger.Error(ex, "Error adding podcasts and episodes");
-			}
-		}
 
-		private async Task<List<Item>> getEpisodesAsync(List<Item> parents)
-		{
-			var results = new List<Item>();
-
-			foreach (var parent in parents)
-			{
 			var children = await getEpisodeChildrenAsync(parent);
 
 			// actual individual episode, not the parent of a series.
 			// for now I'm keeping it inside this method since it fits the work flow, incl. importEpisodes logic
 			if (!children.Any())
-				{
+				return new List<Item>() { parent };
-					results.Add(parent);
-					continue;
-				}
 
 			foreach (var child in children)
 			{
@@ -231,11 +199,7 @@ namespace AudibleUtilities
 						}
 				};
 			}
-
+			return children;
-				results.AddRange(children);
-			}
-
-			return results;
 		}
 
 		private async Task<List<Item>> getEpisodeChildrenAsync(Item parent)
@@ -277,7 +241,7 @@ namespace AudibleUtilities
 					throw;
 				}
 
-				Serilog.Log.Logger.Debug($"Batch {i}: {childrenBatch.Count} results");
+				Serilog.Log.Logger.Debug($"Batch {i}: {childrenBatch.Count} results\t({{parent}})", parent);
 				// the service returned no results. probably indicates an error. stop running batches
 				if (!childrenBatch.Any())
 					break;
@@ -295,7 +259,7 @@ namespace AudibleUtilities
 			if (childrenIds.Count != results.Count)
 			{
 				var ex = new ApplicationException($"Mis-match: Children defined by parent={childrenIds.Count}. Children returned by batches={results.Count}");
-				Serilog.Log.Logger.Error(ex, "Quantity of series episodes defined by parent does not match quantity returned by batch fetching.");
+				Serilog.Log.Logger.Error(ex, "{parent} - Quantity of series episodes defined by parent does not match quantity returned by batch fetching.", parent);
 				throw ex;
 			}
 

Source/FileLiberator/AudioDecodable.cs

@@ -4,7 +4,8 @@ namespace FileLiberator
 {
     public abstract class AudioDecodable : Processable
     {
-        public event EventHandler<Action<byte[]>> RequestCoverArt;
+        public delegate byte[] RequestCoverArtHandler(object sender, EventArgs eventArgs);
+        public event RequestCoverArtHandler RequestCoverArt;
         public event EventHandler<string> TitleDiscovered;
         public event EventHandler<string> AuthorsDiscovered;
         public event EventHandler<string> NarratorsDiscovered;
@@ -32,10 +33,10 @@ namespace FileLiberator
             NarratorsDiscovered?.Invoke(this, narrators);
 		}
 
-        protected void OnRequestCoverArt(Action<byte[]> setCoverArtDel)
+        protected byte[] OnRequestCoverArt()
 		{
             Serilog.Log.Logger.Debug("Event fired {@DebugInfo}", new { Name = nameof(RequestCoverArt) });
-            RequestCoverArt?.Invoke(this, setCoverArtDel);
+            return RequestCoverArt?.Invoke(this, new());
         }
 
         protected void OnCoverImageDiscovered(byte[] coverImage)

Source/FileLiberator/DownloadDecryptBook.cs

@@ -247,7 +247,7 @@ namespace FileLiberator
             if (e is not null)
                 OnCoverImageDiscovered(e);
             else if (Configuration.Instance.AllowLibationFixup)
-                OnRequestCoverArt(abDownloader.SetCoverArt);
+                abDownloader.SetCoverArt(OnRequestCoverArt());
         }
 
         /// <summary>Move new files to 'Books' directory</summary>

Source/LibationWinForms/GridView/GridEntry.cs

@@ -100,9 +100,9 @@ namespace LibationWinForms.GridView
 	{
 #nullable enable
 		public static IEnumerable<SeriesEntry> Series(this IEnumerable<GridEntry> gridEntries)
-			=> gridEntries.Where(i => i is SeriesEntry).Cast<SeriesEntry>();
+			=> gridEntries.OfType<SeriesEntry>();
 		public static IEnumerable<LibraryBookEntry> LibraryBooks(this IEnumerable<GridEntry> gridEntries)
-			=> gridEntries.Where(i => i is LibraryBookEntry).Cast<LibraryBookEntry>();
+			=> gridEntries.OfType<LibraryBookEntry>();
 		public static LibraryBookEntry? FindBookByAsin(this IEnumerable<LibraryBookEntry> gridEntries, string audibleProductID)
 			=> gridEntries.FirstOrDefault(i => i.AudibleProductId == audibleProductID);
 		public static SeriesEntry? FindBookSeriesEntry(this IEnumerable<GridEntry> gridEntries, IEnumerable<SeriesBook> matchSeries)

Source/LibationWinForms/GridView/LiberateDataGridViewImageButtonColumn.cs

@@ -30,10 +30,7 @@ namespace LibationWinForms.GridView
 
 				if (status.IsSeries)
 				{
-					var imageName = status.Expanded ? "minus" : "plus";
+					DrawButtonImage(graphics, status.Expanded ? Properties.Resources.minus: Properties.Resources.plus, cellBounds);
-
-					var bmp = (Bitmap)Properties.Resources.ResourceManager.GetObject(imageName);
-					DrawButtonImage(graphics, bmp, cellBounds);
 
 					ToolTipText = status.Expanded ? "Click to Collpase" : "Click to Expand";
 				}

Source/LibationWinForms/ProcessQueue/ProcessBook.cs

@@ -60,7 +60,6 @@ namespace LibationWinForms.ProcessQueue
 		private Processable CurrentProcessable => _currentProcessable ??= Processes.Dequeue().Invoke();
 		private Processable NextProcessable() => _currentProcessable = null;
 		private Processable _currentProcessable;
-		private Func<byte[]> GetCoverArtDelegate;
 		private readonly Queue<Func<Processable>> Processes = new();
 		private readonly LogMe Logger;
 
@@ -232,11 +231,14 @@ namespace LibationWinForms.ProcessQueue
 			BookText = $"{title}\r\nBy {authorNames}\r\nNarrated by {narratorNames}";
 		}
 
-		public void AudioDecodable_RequestCoverArt(object sender, Action<byte[]> setCoverArtDelegate)
+		private byte[] AudioDecodable_RequestCoverArt(object sender, EventArgs e)
 		{
-			byte[] coverData = GetCoverArtDelegate();
+			byte[] coverData = PictureStorage
-			setCoverArtDelegate(coverData);
+				.GetPictureSynchronously(
+				new PictureDefinition(LibraryBook.Book.PictureId, PictureSize._500x500));
+
 			AudioDecodable_CoverImageDiscovered(this, coverData);
+			return coverData;
 		}
 
 		private void AudioDecodable_CoverImageDiscovered(object sender, byte[] coverArt)
@@ -273,11 +275,6 @@ namespace LibationWinForms.ProcessQueue
 
 			Logger.Info($"{Environment.NewLine}{((Processable)sender).Name} Step, Begin: {libraryBook.Book}");
 
-			GetCoverArtDelegate = () => PictureStorage.GetPictureSynchronously(
-						new PictureDefinition(
-							libraryBook.Book.PictureId,
-							PictureSize._500x500));
-
 			title = libraryBook.Book.Title;
 			authorNames = libraryBook.Book.AuthorNames();
 			narratorNames = libraryBook.Book.NarratorNames();
@@ -286,7 +283,6 @@ namespace LibationWinForms.ProcessQueue
 
 		private async void Processable_Completed(object sender, LibraryBook libraryBook)
 		{
-
 			Logger.Info($"{((Processable)sender).Name} Step, Completed: {libraryBook.Book}");
 			UnlinkProcessable((Processable)sender);