Pre-beta: image download throttling

FileManager/UNTESTED/PictureStorage.cs

@@ -2,77 +2,108 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 
 namespace FileManager
 {
-    /// <summary>
+	public enum PictureSize { _80x80, _300x300, _500x500 }
-    /// Files are small. Entire file is read from disk every time. No volitile storage. Paths are well known
+	public struct PictureDefinition
-    /// </summary>
+	{
+		public string PictureId { get; }
+		public PictureSize Size { get; }
+
+		public PictureDefinition(string pictureId, PictureSize pictureSize)
+		{
+			PictureId = pictureId;
+			Size = pictureSize;
+		}
+	}
     public static class PictureStorage
     {
-        public enum PictureSize { _80x80, _300x300, _500x500 }
-
         // not customizable. don't move to config
         private static string ImagesDirectory { get; }
             = new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("Images").FullName;
 
-        private static string getPath(string pictureId, PictureSize size)
+		private static string getPath(PictureDefinition def)
-            => Path.Combine(ImagesDirectory, $"{pictureId}{size}.jpg");
+			=> Path.Combine(ImagesDirectory, $"{def.PictureId}{def.Size}.jpg");
 
-        public static byte[] GetImage(string pictureId, PictureSize size)
+		private static System.Timers.Timer timer { get; }
+		static PictureStorage()
 		{
-            var path = getPath(pictureId, size);
+			timer = new System.Timers.Timer(700)
-            if (!FileUtility.FileExists(path))
+			{
-                DownloadImages(pictureId);
+				AutoReset = true,
-
+				Enabled = true
-            return File.ReadAllBytes(path);
+			};
+			timer.Elapsed += (_, __) => timerDownload();
 		}
 
-        public static void DownloadImages(string pictureId)
+		private static Dictionary<PictureDefinition, byte[]> cache { get; } = new Dictionary<PictureDefinition, byte[]>();
+		public static (bool isDefault, byte[] bytes) GetPicture(PictureDefinition def)
 		{
-            var path80 = getPath(pictureId, PictureSize._80x80);
+			if (!cache.ContainsKey(def))
-            var path300 = getPath(pictureId, PictureSize._300x300);
+				cache.Add(def, null);
-            var path500 = getPath(pictureId, PictureSize._500x500);
+			return (cache[def] == null, cache[def] ?? getDefaultImage(def.Size));
+		}
 
-            int retry = 0;
+		private static Dictionary<PictureSize, byte[]> defaultImages { get; } = new Dictionary<PictureSize, byte[]>();
-            do
+		public static void SetDefaultImage(PictureSize pictureSize, byte[] bytes)
+			=> defaultImages[pictureSize] = bytes;
+		private static byte[] getDefaultImage(PictureSize size)
+			=> defaultImages.ContainsKey(size)
+			? defaultImages[size]
+			: new byte[0];
+
+		// necessary to avoid IO errors. ReadAllBytes and WriteAllBytes can conflict in some cases, esp when debugging
+		private static bool isProcessing;
+		private static void timerDownload()
 		{
 			try
 			{
-					using var webClient = new System.Net.WebClient();
+				if (isProcessing)
-					// download any that don't exist
+					return;
-					{
+				isProcessing = true;
-						if (!FileUtility.FileExists(path80))
-						{
-							var bytes = webClient.DownloadData(
-								"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL80_.jpg");
-							File.WriteAllBytes(path80, bytes);
-						}
-					}
 
-					{
+				PictureDefinition def;
-						if (!FileUtility.FileExists(path300))
+				string path;
-						{
-							var bytes = webClient.DownloadData(
-								"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL300_.jpg");
-							File.WriteAllBytes(path300, bytes);
-						}
-					}
 
+				while (true)
 				{
-						if (!FileUtility.FileExists(path500))
+					def = cache
-						{
+						.Where(kvp => kvp.Value is null)
-							var bytes = webClient.DownloadData(
+						.Select(kvp => kvp.Key)
-								"https://m.media-amazon.com/images/I/" + pictureId + "._SL500_.jpg");
+						// 80x80 should be 1st since it's enum value == 0
-							File.WriteAllBytes(path500, bytes);
+						.OrderBy(d => d.PictureId)
-						}
+						.FirstOrDefault();
-					}
+					// no more null entries. all requsted images are cached
+					if (string.IsNullOrWhiteSpace(def.PictureId))
+						return;
 
+					path = getPath(def);
+					// we found the next one to download
+					if (!FileUtility.FileExists(path))
 						break;
+
+					// file exists. read into cache. try again
+					// the point is to throttle web calls. therefore only return if we performed a d/l or there are no null cache entries
+					cache[def] = File.ReadAllBytes(path);
 				}
-                catch { retry++; }
+
+				var bytes = download(def);
+				File.WriteAllBytes(path, bytes);
+				cache[def] = bytes;
 			}
-            while (retry < 3);
+			finally
+			{
+				isProcessing = false;
+			}
+		}
+		private static HttpClient imageDownloadClient { get; } = new HttpClient();
+		private static byte[] download(PictureDefinition def)
+		{
+			var sz = def.Size.ToString().Split('x')[1];
+			var bytes = imageDownloadClient.GetByteArrayAsync("ht" + $"tps://images-na.ssl-images-amazon.com/images/I/{def.PictureId}._SL{sz}_.jpg").Result;
+			return bytes;
 		}
 	}
 }

Libation.sln

@@ -60,10 +60,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "inAudibleLite", "_Demos\inA
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.Core", "..\Dinah.Core\Dinah.Core\Dinah.Core.csproj", "{9E951521-2587-4FC6-AD26-FAA9179FB6C4}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.Core.Drawing", "..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj", "{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}"
-EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.Core.Windows.Forms", "..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj", "{1306F62D-CDAC-4269-982A-2EED51F0E318}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.EntityFrameworkCore", "..\Dinah.Core\Dinah.EntityFrameworkCore\Dinah.EntityFrameworkCore.csproj", "{1255D9BA-CE6E-42E4-A253-6376540B9661}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2", "..\LuceneNet303r2\LuceneNet303r2\LuceneNet303r2.csproj", "{35803735-B669-4090-9681-CC7F7FABDC71}"
@@ -80,6 +76,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiClientExample", "
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApplicationServices", "ApplicationServices\ApplicationServices.csproj", "{B95650EA-25F0-449E-BA5D-99126BC5D730}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.Core.WindowsDesktop", "..\Dinah.Core\Dinah.Core.WindowsDesktop\Dinah.Core.WindowsDesktop.csproj", "{059CE32C-9AD6-45E9-A166-790DFFB0B730}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindowsDesktopUtilities", "WindowsDesktopUtilities\WindowsDesktopUtilities.csproj", "{E7EFD64D-6630-4426-B09C-B6862A92E3FD}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -154,14 +154,6 @@ Global
 		{9E951521-2587-4FC6-AD26-FAA9179FB6C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9E951521-2587-4FC6-AD26-FAA9179FB6C4}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9E951521-2587-4FC6-AD26-FAA9179FB6C4}.Release|Any CPU.Build.0 = Release|Any CPU
-		{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}.Release|Any CPU.Build.0 = Release|Any CPU
-		{1306F62D-CDAC-4269-982A-2EED51F0E318}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{1306F62D-CDAC-4269-982A-2EED51F0E318}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{1306F62D-CDAC-4269-982A-2EED51F0E318}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{1306F62D-CDAC-4269-982A-2EED51F0E318}.Release|Any CPU.Build.0 = Release|Any CPU
 		{1255D9BA-CE6E-42E4-A253-6376540B9661}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{1255D9BA-CE6E-42E4-A253-6376540B9661}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1255D9BA-CE6E-42E4-A253-6376540B9661}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -194,6 +186,14 @@ Global
 		{B95650EA-25F0-449E-BA5D-99126BC5D730}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B95650EA-25F0-449E-BA5D-99126BC5D730}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B95650EA-25F0-449E-BA5D-99126BC5D730}.Release|Any CPU.Build.0 = Release|Any CPU
+		{059CE32C-9AD6-45E9-A166-790DFFB0B730}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{059CE32C-9AD6-45E9-A166-790DFFB0B730}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{059CE32C-9AD6-45E9-A166-790DFFB0B730}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{059CE32C-9AD6-45E9-A166-790DFFB0B730}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E7EFD64D-6630-4426-B09C-B6862A92E3FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E7EFD64D-6630-4426-B09C-B6862A92E3FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E7EFD64D-6630-4426-B09C-B6862A92E3FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E7EFD64D-6630-4426-B09C-B6862A92E3FD}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -216,8 +216,6 @@ Global
 		{DF72740C-900A-45DA-A3A6-4DDD68F286F2} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
 		{74D02251-898E-4CAF-80C7-801820622903} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
 		{9E951521-2587-4FC6-AD26-FAA9179FB6C4} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
-		{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
-		{1306F62D-CDAC-4269-982A-2EED51F0E318} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
 		{1255D9BA-CE6E-42E4-A253-6376540B9661} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
 		{35803735-B669-4090-9681-CC7F7FABDC71} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
 		{5A7681A5-60D9-480B-9AC7-63E0812A2548} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
@@ -226,6 +224,8 @@ Global
 		{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
 		{282EEE16-F569-47E1-992F-C6DB8AEC7AA6} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
 		{B95650EA-25F0-449E-BA5D-99126BC5D730} = {41CDCC73-9B81-49DD-9570-C54406E852AF}
+		{059CE32C-9AD6-45E9-A166-790DFFB0B730} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
+		{E7EFD64D-6630-4426-B09C-B6862A92E3FD} = {F0CBB7A7-D3FB-41FF-8F47-CF3F6A592249}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9}

LibationWinForm/LibationWinForm.csproj

@@ -9,9 +9,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj" />
-    <ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj" />
     <ProjectReference Include="..\FileLiberator\FileLiberator.csproj" />
+    <ProjectReference Include="..\WindowsDesktopUtilities\WindowsDesktopUtilities.csproj" />
   </ItemGroup>
 
   <ItemGroup>

LibationWinForm/Properties/Resources.Designer.cs

@@ -60,6 +60,36 @@ namespace LibationWinForm.Properties {
             }
         }
         
+        /// <summary>
+        ///   Looks up a localized resource of type System.Drawing.Bitmap.
+        /// </summary>
+        internal static System.Drawing.Bitmap default_cover_300x300 {
+            get {
+                object obj = ResourceManager.GetObject("default_cover_300x300", resourceCulture);
+                return ((System.Drawing.Bitmap)(obj));
+            }
+        }
+        
+        /// <summary>
+        ///   Looks up a localized resource of type System.Drawing.Bitmap.
+        /// </summary>
+        internal static System.Drawing.Bitmap default_cover_500x500 {
+            get {
+                object obj = ResourceManager.GetObject("default_cover_500x500", resourceCulture);
+                return ((System.Drawing.Bitmap)(obj));
+            }
+        }
+        
+        /// <summary>
+        ///   Looks up a localized resource of type System.Drawing.Bitmap.
+        /// </summary>
+        internal static System.Drawing.Bitmap default_cover_80x80 {
+            get {
+                object obj = ResourceManager.GetObject("default_cover_80x80", resourceCulture);
+                return ((System.Drawing.Bitmap)(obj));
+            }
+        }
+        
         /// <summary>
         ///   Looks up a localized resource of type System.Drawing.Bitmap.
         /// </summary>

LibationWinForm/Properties/Resources.resx

@@ -118,6 +118,15 @@
     <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </resheader>
   <assembly alias="System.Windows.Forms" name="System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" />
+  <data name="default_cover_300x300" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\img-coverart-prod-unavailable_300x300.jpg;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
+  </data>
+  <data name="default_cover_500x500" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\img-coverart-prod-unavailable_500x500.jpg;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
+  </data>
+  <data name="default_cover_80x80" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>..\Resources\img-coverart-prod-unavailable_80x80.jpg;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
+  </data>
   <data name="edit_tags_25x25" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>..\Resources\edit-tags-25x25.png;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
   </data>

LibationWinForm/UNTESTED/BookLiberation/DecryptForm.cs

@@ -56,10 +56,10 @@ namespace LibationWinForm.BookLiberation
             => bookInfoLbl.UIThread(() => bookInfoLbl.Text = $"{title}\r\nBy {authorNames}\r\nNarrated by {narratorNames}");
 
         public void SetCoverImage(byte[] coverBytes)
-            => pictureBox1.UIThread(() => pictureBox1.Image = ImageConverter.GetPictureFromBytes(coverBytes));
+            => pictureBox1.UIThread(() => pictureBox1.Image = ImageReader.ToImage(coverBytes));
 
-        public void AppendError(Exception ex) => AppendText("ERROR: " + ex.Message);
+        public static void AppendError(Exception ex) => AppendText("ERROR: " + ex.Message);
-        public void AppendText(string text) =>
+        public static void AppendText(string text) =>
             // redirected to log textbox
             Console.WriteLine($"{DateTime.Now} {text}")
             //logTb.UIThread(() => logTb.AppendText($"{DateTime.Now} {text}{Environment.NewLine}"))

LibationWinForm/UNTESTED/Form1.cs

@@ -6,6 +6,7 @@ using System.Windows.Forms;
 using DataLayer;
 using Dinah.Core;
 using Dinah.Core.Collections.Generic;
+using Dinah.Core.Drawing;
 using Dinah.Core.Windows.Forms;
 using FileManager;
 
@@ -41,6 +42,12 @@ namespace LibationWinForm
             // call static ctor. There are bad race conditions if static ctor is first executed when we're running in parallel in setBackupCountsAsync()
             var foo = FilePathCache.JsonFile;
 
+			// load default/missing cover images. this will also initiate the background image downloader
+			var format = System.Drawing.Imaging.ImageFormat.Jpeg;
+			PictureStorage.SetDefaultImage(PictureSize._80x80, Properties.Resources.default_cover_80x80.ToBytes(format));
+			PictureStorage.SetDefaultImage(PictureSize._300x300, Properties.Resources.default_cover_300x300.ToBytes(format));
+			PictureStorage.SetDefaultImage(PictureSize._500x500, Properties.Resources.default_cover_500x500.ToBytes(format));
+
 			reloadGrid();
 
             // also applies filter. ONLY call AFTER loading grid

LibationWinForm/UNTESTED/GridEntry.cs

@@ -37,9 +37,7 @@ namespace LibationWinForm
 		public bool TryGetFormatted(string key, out string value) => formatReplacements.TryGetValue(key, out value);
 
 		public Image Cover =>
-			Dinah.Core.Drawing.ImageConverter.GetPictureFromBytes(
+			WindowsDesktopUtilities.WinAudibleImageServer.GetImage(book.PictureId, FileManager.PictureSize._80x80);
-				FileManager.PictureStorage.GetImage(book.PictureId, FileManager.PictureStorage.PictureSize._80x80)
-				);
 
 		public string Title
 		{

WindowsDesktopUtilities/UNTESTED/WinAudibleImageServer.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using Dinah.Core.Drawing;
+using FileManager;
+
+namespace WindowsDesktopUtilities
+{
+	public static class WinAudibleImageServer
+	{
+		private static Dictionary<PictureDefinition, Image> cache { get; } = new Dictionary<PictureDefinition, Image>();
+
+		public static Image GetImage(string pictureId, PictureSize size)
+		{
+			var def = new PictureDefinition(pictureId, size);
+			if (!cache.ContainsKey(def))
+			{
+				(var isDefault, var bytes) = PictureStorage.GetPicture(def);
+
+				var image = ImageReader.ToImage(bytes);
+				if (isDefault)
+					return image;
+				cache[def] = image;
+			}
+			return cache[def];
+		}
+	}
+}

WindowsDesktopUtilities/WindowsDesktopUtilities.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">
+
+  <PropertyGroup>
+    <OutputType>Library</OutputType>
+    <TargetFramework>netcoreapp3.0</TargetFramework>
+    <UseWindowsForms>true</UseWindowsForms>
+    <ApplicationIcon />
+    <StartupObject />
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Dinah.Core\Dinah.Core.WindowsDesktop\Dinah.Core.WindowsDesktop.csproj" />
+    <ProjectReference Include="..\FileManager\FileManager.csproj" />
+  </ItemGroup>
+
+</Project>

_Demos/inAudibleLite/Form1.cs

@@ -49,7 +49,7 @@ namespace inAudibleLite
 			this.txtInputFile.Text = openFileDialog.FileName;
 			converter = await AaxToM4bConverter.CreateAsync(this.txtInputFile.Text, this.decryptKeyTb.Text);
 
-			pictureBox1.Image = Dinah.Core.Drawing.ImageConverter.GetPictureFromBytes(converter.coverBytes);
+			pictureBox1.Image = Dinah.Core.Drawing.ImageReader.ToImage(converter.coverBytes);
 
 			this.txtOutputFile.Text = converter.outputFileName;
 

_Demos/inAudibleLite/inAudibleLite.csproj

@@ -7,8 +7,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj" />
+    <ProjectReference Include="..\..\..\Dinah.Core\Dinah.Core.WindowsDesktop\Dinah.Core.WindowsDesktop.csproj" />
-    <ProjectReference Include="..\..\..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj" />
     <ProjectReference Include="..\..\AaxDecrypter\AaxDecrypter.csproj" />
   </ItemGroup>
 

__TODO.txt

@@ -1,5 +1,5 @@
 -- begin BETA ---------------------------------------------------------------------------------------------------------------------
-throttle needed for downloading images, pdf.s
+throttle needed for downloading pdf.s
 
 no mdf,ldf files created in C:\Users\[username]