tyler/Libation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pre-beta: image download throttling

Browse Source
This commit is contained in:
Robert McRackan 2019-11-13 08:37:57 -05:00
parent 01a914c390
commit 88d49acdad
16 changed files with 200 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
FileManager/UNTESTED/PictureStorage.cs
View File

@ -2,77 +2,108 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
namespace FileManager
{
/// <summary>
/// Files are small. Entire file is read from disk every time. No volitile storage. Paths are well known
/// </summary>
public enum PictureSize { _80x80, _300x300, _500x500 }
public struct PictureDefinition
{
public string PictureId { get; }
public PictureSize Size { get; }
public PictureDefinition(string pictureId, PictureSize pictureSize)
{
PictureId = pictureId;
Size = pictureSize;
}
}
public static class PictureStorage
{
public enum PictureSize { _80x80, _300x300, _500x500 }
// not customizable. don't move to config
private static string ImagesDirectory { get; }
= new DirectoryInfo(Configuration.Instance.LibationFiles).CreateSubdirectory("Images").FullName;
private static string getPath(string pictureId, PictureSize size)
=> Path.Combine(ImagesDirectory, $"{pictureId}{size}.jpg");
private static string getPath(PictureDefinition def)
=> Path.Combine(ImagesDirectory, $"{def.PictureId}{def.Size}.jpg");
public static byte[] GetImage(string pictureId, PictureSize size)
{
var path = getPath(pictureId, size);
if (!FileUtility.FileExists(path))
DownloadImages(pictureId);
private static System.Timers.Timer timer { get; }
static PictureStorage()
{
timer = new System.Timers.Timer(700)
{
AutoReset = true,
Enabled = true
};
timer.Elapsed += (_, __) => timerDownload();
}
return File.ReadAllBytes(path);
}
private static Dictionary<PictureDefinition, byte[]> cache { get; } = new Dictionary<PictureDefinition, byte[]>();
public static (bool isDefault, byte[] bytes) GetPicture(PictureDefinition def)
{
if (!cache.ContainsKey(def))
cache.Add(def, null);
return (cache[def] == null, cache[def] ?? getDefaultImage(def.Size));
}
public static void DownloadImages(string pictureId)
{
var path80 = getPath(pictureId, PictureSize._80x80);
var path300 = getPath(pictureId, PictureSize._300x300);
var path500 = getPath(pictureId, PictureSize._500x500);
private static Dictionary<PictureSize, byte[]> defaultImages { get; } = new Dictionary<PictureSize, byte[]>();
public static void SetDefaultImage(PictureSize pictureSize, byte[] bytes)
=> defaultImages[pictureSize] = bytes;
private static byte[] getDefaultImage(PictureSize size)
=> defaultImages.ContainsKey(size)
? defaultImages[size]
: new byte[0];
int retry = 0;
do
{
try
{
using var webClient = new System.Net.WebClient();
// download any that don't exist
{
if (!FileUtility.FileExists(path80))
{
var bytes = webClient.DownloadData(
"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL80_.jpg");
File.WriteAllBytes(path80, bytes);
}
}
// necessary to avoid IO errors. ReadAllBytes and WriteAllBytes can conflict in some cases, esp when debugging
private static bool isProcessing;
private static void timerDownload()
{
try
{
if (isProcessing)
return;
isProcessing = true;
{
if (!FileUtility.FileExists(path300))
{
var bytes = webClient.DownloadData(
"https://images-na.ssl-images-amazon.com/images/I/" + pictureId + "._SL300_.jpg");
File.WriteAllBytes(path300, bytes);
}
}
PictureDefinition def;
string path;
{
if (!FileUtility.FileExists(path500))
{
var bytes = webClient.DownloadData(
"https://m.media-amazon.com/images/I/" + pictureId + "._SL500_.jpg");
File.WriteAllBytes(path500, bytes);
}
}
while (true)
{
def = cache
.Where(kvp => kvp.Value is null)
.Select(kvp => kvp.Key)
// 80x80 should be 1st since it's enum value == 0
.OrderBy(d => d.PictureId)
.FirstOrDefault();
// no more null entries. all requsted images are cached
if (string.IsNullOrWhiteSpace(def.PictureId))
return;
break;
path = getPath(def);
// we found the next one to download
if (!FileUtility.FileExists(path))
break;
// file exists. read into cache. try again
// the point is to throttle web calls. therefore only return if we performed a d/l or there are no null cache entries
cache[def] = File.ReadAllBytes(path);
}
catch { retry++; }
}
while (retry < 3);
}
}
var bytes = download(def);
File.WriteAllBytes(path, bytes);
cache[def] = bytes;
}
finally
{
isProcessing = false;
}
}
private static HttpClient imageDownloadClient { get; } = new HttpClient();
private static byte[] download(PictureDefinition def)
{
var sz = def.Size.ToString().Split('x')[1];
var bytes = imageDownloadClient.GetByteArrayAsync("ht" + $"tps://images-na.ssl-images-amazon.com/images/I/{def.PictureId}._SL{sz}_.jpg").Result;
return bytes;
}
}
}

28
Libation.sln
View File

@ -60,10 +60,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "inAudibleLite", "_Demos\inA
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.Core", "..\Dinah.Core\Dinah.Core\Dinah.Core.csproj", "{9E951521-2587-4FC6-AD26-FAA9179FB6C4}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.Core.Drawing", "..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj", "{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.Core.Windows.Forms", "..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj", "{1306F62D-CDAC-4269-982A-2EED51F0E318}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.EntityFrameworkCore", "..\Dinah.Core\Dinah.EntityFrameworkCore\Dinah.EntityFrameworkCore.csproj", "{1255D9BA-CE6E-42E4-A253-6376540B9661}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneNet303r2", "..\LuceneNet303r2\LuceneNet303r2\LuceneNet303r2.csproj", "{35803735-B669-4090-9681-CC7F7FABDC71}"
@ -80,6 +76,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AudibleApiClientExample", "
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApplicationServices", "ApplicationServices\ApplicationServices.csproj", "{B95650EA-25F0-449E-BA5D-99126BC5D730}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Dinah.Core.WindowsDesktop", "..\Dinah.Core\Dinah.Core.WindowsDesktop\Dinah.Core.WindowsDesktop.csproj", "{059CE32C-9AD6-45E9-A166-790DFFB0B730}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindowsDesktopUtilities", "WindowsDesktopUtilities\WindowsDesktopUtilities.csproj", "{E7EFD64D-6630-4426-B09C-B6862A92E3FD}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -154,14 +154,6 @@ Global
{9E951521-2587-4FC6-AD26-FAA9179FB6C4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9E951521-2587-4FC6-AD26-FAA9179FB6C4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9E951521-2587-4FC6-AD26-FAA9179FB6C4}.Release|Any CPU.Build.0 = Release|Any CPU
{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE}.Release|Any CPU.Build.0 = Release|Any CPU
{1306F62D-CDAC-4269-982A-2EED51F0E318}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1306F62D-CDAC-4269-982A-2EED51F0E318}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1306F62D-CDAC-4269-982A-2EED51F0E318}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1306F62D-CDAC-4269-982A-2EED51F0E318}.Release|Any CPU.Build.0 = Release|Any CPU
{1255D9BA-CE6E-42E4-A253-6376540B9661}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1255D9BA-CE6E-42E4-A253-6376540B9661}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1255D9BA-CE6E-42E4-A253-6376540B9661}.Release|Any CPU.ActiveCfg = Release|Any CPU
@ -194,6 +186,14 @@ Global
{B95650EA-25F0-449E-BA5D-99126BC5D730}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B95650EA-25F0-449E-BA5D-99126BC5D730}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B95650EA-25F0-449E-BA5D-99126BC5D730}.Release|Any CPU.Build.0 = Release|Any CPU
{059CE32C-9AD6-45E9-A166-790DFFB0B730}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{059CE32C-9AD6-45E9-A166-790DFFB0B730}.Debug|Any CPU.Build.0 = Debug|Any CPU
{059CE32C-9AD6-45E9-A166-790DFFB0B730}.Release|Any CPU.ActiveCfg = Release|Any CPU
{059CE32C-9AD6-45E9-A166-790DFFB0B730}.Release|Any CPU.Build.0 = Release|Any CPU
{E7EFD64D-6630-4426-B09C-B6862A92E3FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E7EFD64D-6630-4426-B09C-B6862A92E3FD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E7EFD64D-6630-4426-B09C-B6862A92E3FD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E7EFD64D-6630-4426-B09C-B6862A92E3FD}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -216,8 +216,6 @@ Global
{DF72740C-900A-45DA-A3A6-4DDD68F286F2} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
{74D02251-898E-4CAF-80C7-801820622903} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
{9E951521-2587-4FC6-AD26-FAA9179FB6C4} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
{2CAAD73E-E2F9-4888-B04A-3F3803DABDAE} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
{1306F62D-CDAC-4269-982A-2EED51F0E318} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
{1255D9BA-CE6E-42E4-A253-6376540B9661} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
{35803735-B669-4090-9681-CC7F7FABDC71} = {7FBBB086-0807-4998-85BF-6D1A49C8AD05}
{5A7681A5-60D9-480B-9AC7-63E0812A2548} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
@ -226,6 +224,8 @@ Global
{6069D7F6-BEA0-4917-AFD4-4EB680CB0EDD} = {38E6C6D9-963A-4C5B-89F4-F2F14885ADFD}
{282EEE16-F569-47E1-992F-C6DB8AEC7AA6} = {F61184E7-2426-4A13-ACEF-5689928E2CE2}
{B95650EA-25F0-449E-BA5D-99126BC5D730} = {41CDCC73-9B81-49DD-9570-C54406E852AF}
{059CE32C-9AD6-45E9-A166-790DFFB0B730} = {43E3ACB3-E0BC-4370-8DBB-E3720C8C8FD1}
{E7EFD64D-6630-4426-B09C-B6862A92E3FD} = {F0CBB7A7-D3FB-41FF-8F47-CF3F6A592249}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {615E00ED-BAEF-4E8E-A92A-9B82D87942A9}

3
LibationWinForm/LibationWinForm.csproj
View File

@ -9,9 +9,8 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj" />
<ProjectReference Include="..\..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj" />
<ProjectReference Include="..\FileLiberator\FileLiberator.csproj" />
<ProjectReference Include="..\WindowsDesktopUtilities\WindowsDesktopUtilities.csproj" />
</ItemGroup>
<ItemGroup>

30
LibationWinForm/Properties/Resources.Designer.cs generated
View File

@ -60,6 +60,36 @@ namespace LibationWinForm.Properties {
}
}
/// <summary>
/// Looks up a localized resource of type System.Drawing.Bitmap.
/// </summary>
internal static System.Drawing.Bitmap default_cover_300x300 {
get {
object obj = ResourceManager.GetObject("default_cover_300x300", resourceCulture);
return ((System.Drawing.Bitmap)(obj));
}
}
/// <summary>
/// Looks up a localized resource of type System.Drawing.Bitmap.
/// </summary>
internal static System.Drawing.Bitmap default_cover_500x500 {
get {
object obj = ResourceManager.GetObject("default_cover_500x500", resourceCulture);
return ((System.Drawing.Bitmap)(obj));
}
}
/// <summary>
/// Looks up a localized resource of type System.Drawing.Bitmap.
/// </summary>
internal static System.Drawing.Bitmap default_cover_80x80 {
get {
object obj = ResourceManager.GetObject("default_cover_80x80", resourceCulture);
return ((System.Drawing.Bitmap)(obj));
}
}
/// <summary>
/// Looks up a localized resource of type System.Drawing.Bitmap.
/// </summary>

9
LibationWinForm/Properties/Resources.resx
View File

@ -118,6 +118,15 @@
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<assembly alias="System.Windows.Forms" name="System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" />
<data name="default_cover_300x300" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\img-coverart-prod-unavailable_300x300.jpg;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
</data>
<data name="default_cover_500x500" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\img-coverart-prod-unavailable_500x500.jpg;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
</data>
<data name="default_cover_80x80" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\img-coverart-prod-unavailable_80x80.jpg;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
</data>
<data name="edit_tags_25x25" type="System.Resources.ResXFileRef, System.Windows.Forms">
<value>..\Resources\edit-tags-25x25.png;System.Drawing.Bitmap, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a</value>
</data>

BIN
LibationWinForm/Resources/img-coverart-prod-unavailable_300x300.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.0 KiB

BIN
LibationWinForm/Resources/img-coverart-prod-unavailable_500x500.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 11 KiB

BIN
LibationWinForm/Resources/img-coverart-prod-unavailable_80x80.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.5 KiB

6
LibationWinForm/UNTESTED/BookLiberation/DecryptForm.cs
View File

@ -56,10 +56,10 @@ namespace LibationWinForm.BookLiberation
=> bookInfoLbl.UIThread(() => bookInfoLbl.Text = $"{title}\r\nBy {authorNames}\r\nNarrated by {narratorNames}");
public void SetCoverImage(byte[] coverBytes)
=> pictureBox1.UIThread(() => pictureBox1.Image = ImageConverter.GetPictureFromBytes(coverBytes));
=> pictureBox1.UIThread(() => pictureBox1.Image = ImageReader.ToImage(coverBytes));
public void AppendError(Exception ex) => AppendText("ERROR: " + ex.Message);
public void AppendText(string text) =>
public static void AppendError(Exception ex) => AppendText("ERROR: " + ex.Message);
public static void AppendText(string text) =>
// redirected to log textbox
Console.WriteLine($"{DateTime.Now} {text}")
//logTb.UIThread(() => logTb.AppendText($"{DateTime.Now} {text}{Environment.NewLine}"))

9
LibationWinForm/UNTESTED/Form1.cs
View File

@ -6,6 +6,7 @@ using System.Windows.Forms;
using DataLayer;
using Dinah.Core;
using Dinah.Core.Collections.Generic;
using Dinah.Core.Drawing;
using Dinah.Core.Windows.Forms;
using FileManager;
@ -41,7 +42,13 @@ namespace LibationWinForm
// call static ctor. There are bad race conditions if static ctor is first executed when we're running in parallel in setBackupCountsAsync()
var foo = FilePathCache.JsonFile;
reloadGrid();
// load default/missing cover images. this will also initiate the background image downloader
var format = System.Drawing.Imaging.ImageFormat.Jpeg;
PictureStorage.SetDefaultImage(PictureSize._80x80, Properties.Resources.default_cover_80x80.ToBytes(format));
PictureStorage.SetDefaultImage(PictureSize._300x300, Properties.Resources.default_cover_300x300.ToBytes(format));
PictureStorage.SetDefaultImage(PictureSize._500x500, Properties.Resources.default_cover_500x500.ToBytes(format));
reloadGrid();
// also applies filter. ONLY call AFTER loading grid
loadInitialQuickFilterState();

4
LibationWinForm/UNTESTED/GridEntry.cs
View File

@ -37,9 +37,7 @@ namespace LibationWinForm
public bool TryGetFormatted(string key, out string value) => formatReplacements.TryGetValue(key, out value);
public Image Cover =>
Dinah.Core.Drawing.ImageConverter.GetPictureFromBytes(
FileManager.PictureStorage.GetImage(book.PictureId, FileManager.PictureStorage.PictureSize._80x80)
);
WindowsDesktopUtilities.WinAudibleImageServer.GetImage(book.PictureId, FileManager.PictureSize._80x80);
public string Title
{

28
WindowsDesktopUtilities/UNTESTED/WinAudibleImageServer.cs Normal file
View File

@ -0,0 +1,28 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using Dinah.Core.Drawing;
using FileManager;
namespace WindowsDesktopUtilities
{
public static class WinAudibleImageServer
{
private static Dictionary<PictureDefinition, Image> cache { get; } = new Dictionary<PictureDefinition, Image>();
public static Image GetImage(string pictureId, PictureSize size)
{
var def = new PictureDefinition(pictureId, size);
if (!cache.ContainsKey(def))
{
(var isDefault, var bytes) = PictureStorage.GetPicture(def);
var image = ImageReader.ToImage(bytes);
if (isDefault)
return image;
cache[def] = image;
}
return cache[def];
}
}
}

16
WindowsDesktopUtilities/WindowsDesktopUtilities.csproj Normal file
View File

@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk.WindowsDesktop">
<PropertyGroup>
<OutputType>Library</OutputType>
<TargetFramework>netcoreapp3.0</TargetFramework>
<UseWindowsForms>true</UseWindowsForms>
<ApplicationIcon />
<StartupObject />
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Dinah.Core\Dinah.Core.WindowsDesktop\Dinah.Core.WindowsDesktop.csproj" />
<ProjectReference Include="..\FileManager\FileManager.csproj" />
</ItemGroup>
</Project>

2
_Demos/inAudibleLite/Form1.cs
View File

@ -49,7 +49,7 @@ namespace inAudibleLite
this.txtInputFile.Text = openFileDialog.FileName;
converter = await AaxToM4bConverter.CreateAsync(this.txtInputFile.Text, this.decryptKeyTb.Text);
pictureBox1.Image = Dinah.Core.Drawing.ImageConverter.GetPictureFromBytes(converter.coverBytes);
pictureBox1.Image = Dinah.Core.Drawing.ImageReader.ToImage(converter.coverBytes);
this.txtOutputFile.Text = converter.outputFileName;

3
_Demos/inAudibleLite/inAudibleLite.csproj
View File

@ -7,8 +7,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\Dinah.Core\Dinah.Core.Drawing\Dinah.Core.Drawing.csproj" />
<ProjectReference Include="..\..\..\Dinah.Core\Dinah.Core.Windows.Forms\Dinah.Core.Windows.Forms.csproj" />
<ProjectReference Include="..\..\..\Dinah.Core\Dinah.Core.WindowsDesktop\Dinah.Core.WindowsDesktop.csproj" />
<ProjectReference Include="..\..\AaxDecrypter\AaxDecrypter.csproj" />
</ItemGroup>

2
__TODO.txt
View File

@ -1,5 +1,5 @@
-- begin BETA ---------------------------------------------------------------------------------------------------------------------
throttle needed for downloading images, pdf.s
throttle needed for downloading pdf.s
no mdf,ldf files created in C:\Users\[username]