From ce318347336a8e93a8579f7465418ab3c9b70277 Mon Sep 17 00:00:00 2001 From: Tsanie Lily Date: Mon, 25 May 2020 00:44:39 +0800 Subject: [PATCH] feature: parallels to download illusts --- .../Properties/AndroidManifest.xml | 2 +- Pixiview.iOS.OpenExtension/Info.plist | 4 +- Pixiview.iOS/Info.plist | 4 +- Pixiview/Illust/IllustCollectionPage.cs | 7 +- Pixiview/Illust/RecommendsPage.xaml.cs | 2 +- Pixiview/Illust/ViewIllustPage.xaml.cs | 4 +- Pixiview/Utils/Extensions.cs | 6 +- Pixiview/Utils/HttpUtility.cs | 108 ++++++++++++++++++ Pixiview/Utils/Stores.cs | 43 ++++++- 9 files changed, 164 insertions(+), 16 deletions(-) diff --git a/Pixiview.Android/Properties/AndroidManifest.xml b/Pixiview.Android/Properties/AndroidManifest.xml index ab7e634..5a146a3 100644 --- a/Pixiview.Android/Properties/AndroidManifest.xml +++ b/Pixiview.Android/Properties/AndroidManifest.xml @@ -1,5 +1,5 @@  - + diff --git a/Pixiview.iOS.OpenExtension/Info.plist b/Pixiview.iOS.OpenExtension/Info.plist index 2107d99..2f7adf2 100644 --- a/Pixiview.iOS.OpenExtension/Info.plist +++ b/Pixiview.iOS.OpenExtension/Info.plist @@ -29,8 +29,8 @@ com.apple.share-services CFBundleShortVersionString - 1.0.524 + 1.0.525 CFBundleVersion - 16 + 18 diff --git a/Pixiview.iOS/Info.plist b/Pixiview.iOS/Info.plist index 98e6103..42bfe5e 100644 --- a/Pixiview.iOS/Info.plist +++ b/Pixiview.iOS/Info.plist @@ -79,9 +79,9 @@ CFBundleShortVersionString - 1.0.524 + 1.0.525 CFBundleVersion - 16 + 18 CFBundleDevelopmentRegion China diff --git a/Pixiview/Illust/IllustCollectionPage.cs b/Pixiview/Illust/IllustCollectionPage.cs index a0351fc..1b5b64f 100644 --- a/Pixiview/Illust/IllustCollectionPage.cs +++ b/Pixiview/Illust/IllustCollectionPage.cs @@ -717,9 +717,12 @@ namespace Pixiview.Illust return true; }, tagIndex: tag); - lock (sync) + if (task != null) { - tasks.Enqueue(task); + lock (sync) + { + tasks.Enqueue(task); + } } } diff --git a/Pixiview/Illust/RecommendsPage.xaml.cs b/Pixiview/Illust/RecommendsPage.xaml.cs index 5b0aeee..06033aa 100644 --- a/Pixiview/Illust/RecommendsPage.xaml.cs +++ b/Pixiview/Illust/RecommendsPage.xaml.cs @@ -315,7 +315,7 @@ namespace Pixiview.Illust if (illust.ImageUrl != null) { var url = Configs.GetThumbnailUrl(illust.ImageUrl); - var image = Stores.LoadPreviewImage(url, true, force: false); + var image = Stores.LoadPreviewImage(url, false); if (image == null) { image = Stores.LoadThumbnailImage(url, true); diff --git a/Pixiview/Illust/ViewIllustPage.xaml.cs b/Pixiview/Illust/ViewIllustPage.xaml.cs index c6f45ee..7d6c279 100644 --- a/Pixiview/Illust/ViewIllustPage.xaml.cs +++ b/Pixiview/Illust/ViewIllustPage.xaml.cs @@ -400,7 +400,7 @@ namespace Pixiview.Illust task.Dispose(); task = null; } - task = ParallelTask.Start("view.illusts", 0, items.Length, Configs.MaxPageThreads, i => + task = ParallelTask.Start("view.illusts", 0, items.Length, 2, i => { DoLoadImage(i); return true; @@ -439,7 +439,7 @@ namespace Pixiview.Illust } } item.Loading = true; - var image = Stores.LoadPreviewImage(item.PreviewUrl, true, force: force); + var image = Stores.LoadPreviewImage(item.PreviewUrl, true, IllustItem.Id, force: force); if (image != null) { item.Image = image; diff --git a/Pixiview/Utils/Extensions.cs b/Pixiview/Utils/Extensions.cs index c5184cf..d288abf 100644 --- a/Pixiview/Utils/Extensions.cs +++ b/Pixiview/Utils/Extensions.cs @@ -107,7 +107,8 @@ namespace Pixiview.Utils return null; } var task = new ParallelTask(tag, from, toExclusive, maxCount, action, tagIndex, complete); - return task.Start(); + task.Start(); + return task; } private readonly object sync = new object(); @@ -141,10 +142,9 @@ namespace Pixiview.Utils this.complete = complete; } - public ParallelTask Start() + public void Start() { _ = ThreadPool.QueueUserWorkItem(DoStart); - return this; } private void DoStart(object state) diff --git a/Pixiview/Utils/HttpUtility.cs b/Pixiview/Utils/HttpUtility.cs index 093f613..be5bf9d 100644 --- a/Pixiview/Utils/HttpUtility.cs +++ b/Pixiview/Utils/HttpUtility.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -6,6 +7,7 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; +using System.Threading.Tasks; using Newtonsoft.Json; namespace Pixiview.Utils @@ -204,6 +206,112 @@ namespace Pixiview.Utils } } + public static Task DownloadImageAsync(string url, string id, string working, string folder) + { + try + { + var directory = Path.Combine(working, folder); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + var file = Path.Combine(directory, Path.GetFileName(url)); + var proxy = Configs.Proxy; + var referer = new Uri(string.Format(Configs.UrlIllust, id)); + var handler = new HttpClientHandler + { + UseCookies = false + }; + if (proxy != null) + { + handler.Proxy = proxy; + handler.UseProxy = true; + } + var client = new HttpClient(handler) + { + Timeout = TimeSpan.FromSeconds(30) + }; + long size; + DateTimeOffset lastModified; + using (var request = new HttpRequestMessage(HttpMethod.Head, url)) + { + var headers = request.Headers; + headers.Add("Accept", Configs.AcceptPureImage); + headers.Add("Accept-Language", Configs.AcceptLanguage); + headers.Referrer = referer; + headers.Add("User-Agent", Configs.UserAgent); + using (var response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result) + { + size = response.Content.Headers.ContentLength.Value; + lastModified = response.Content.Headers.LastModified.Value; +#if DEBUG + App.DebugPrint($"content length: {size:n0} bytes, last modified: {lastModified}"); +#endif + } + } + + // segments + const int SIZE = 50000; + var list = new List<(long from, long to)>(); + for (var i = 0L; i < size; i += SIZE) + { + long to; + if (i + SIZE >= size) + { + to = size - 1; + } + else + { + to = i + SIZE - 1; + } + list.Add((i, to)); + } + + var data = new byte[size]; + var task = new TaskCompletionSource(); + + ParallelTask.Start($"download.async.{id}", 0, list.Count, 6, i => + { + var (from, to) = list[i]; + using (var request = new HttpRequestMessage(HttpMethod.Get, url)) + { + var headers = request.Headers; + headers.Add("Accept", Configs.AcceptPureImage); + headers.Add("Accept-Language", Configs.AcceptLanguage); + headers.Add("Accept-Encoding", "identity"); + headers.IfRange = new RangeConditionHeaderValue(lastModified); + headers.Range = new RangeHeaderValue(from, to); + headers.Referrer = referer; + headers.Add("User-Agent", Configs.UserAgent); + using (var response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result) + using (var ms = new MemoryStream(data, (int)from, (int)(to - from + 1))) + { + response.Content.CopyToAsync(ms).Wait(); +#if DEBUG + App.DebugPrint($"downloaded range: from ({from:n0}) to ({to:n0})"); +#endif + } + } + return true; + }, + complete: () => + { + using (var fs = File.OpenWrite(file)) + { + fs.Write(data, 0, data.Length); + } + task.SetResult(file); + }); + + return task.Task; + } + catch (Exception ex) + { + App.DebugError("image.download.async", ex.Message); + return Task.FromResult(null); + } + } + private static HttpResponseMessage Download(string url, Action headerAction, HttpContent post = null) { #if DEBUG diff --git a/Pixiview/Utils/Stores.cs b/Pixiview/Utils/Stores.cs index 289d0e9..da6db8e 100644 --- a/Pixiview/Utils/Stores.cs +++ b/Pixiview/Utils/Stores.cs @@ -6,6 +6,7 @@ using System.Net; using System.Net.Http; using System.Text; using System.Text.RegularExpressions; +using System.Threading.Tasks; using Newtonsoft.Json; using Pixiview.Illust; using Xamarin.Essentials; @@ -590,8 +591,12 @@ namespace Pixiview.Utils return LoadImage(url, PersonalFolder, imageFolder, true); } - public static ImageSource LoadPreviewImage(string url, bool downloading, bool force = false) + public static ImageSource LoadPreviewImage(string url, bool downloading, string id = null, bool force = false) { + if (downloading) + { + return LoadImageAsync(url, id, PersonalFolder, previewFolder, force).Result; + } return LoadImage(url, PersonalFolder, previewFolder, downloading, force); } @@ -656,6 +661,37 @@ namespace Pixiview.Utils } return image; } + + private static Task LoadImageAsync(string url, string id, string working, string folder, bool force = false) + { + var file = Path.Combine(working, folder, Path.GetFileName(url)); + ImageSource image; + if (!force && File.Exists(file)) + { + try + { + image = ImageSource.FromFile(file); + } + catch (Exception ex) + { + App.DebugError("image.load", $"failed to load image from file: {file}, error: {ex.Message}"); + image = null; + } + } + else + { + image = null; + } + if (image == null) + { + file = HttpUtility.DownloadImageAsync(url, id, working, folder).Result; + if (file != null) + { + image = ImageSource.FromFile(file); + } + } + return Task.FromResult(image); + } } public class IllustFavorite @@ -772,9 +808,9 @@ namespace Pixiview.Utils } #if __IOS__ - public static System.Threading.Tasks.Task RequestCookieContainer(WebKit.WKHttpCookieStore cookieStore) + public static Task RequestCookieContainer(WebKit.WKHttpCookieStore cookieStore) { - var task = new System.Threading.Tasks.TaskCompletionSource(); + var task = new TaskCompletionSource(); cookieStore.GetAllCookies(cookies => { var list = new List(); @@ -825,6 +861,7 @@ namespace Pixiview.Utils public const string UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"; public const string AcceptImage = "image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5"; + public const string AcceptPureImage = "image/*,*/*;q=0.8"; public const string AcceptJson = "application/json"; public const string AcceptUrlEncoded = "application/x-www-form-urlencoded"; //public const string AcceptEncoding = "gzip, deflate";