From 9ca58cc81429a5057e3a8b0e0f9f66afc61f8b97 Mon Sep 17 00:00:00 2001 From: Tsanie Lily Date: Thu, 21 May 2020 21:44:10 +0800 Subject: [PATCH] optimize parallel task, and clear share bookmarkId --- Pixiview/App.cs | 4 + Pixiview/AppShell.xaml.cs | 2 +- Pixiview/Illust/FavoritesPage.xaml.cs | 111 +++++++++++++++++++++--- Pixiview/Illust/IllustCollectionPage.cs | 49 ++++++++--- Pixiview/Illust/MainPage.xaml.cs | 8 +- Pixiview/Illust/RecommendsPage.xaml.cs | 6 +- Pixiview/Illust/ViewIllustPage.xaml.cs | 4 +- Pixiview/Utils/Extensions.cs | 55 +++++++++--- Pixiview/Utils/Stores.cs | 68 ++++++++++----- 9 files changed, 237 insertions(+), 70 deletions(-) diff --git a/Pixiview/App.cs b/Pixiview/App.cs index dd4a760..b2df18c 100644 --- a/Pixiview/App.cs +++ b/Pixiview/App.cs @@ -170,6 +170,10 @@ namespace Pixiview try { favObj = Stores.LoadFavoritesIllusts(url); + foreach (var o in favObj.Illusts) + { + o.BookmarkId = null; + } } catch (Exception ex) { diff --git a/Pixiview/AppShell.xaml.cs b/Pixiview/AppShell.xaml.cs index 309a5e7..1ae5c40 100644 --- a/Pixiview/AppShell.xaml.cs +++ b/Pixiview/AppShell.xaml.cs @@ -175,7 +175,7 @@ namespace Pixiview UserProfileImage = img == null ? StyleDefinition.ProfileNone - : Stores.LoadUserProfileImage(img); + : Stores.LoadUserProfileImage(img, true); } } diff --git a/Pixiview/Illust/FavoritesPage.xaml.cs b/Pixiview/Illust/FavoritesPage.xaml.cs index 6a15402..0aba5d6 100644 --- a/Pixiview/Illust/FavoritesPage.xaml.cs +++ b/Pixiview/Illust/FavoritesPage.xaml.cs @@ -33,6 +33,7 @@ namespace Pixiview.Illust private int startIndex; private int nextIndex; private bool flag = false; + private ParallelTask task; public FavoritesPage() { @@ -67,6 +68,16 @@ namespace Pixiview.Illust } } + public override void OnUnload() + { + if (task != null) + { + task.Dispose(); + task = null; + } + base.OnUnload(); + } + protected override IEnumerable DoGetIllustList(IllustItem[] data) { return data; @@ -300,14 +311,13 @@ namespace Pixiview.Illust if (list.Length > 0) { +#if LOG for (var i = 0; i < list.Length; i++) { var item = list[i]; -#if LOG App.DebugPrint($"add bookmark ({item.BookmarkId}) - {item.Id}: {item.Title}"); -#endif - item.Image = StyleDefinition.DownloadBackground; } +#endif nows.InsertRange(0, list); } } @@ -316,21 +326,98 @@ namespace Pixiview.Illust return; } - ParallelTask.Start("favorite.loadimages", 0, list.Length, Configs.MaxPageThreads, i => + SyncRemoteFavorites(list); + } + + private void SyncRemoteFavorites(IllustItem[] list) + { + for (var i = 0; i < list.Length; i++) { - var illustItem = list[i]; - var data = Stores.LoadIllustPreloadData(illustItem.Id); - if (data != null && data.illust.TryGetValue(illustItem.Id, out var illust)) + var item = list[i]; + var data = Stores.LoadIllustPreloadData(item.Id, false); + if (data != null && data.illust.TryGetValue(item.Id, out var illust)) { - illust.CopyToItem(illustItem); + illust.CopyToItem(item); if (data.user.TryGetValue(illust.userId, out var user)) { - illustItem.ProfileUrl = user.image; + item.ProfileUrl = user.image; + } + var url = item.ImageUrl; + if (url != null) + { + var image = Stores.LoadPreviewImage(url, false); + if (image == null) + { + image = Stores.LoadThumbnailImage(url, false); + } + if (image != null) + { + item.Image = image; + } + } + url = item.ProfileUrl; + if (url == null) + { + item.ProfileImage = StyleDefinition.ProfileNone; + } + else + { + var image = Stores.LoadUserProfileImage(url, false); + if (image != null) + { + item.ProfileImage = image; + } } - return true; } - App.DebugError("load.favorite", $"cannot fetch preload data, {illustItem.Id}, disposed."); - return false; + } + + list = list.Where(i => i.Image == null || i.ProfileImage == null).ToArray(); + + if (task != null) + { + task.Dispose(); + task = null; + } + + task = ParallelTask.Start("favorite.loadimages", 0, list.Length, Configs.MaxPageThreads, i => + { + var item = list[i]; + if (item.ImageUrl == null) + { + var data = Stores.LoadIllustPreloadData(item.Id, true, force: true); + if (data != null && data.illust.TryGetValue(item.Id, out var illust)) + { + illust.CopyToItem(item); + if (data.user.TryGetValue(illust.userId, out var user)) + { + item.ProfileUrl = user.image; + } + } + else + { + App.DebugError("load.favorite", $"cannot fetch preload data, {item.Id}, disposed."); + return false; + } + } + if (item.Image == null && item.ImageUrl != null) + { + item.Image = StyleDefinition.DownloadBackground; + var image = Stores.LoadThumbnailImage(item.ImageUrl, true, force: true); + if (image != null) + { + item.Image = image; + } + } + if (item.ProfileImage == null && item.ProfileUrl != null) + { + item.ProfileImage = StyleDefinition.ProfileNone; + var userImage = Stores.LoadUserProfileImage(item.ProfileUrl, true, force: true); + if (userImage != null) + { + item.ProfileImage = userImage; + } + } + return true; }, () => { Stores.SaveFavoritesIllusts(); diff --git a/Pixiview/Illust/IllustCollectionPage.cs b/Pixiview/Illust/IllustCollectionPage.cs index 42ca64c..727f058 100644 --- a/Pixiview/Illust/IllustCollectionPage.cs +++ b/Pixiview/Illust/IllustCollectionPage.cs @@ -598,7 +598,32 @@ namespace Pixiview.Illust { if (item.Image == null) { - item.Image = StyleDefinition.DownloadBackground; + var url = Configs.GetThumbnailUrl(item.ImageUrl); + var image = Stores.LoadPreviewImage(url, false); + if (image == null) + { + image = Stores.LoadThumbnailImage(url, false); + } + if (image != null) + { + item.Image = image; + } + item.ImageUrl = url; + } + if (item.ProfileImage == null) + { + if (item.ProfileUrl == null) + { + item.ProfileImage = StyleDefinition.ProfileNone; + } + else + { + var image = Stores.LoadUserProfileImage(item.ProfileUrl, false); + if (image != null) + { + item.ProfileImage = image; + } + } } item.IsFavorite = item.BookmarkId != null || @@ -616,30 +641,30 @@ namespace Pixiview.Illust task.Dispose(); task = null; } - task = ParallelTask.Start("collection.load", 0, collection.Count, Configs.MaxThreads, i => + var list = collection.Where(i => i.Image == null || i.ProfileImage == null).ToArray(); + + task = ParallelTask.Start("collection.load", 0, list.Length, Configs.MaxThreads, i => { if (!collection.Running) { return false; } - var illust = collection[i]; - if (illust.ImageUrl != null) + var illust = list[i]; + + if (illust.Image == null && illust.ImageUrl != null) { - var url = Configs.GetThumbnailUrl(illust.ImageUrl); - var image = Stores.LoadPreviewImage(url, false); - if (image == null) - { - image = Stores.LoadThumbnailImage(url); - } + illust.Image = StyleDefinition.DownloadBackground; + var image = Stores.LoadThumbnailImage(illust.ImageUrl, true, force: true); if (image != null) { illust.Image = image; } } - if (illust.ProfileUrl != null) + if (illust.ProfileImage == null && illust.ProfileUrl != null) { - var userImage = Stores.LoadUserProfileImage(illust.ProfileUrl); + illust.ProfileImage = StyleDefinition.ProfileNone; + var userImage = Stores.LoadUserProfileImage(illust.ProfileUrl, true, force: true); if (userImage != null) { illust.ProfileImage = userImage; diff --git a/Pixiview/Illust/MainPage.xaml.cs b/Pixiview/Illust/MainPage.xaml.cs index fcf7565..dddf4b8 100644 --- a/Pixiview/Illust/MainPage.xaml.cs +++ b/Pixiview/Illust/MainPage.xaml.cs @@ -62,8 +62,9 @@ namespace Pixiview.Illust { return null; } - return data.body.page.follow.Select(i => + var illusts = data.body.page.follow.Select(i => data.body.thumbnails.illust.FirstOrDefault(l => l.illustId == i.ToString())?.ConvertToItem()); + return illusts; } protected override IllustData DoLoadIllustData(bool force) @@ -78,7 +79,10 @@ namespace Pixiview.Illust return; } await ScrollToTopAsync(scrollView); - StartLoad(true); + + + lastUpdated = default; + StartLoad(false); } private void SearchBar_SearchButtonPressed(object sender, EventArgs e) diff --git a/Pixiview/Illust/RecommendsPage.xaml.cs b/Pixiview/Illust/RecommendsPage.xaml.cs index b7f8eb5..0a9e703 100644 --- a/Pixiview/Illust/RecommendsPage.xaml.cs +++ b/Pixiview/Illust/RecommendsPage.xaml.cs @@ -308,7 +308,7 @@ namespace Pixiview.Illust { if (user.ProfileUrl != null) { - var userImage = Stores.LoadUserProfileImage(user.ProfileUrl); + var userImage = Stores.LoadUserProfileImage(user.ProfileUrl, true); if (userImage != null) { user.ProfileImage = userImage; @@ -326,10 +326,10 @@ namespace Pixiview.Illust if (illust.ImageUrl != null) { var url = Configs.GetThumbnailUrl(illust.ImageUrl); - var image = Stores.LoadPreviewImage(url, false); + var image = Stores.LoadPreviewImage(url, true, force: false); if (image == null) { - image = Stores.LoadThumbnailImage(url); + image = Stores.LoadThumbnailImage(url, true); } if (image != null) { diff --git a/Pixiview/Illust/ViewIllustPage.xaml.cs b/Pixiview/Illust/ViewIllustPage.xaml.cs index 8c19cdb..5f4d25f 100644 --- a/Pixiview/Illust/ViewIllustPage.xaml.cs +++ b/Pixiview/Illust/ViewIllustPage.xaml.cs @@ -439,7 +439,7 @@ namespace Pixiview.Illust } } item.Loading = true; - var image = Stores.LoadPreviewImage(item.PreviewUrl, force: force); + var image = Stores.LoadPreviewImage(item.PreviewUrl, true, force: force); if (image != null) { item.Image = image; @@ -649,7 +649,7 @@ namespace Pixiview.Illust isPreloading = true; var illustItem = IllustItem; // force to reload - var preload = Stores.LoadIllustPreloadData(illustItem.Id, force); + var preload = Stores.LoadIllustPreloadData(illustItem.Id, true, force: force); if (preload != null && preload.illust.TryGetValue(illustItem.Id, out var illust)) { illust.CopyToItem(illustItem); diff --git a/Pixiview/Utils/Extensions.cs b/Pixiview/Utils/Extensions.cs index b0650cf..b81ced3 100644 --- a/Pixiview/Utils/Extensions.cs +++ b/Pixiview/Utils/Extensions.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Xamarin.Forms; @@ -108,9 +107,7 @@ namespace Pixiview.Utils return null; } var task = new ParallelTask(tag, from, toExclusive, maxCount, action, complete); - Task.Run(task.Start); - //ThreadPool.QueueUserWorkItem(task.Start); - return task; + return task.Start(); } private readonly object sync = new object(); @@ -142,9 +139,19 @@ namespace Pixiview.Utils this.complete = complete; } - private void Start() + public ParallelTask Start() { - var list = new List(); + _ = ThreadPool.QueueUserWorkItem(DoStart); + return this; + } + + private void DoStart(object state) + { +#if LOG + var sw = new System.Diagnostics.Stopwatch(); + long lastElapsed = 0; + sw.Start(); +#endif for (int i = from; i < to; i++) { var index = i; @@ -154,20 +161,30 @@ namespace Pixiview.Utils { break; } +#if LOG + var elapsed = sw.ElapsedMilliseconds; + if (elapsed - lastElapsed > 60000) + { + lastElapsed = elapsed; + App.DebugPrint($"WARNING: parallel task ({tag}), {count} tasks in queue, cost too much time ({elapsed:n0}ms)"); + } +#endif if (disposed) { #if LOG - App.DebugPrint($"parallel task determinate, disposed ({tag})"); + sw.Stop(); + App.DebugPrint($"parallel task determinate, disposed ({tag}), cost time ({elapsed:n0}ms)"); #endif return; } - Thread.Sleep(100); + Thread.Sleep(16); } lock (sync) { count++; } - list.Add(Task.Run(() => + ThreadPool.QueueUserWorkItem(o => + //Task.Run(() => { try { @@ -178,7 +195,7 @@ namespace Pixiview.Utils } catch (Exception ex) { - App.DebugError("parallel.start ({tag})", $"failed to run action, index: {i}, error: {ex.Message}"); + App.DebugError($"parallel.start ({tag})", $"failed to run action, index: {index}, error: {ex}"); } finally { @@ -187,21 +204,31 @@ namespace Pixiview.Utils count--; } } - })); + }); } while (count > 0) { +#if LOG + var elapsed = sw.ElapsedMilliseconds; + if (elapsed - lastElapsed > 60000) + { + lastElapsed = elapsed; + App.DebugPrint($"WARNING: parallel task ({tag}), {count} tasks are waiting for end, cost too much time ({elapsed:n0}ms)"); + } +#endif if (disposed) { #if LOG - App.DebugPrint($"parallel task determinate, disposed ({tag})"); + sw.Stop(); + App.DebugPrint($"parallel task determinate, disposed ({tag}), cost time ({elapsed:n0}ms)"); #endif return; } - Thread.Sleep(100); + Thread.Sleep(16); } #if LOG - App.DebugPrint($"parallel task done ({tag})"); + sw.Stop(); + App.DebugPrint($"parallel task done ({tag}), cost time ({sw.ElapsedMilliseconds:n0}ms)"); #endif complete?.Invoke(); } diff --git a/Pixiview/Utils/Stores.cs b/Pixiview/Utils/Stores.cs index 7192d1c..bcbcf7f 100644 --- a/Pixiview/Utils/Stores.cs +++ b/Pixiview/Utils/Stores.cs @@ -379,29 +379,49 @@ namespace Pixiview.Utils return true; } - public static IllustPreloadBody LoadIllustPreloadData(string id, bool force = false) + public static IllustPreloadBody LoadIllustPreloadData(string id, bool downloading, bool force = false) { var file = Path.Combine(CacheFolder, preloadsFolder, $"{id}.json"); - var result = HttpUtility.LoadObject( - file, - string.Format(Configs.UrlIllust, id), - null, - out _, - force: force, - action: content => + IllustPreloadBody result; + if (!force) + { + result = ReadObject(file); + if (result != null) { - var index = content.IndexOf(Configs.SuffixPreload); - if (index > 0) + return result; + } + else if (!downloading) + { + return null; + } + } + if (downloading) + { + result = HttpUtility.LoadObject( + file, + string.Format(Configs.UrlIllust, id), + null, + out _, + force: force, + action: content => { - index += Configs.SuffixPreloadLength; - var end = content.IndexOf('\'', index); - if (end > index) + var index = content.IndexOf(Configs.SuffixPreload); + if (index > 0) { - content = content.Substring(index, end - index); + index += Configs.SuffixPreloadLength; + var end = content.IndexOf('\'', index); + if (end > index) + { + content = content.Substring(index, end - index); + } } - } - return content; - }); + return content; + }); + } + else + { + result = null; + } if (result == null) { App.DebugPrint($"error when load preload data: force({force})"); @@ -567,22 +587,22 @@ namespace Pixiview.Utils public static ImageSource LoadIllustImage(string url) { - return LoadImage(url, PersonalFolder, imageFolder); + return LoadImage(url, PersonalFolder, imageFolder, true); } - public static ImageSource LoadPreviewImage(string url, bool downloading = true, bool force = false) + public static ImageSource LoadPreviewImage(string url, bool downloading, bool force = false) { return LoadImage(url, PersonalFolder, previewFolder, downloading, force); } - public static ImageSource LoadThumbnailImage(string url) + public static ImageSource LoadThumbnailImage(string url, bool downloading, bool force = false) { - return LoadImage(url, CacheFolder, thumbFolder); + return LoadImage(url, CacheFolder, thumbFolder, downloading, force); } - public static ImageSource LoadUserProfileImage(string url) + public static ImageSource LoadUserProfileImage(string url, bool downloading, bool force = false) { - return LoadImage(url, CacheFolder, userFolder); + return LoadImage(url, CacheFolder, userFolder, downloading, force); } public static bool CheckIllustImage(string url) @@ -606,7 +626,7 @@ namespace Pixiview.Utils return null; } - private static ImageSource LoadImage(string url, string working, string folder, bool downloading = true, bool force = false) + private static ImageSource LoadImage(string url, string working, string folder, bool downloading, bool force = false) { var file = Path.Combine(working, folder, Path.GetFileName(url)); ImageSource image;