tsanie/Pixiview
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

optimize parallel task, and clear share bookmarkId

Browse Source
This commit is contained in:
Tsanie Lily 2020-05-21 21:44:10 +08:00
parent f8d85ef729
commit 9ca58cc814
9 changed files with 237 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Pixiview/App.cs
View File

@ -170,6 +170,10 @@ namespace Pixiview
try
{
favObj = Stores.LoadFavoritesIllusts(url);
foreach (var o in favObj.Illusts)
{
o.BookmarkId = null;
}
}
catch (Exception ex)
{

2
Pixiview/AppShell.xaml.cs
View File

@ -175,7 +175,7 @@ namespace Pixiview
UserProfileImage = img == null
? StyleDefinition.ProfileNone
: Stores.LoadUserProfileImage(img);
: Stores.LoadUserProfileImage(img, true);
}
}

111
Pixiview/Illust/FavoritesPage.xaml.cs
View File

@ -33,6 +33,7 @@ namespace Pixiview.Illust
private int startIndex;
private int nextIndex;
private bool flag = false;
private ParallelTask task;
public FavoritesPage()
{
@ -67,6 +68,16 @@ namespace Pixiview.Illust
}
}
public override void OnUnload()
{
if (task != null)
{
task.Dispose();
task = null;
}
base.OnUnload();
}
protected override IEnumerable<IllustItem> DoGetIllustList(IllustItem[] data)
{
return data;
@ -300,14 +311,13 @@ namespace Pixiview.Illust
if (list.Length > 0)
{
#if LOG
for (var i = 0; i < list.Length; i++)
{
var item = list[i];
#if LOG
App.DebugPrint($"add bookmark ({item.BookmarkId}) - {item.Id}: {item.Title}");
#endif
item.Image = StyleDefinition.DownloadBackground;
}
#endif
nows.InsertRange(0, list);
}
}
@ -316,21 +326,98 @@ namespace Pixiview.Illust
return;
}
ParallelTask.Start("favorite.loadimages", 0, list.Length, Configs.MaxPageThreads, i =>
SyncRemoteFavorites(list);
}
private void SyncRemoteFavorites(IllustItem[] list)
{
for (var i = 0; i < list.Length; i++)
{
var illustItem = list[i];
var data = Stores.LoadIllustPreloadData(illustItem.Id);
if (data != null && data.illust.TryGetValue(illustItem.Id, out var illust))
var item = list[i];
var data = Stores.LoadIllustPreloadData(item.Id, false);
if (data != null && data.illust.TryGetValue(item.Id, out var illust))
{
illust.CopyToItem(illustItem);
illust.CopyToItem(item);
if (data.user.TryGetValue(illust.userId, out var user))
{
illustItem.ProfileUrl = user.image;
item.ProfileUrl = user.image;
}
var url = item.ImageUrl;
if (url != null)
{
var image = Stores.LoadPreviewImage(url, false);
if (image == null)
{
image = Stores.LoadThumbnailImage(url, false);
}
if (image != null)
{
item.Image = image;
}
}
url = item.ProfileUrl;
if (url == null)
{
item.ProfileImage = StyleDefinition.ProfileNone;
}
else
{
var image = Stores.LoadUserProfileImage(url, false);
if (image != null)
{
item.ProfileImage = image;
}
}
return true;
}
App.DebugError("load.favorite", $"cannot fetch preload data, {illustItem.Id}, disposed.");
return false;
}
list = list.Where(i => i.Image == null || i.ProfileImage == null).ToArray();
if (task != null)
{
task.Dispose();
task = null;
}
task = ParallelTask.Start("favorite.loadimages", 0, list.Length, Configs.MaxPageThreads, i =>
{
var item = list[i];
if (item.ImageUrl == null)
{
var data = Stores.LoadIllustPreloadData(item.Id, true, force: true);
if (data != null && data.illust.TryGetValue(item.Id, out var illust))
{
illust.CopyToItem(item);
if (data.user.TryGetValue(illust.userId, out var user))
{
item.ProfileUrl = user.image;
}
}
else
{
App.DebugError("load.favorite", $"cannot fetch preload data, {item.Id}, disposed.");
return false;
}
}
if (item.Image == null && item.ImageUrl != null)
{
item.Image = StyleDefinition.DownloadBackground;
var image = Stores.LoadThumbnailImage(item.ImageUrl, true, force: true);
if (image != null)
{
item.Image = image;
}
}
if (item.ProfileImage == null && item.ProfileUrl != null)
{
item.ProfileImage = StyleDefinition.ProfileNone;
var userImage = Stores.LoadUserProfileImage(item.ProfileUrl, true, force: true);
if (userImage != null)
{
item.ProfileImage = userImage;
}
}
return true;
}, () =>
{
Stores.SaveFavoritesIllusts();

49
Pixiview/Illust/IllustCollectionPage.cs
View File

@ -598,7 +598,32 @@ namespace Pixiview.Illust
{
if (item.Image == null)
{
item.Image = StyleDefinition.DownloadBackground;
var url = Configs.GetThumbnailUrl(item.ImageUrl);
var image = Stores.LoadPreviewImage(url, false);
if (image == null)
{
image = Stores.LoadThumbnailImage(url, false);
}
if (image != null)
{
item.Image = image;
}
item.ImageUrl = url;
}
if (item.ProfileImage == null)
{
if (item.ProfileUrl == null)
{
item.ProfileImage = StyleDefinition.ProfileNone;
}
else
{
var image = Stores.LoadUserProfileImage(item.ProfileUrl, false);
if (image != null)
{
item.ProfileImage = image;
}
}
}
item.IsFavorite =
item.BookmarkId != null ||
@ -616,30 +641,30 @@ namespace Pixiview.Illust
task.Dispose();
task = null;
}
task = ParallelTask.Start("collection.load", 0, collection.Count, Configs.MaxThreads, i =>
var list = collection.Where(i => i.Image == null || i.ProfileImage == null).ToArray();
task = ParallelTask.Start("collection.load", 0, list.Length, Configs.MaxThreads, i =>
{
if (!collection.Running)
{
return false;
}
var illust = collection[i];
if (illust.ImageUrl != null)
var illust = list[i];
if (illust.Image == null && illust.ImageUrl != null)
{
var url = Configs.GetThumbnailUrl(illust.ImageUrl);
var image = Stores.LoadPreviewImage(url, false);
if (image == null)
{
image = Stores.LoadThumbnailImage(url);
}
illust.Image = StyleDefinition.DownloadBackground;
var image = Stores.LoadThumbnailImage(illust.ImageUrl, true, force: true);
if (image != null)
{
illust.Image = image;
}
}
if (illust.ProfileUrl != null)
if (illust.ProfileImage == null && illust.ProfileUrl != null)
{
var userImage = Stores.LoadUserProfileImage(illust.ProfileUrl);
illust.ProfileImage = StyleDefinition.ProfileNone;
var userImage = Stores.LoadUserProfileImage(illust.ProfileUrl, true, force: true);
if (userImage != null)
{
illust.ProfileImage = userImage;

8
Pixiview/Illust/MainPage.xaml.cs
View File

@ -62,8 +62,9 @@ namespace Pixiview.Illust
{
return null;
}
return data.body.page.follow.Select(i =>
var illusts = data.body.page.follow.Select(i =>
data.body.thumbnails.illust.FirstOrDefault(l => l.illustId == i.ToString())?.ConvertToItem());
return illusts;
}
protected override IllustData DoLoadIllustData(bool force)
@ -78,7 +79,10 @@ namespace Pixiview.Illust
return;
}
await ScrollToTopAsync(scrollView);
StartLoad(true);
lastUpdated = default;
StartLoad(false);
}
private void SearchBar_SearchButtonPressed(object sender, EventArgs e)

6
Pixiview/Illust/RecommendsPage.xaml.cs
View File

@ -308,7 +308,7 @@ namespace Pixiview.Illust
{
if (user.ProfileUrl != null)
{
var userImage = Stores.LoadUserProfileImage(user.ProfileUrl);
var userImage = Stores.LoadUserProfileImage(user.ProfileUrl, true);
if (userImage != null)
{
user.ProfileImage = userImage;
@ -326,10 +326,10 @@ namespace Pixiview.Illust
if (illust.ImageUrl != null)
{
var url = Configs.GetThumbnailUrl(illust.ImageUrl);
var image = Stores.LoadPreviewImage(url, false);
var image = Stores.LoadPreviewImage(url, true, force: false);
if (image == null)
{
image = Stores.LoadThumbnailImage(url);
image = Stores.LoadThumbnailImage(url, true);
}
if (image != null)
{

4
Pixiview/Illust/ViewIllustPage.xaml.cs
View File

@ -439,7 +439,7 @@ namespace Pixiview.Illust
}
}
item.Loading = true;
var image = Stores.LoadPreviewImage(item.PreviewUrl, force: force);
var image = Stores.LoadPreviewImage(item.PreviewUrl, true, force: force);
if (image != null)
{
item.Image = image;
@ -649,7 +649,7 @@ namespace Pixiview.Illust
isPreloading = true;
var illustItem = IllustItem;
// force to reload
var preload = Stores.LoadIllustPreloadData(illustItem.Id, force);
var preload = Stores.LoadIllustPreloadData(illustItem.Id, true, force: force);
if (preload != null && preload.illust.TryGetValue(illustItem.Id, out var illust))
{
illust.CopyToItem(illustItem);

55
Pixiview/Utils/Extensions.cs
View File

@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Xamarin.Forms;
@ -108,9 +107,7 @@ namespace Pixiview.Utils
return null;
}
var task = new ParallelTask(tag, from, toExclusive, maxCount, action, complete);
Task.Run(task.Start);
//ThreadPool.QueueUserWorkItem(task.Start);
return task;
return task.Start();
}
private readonly object sync = new object();
@ -142,9 +139,19 @@ namespace Pixiview.Utils
this.complete = complete;
}
private void Start()
public ParallelTask Start()
{
var list = new List<Task>();
_ = ThreadPool.QueueUserWorkItem(DoStart);
return this;
}
private void DoStart(object state)
{
#if LOG
var sw = new System.Diagnostics.Stopwatch();
long lastElapsed = 0;
sw.Start();
#endif
for (int i = from; i < to; i++)
{
var index = i;
@ -154,20 +161,30 @@ namespace Pixiview.Utils
{
break;
}
#if LOG
var elapsed = sw.ElapsedMilliseconds;
if (elapsed - lastElapsed > 60000)
{
lastElapsed = elapsed;
App.DebugPrint($"WARNING: parallel task ({tag}), {count} tasks in queue, cost too much time ({elapsed:n0}ms)");
}
#endif
if (disposed)
{
#if LOG
App.DebugPrint($"parallel task determinate, disposed ({tag})");
sw.Stop();
App.DebugPrint($"parallel task determinate, disposed ({tag}), cost time ({elapsed:n0}ms)");
#endif
return;
}
Thread.Sleep(100);
Thread.Sleep(16);
}
lock (sync)
{
count++;
}
list.Add(Task.Run(() =>
ThreadPool.QueueUserWorkItem(o =>
//Task.Run(() =>
{
try
{
@ -178,7 +195,7 @@ namespace Pixiview.Utils
}
catch (Exception ex)
{
App.DebugError("parallel.start ({tag})", $"failed to run action, index: {i}, error: {ex.Message}");
App.DebugError($"parallel.start ({tag})", $"failed to run action, index: {index}, error: {ex}");
}
finally
{
@ -187,21 +204,31 @@ namespace Pixiview.Utils
count--;
}
}
}));
});
}
while (count > 0)
{
#if LOG
var elapsed = sw.ElapsedMilliseconds;
if (elapsed - lastElapsed > 60000)
{
lastElapsed = elapsed;
App.DebugPrint($"WARNING: parallel task ({tag}), {count} tasks are waiting for end, cost too much time ({elapsed:n0}ms)");
}
#endif
if (disposed)
{
#if LOG
App.DebugPrint($"parallel task determinate, disposed ({tag})");
sw.Stop();
App.DebugPrint($"parallel task determinate, disposed ({tag}), cost time ({elapsed:n0}ms)");
#endif
return;
}
Thread.Sleep(100);
Thread.Sleep(16);
}
#if LOG
App.DebugPrint($"parallel task done ({tag})");
sw.Stop();
App.DebugPrint($"parallel task done ({tag}), cost time ({sw.ElapsedMilliseconds:n0}ms)");
#endif
complete?.Invoke();
}

68
Pixiview/Utils/Stores.cs
View File

@ -379,29 +379,49 @@ namespace Pixiview.Utils
return true;
}
public static IllustPreloadBody LoadIllustPreloadData(string id, bool force = false)
public static IllustPreloadBody LoadIllustPreloadData(string id, bool downloading, bool force = false)
{
var file = Path.Combine(CacheFolder, preloadsFolder, $"{id}.json");
var result = HttpUtility.LoadObject<IllustPreloadBody>(
file,
string.Format(Configs.UrlIllust, id),
null,
out _,
force: force,
action: content =>
IllustPreloadBody result;
if (!force)
{
result = ReadObject<IllustPreloadBody>(file);
if (result != null)
{
var index = content.IndexOf(Configs.SuffixPreload);
if (index > 0)
return result;
}
else if (!downloading)
{
return null;
}
}
if (downloading)
{
result = HttpUtility.LoadObject<IllustPreloadBody>(
file,
string.Format(Configs.UrlIllust, id),
null,
out _,
force: force,
action: content =>
{
index += Configs.SuffixPreloadLength;
var end = content.IndexOf('\'', index);
if (end > index)
var index = content.IndexOf(Configs.SuffixPreload);
if (index > 0)
{
content = content.Substring(index, end - index);
index += Configs.SuffixPreloadLength;
var end = content.IndexOf('\'', index);
if (end > index)
{
content = content.Substring(index, end - index);
}
}
}
return content;
});
return content;
});
}
else
{
result = null;
}
if (result == null)
{
App.DebugPrint($"error when load preload data: force({force})");
@ -567,22 +587,22 @@ namespace Pixiview.Utils
public static ImageSource LoadIllustImage(string url)
{
return LoadImage(url, PersonalFolder, imageFolder);
return LoadImage(url, PersonalFolder, imageFolder, true);
}
public static ImageSource LoadPreviewImage(string url, bool downloading = true, bool force = false)
public static ImageSource LoadPreviewImage(string url, bool downloading, bool force = false)
{
return LoadImage(url, PersonalFolder, previewFolder, downloading, force);
}
public static ImageSource LoadThumbnailImage(string url)
public static ImageSource LoadThumbnailImage(string url, bool downloading, bool force = false)
{
return LoadImage(url, CacheFolder, thumbFolder);
return LoadImage(url, CacheFolder, thumbFolder, downloading, force);
}
public static ImageSource LoadUserProfileImage(string url)
public static ImageSource LoadUserProfileImage(string url, bool downloading, bool force = false)
{
return LoadImage(url, CacheFolder, userFolder);
return LoadImage(url, CacheFolder, userFolder, downloading, force);
}
public static bool CheckIllustImage(string url)
@ -606,7 +626,7 @@ namespace Pixiview.Utils
return null;
}
private static ImageSource LoadImage(string url, string working, string folder, bool downloading = true, bool force = false)
private static ImageSource LoadImage(string url, string working, string folder, bool downloading, bool force = false)
{
var file = Path.Combine(working, folder, Path.GetFileName(url));
ImageSource image;