465 lines
18 KiB
C#
465 lines
18 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Net.Http;
|
|
using System.Net.Http.Headers;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace Gallery.Utils
|
|
{
|
|
public class HttpUtility
|
|
{
|
|
public static T LoadObject<T>(string file, string url, string referer, out string error,
|
|
bool force = false,
|
|
bool nojson = false,
|
|
HttpContent post = null,
|
|
Action<HttpRequestHeaders> header = null,
|
|
Func<T, string> namehandler = null,
|
|
Func<string, string> action = null,
|
|
Func<string, T> @return = null)
|
|
{
|
|
string content = null;
|
|
if (post == null && !force && file != null && File.Exists(file))
|
|
{
|
|
try
|
|
{
|
|
content = File.ReadAllText(file);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
App.DebugError("load", $"failed to read file: {file}, error: {ex.Message}");
|
|
}
|
|
}
|
|
if (content == null)
|
|
{
|
|
bool noToken = string.IsNullOrEmpty(Configs.CsrfToken);
|
|
if (noToken)
|
|
{
|
|
post = null;
|
|
}
|
|
var response = Download(url, headers =>
|
|
{
|
|
if (referer != null)
|
|
{
|
|
headers.Referrer = new Uri(referer);
|
|
}
|
|
headers.Add("User-Agent", Configs.UserAgent);
|
|
headers.Add("Accept", Configs.AcceptJson);
|
|
var cookie = Configs.Cookie;
|
|
if (cookie != null)
|
|
{
|
|
headers.Add("Cookie", cookie);
|
|
}
|
|
if (post != null && !noToken)
|
|
{
|
|
headers.Add("Origin", Configs.Referer);
|
|
headers.Add("X-Csrf-Token", Configs.CsrfToken);
|
|
}
|
|
if (header == null)
|
|
{
|
|
var userId = Configs.UserId;
|
|
if (userId != null)
|
|
{
|
|
headers.Add("X-User-Id", userId);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
header(headers);
|
|
}
|
|
}, post);
|
|
if (response == null)
|
|
{
|
|
error = "response is null";
|
|
return default;
|
|
}
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
App.DebugPrint($"http failed with code: {(int)response.StatusCode} - {response.StatusCode}");
|
|
error = response.StatusCode.ToString();
|
|
return default;
|
|
}
|
|
using (response)
|
|
{
|
|
try
|
|
{
|
|
content = response.Content.ReadAsStringAsync().Result;
|
|
if (action != null)
|
|
{
|
|
content = action(content);
|
|
}
|
|
if (@return != null)
|
|
{
|
|
error = null;
|
|
return @return(content);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
App.DebugError("load.stream", $"failed to read stream, error: {ex.Message}");
|
|
error = ex.Message;
|
|
return default;
|
|
}
|
|
|
|
if (content == null)
|
|
{
|
|
content = string.Empty;
|
|
}
|
|
bool rtn = false;
|
|
T result = default;
|
|
if (namehandler != null)
|
|
{
|
|
try
|
|
{
|
|
result = JsonSerializer.Deserialize<T>(content);
|
|
file = namehandler(result);
|
|
rtn = true;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
var memo = content.Length < 20 ? content : content.Substring(0, 20) + "...";
|
|
App.DebugError("load", $"failed to parse illust JSON object, content: {memo}, error: {ex.Message}");
|
|
error = content;
|
|
return default;
|
|
}
|
|
}
|
|
|
|
if (file != null)
|
|
{
|
|
try
|
|
{
|
|
var folder = Path.GetDirectoryName(file);
|
|
if (!Directory.Exists(folder))
|
|
{
|
|
Directory.CreateDirectory(folder);
|
|
}
|
|
File.WriteAllText(file, content, Encoding.UTF8);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
App.DebugError("save", $"failed to save illust JSON object, error: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
if (rtn)
|
|
{
|
|
error = null;
|
|
return result;
|
|
}
|
|
}
|
|
}
|
|
try
|
|
{
|
|
error = null;
|
|
if (nojson)
|
|
{
|
|
return JsonSerializer.Deserialize<T>("{}");
|
|
}
|
|
else
|
|
{
|
|
return JsonSerializer.Deserialize<T>(content);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
var memo = content.Length < 20 ? content : content.Substring(0, 20) + "...";
|
|
App.DebugError("load", $"failed to parse illust JSON object, content: {memo}, error: {ex.Message}");
|
|
error = content;
|
|
return default;
|
|
}
|
|
}
|
|
|
|
public static string DownloadImage(string url, string working, string folder)
|
|
{
|
|
try
|
|
{
|
|
var directory = Path.Combine(working, folder);
|
|
if (!Directory.Exists(directory))
|
|
{
|
|
Directory.CreateDirectory(directory);
|
|
}
|
|
var file = Path.Combine(directory, Path.GetFileName(url));
|
|
var response = Download(url, headers =>
|
|
{
|
|
headers.Referrer = new Uri(Configs.Referer);
|
|
headers.Add("User-Agent", Configs.UserAgent);
|
|
headers.Add("Accept", Configs.AcceptPureImage);
|
|
});
|
|
if (response == null)
|
|
{
|
|
return null;
|
|
}
|
|
using (response)
|
|
using (var fs = File.OpenWrite(file))
|
|
{
|
|
response.Content.CopyToAsync(fs).Wait();
|
|
//if (response.Headers.Date != null)
|
|
//{
|
|
// File.SetLastWriteTimeUtc(file, response.Headers.Date.Value.UtcDateTime);
|
|
//}
|
|
}
|
|
return file;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
App.DebugError("image.download", ex.Message);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
public static Task<string> DownloadImageAsync(string url, string id, string working, string folder)
|
|
{
|
|
try
|
|
{
|
|
var directory = Path.Combine(working, folder);
|
|
if (!Directory.Exists(directory))
|
|
{
|
|
Directory.CreateDirectory(directory);
|
|
}
|
|
var file = Path.Combine(directory, Path.GetFileName(url));
|
|
var proxy = Configs.Proxy;
|
|
var referer = new Uri(string.Format(Configs.RefererIllust, id));
|
|
var handler = new HttpClientHandler
|
|
{
|
|
UseCookies = false
|
|
};
|
|
if (proxy != null)
|
|
{
|
|
handler.Proxy = proxy;
|
|
handler.UseProxy = true;
|
|
}
|
|
var client = new HttpClient(handler)
|
|
{
|
|
Timeout = TimeSpan.FromSeconds(30)
|
|
};
|
|
long size;
|
|
DateTimeOffset lastModified;
|
|
using (var request = new HttpRequestMessage(HttpMethod.Head, url))
|
|
{
|
|
var headers = request.Headers;
|
|
headers.Add("Accept", Configs.AcceptPureImage);
|
|
headers.Add("Accept-Language", Configs.AcceptLanguage);
|
|
headers.Referrer = referer;
|
|
headers.Add("User-Agent", Configs.UserAgent);
|
|
using (var response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result)
|
|
{
|
|
size = response.Content.Headers.ContentLength.Value;
|
|
lastModified = response.Content.Headers.LastModified.Value;
|
|
#if DEBUG
|
|
App.DebugPrint($"content length: {size:n0} bytes, last modified: {lastModified}");
|
|
#endif
|
|
}
|
|
}
|
|
|
|
// segments
|
|
const int SIZE = 150000;
|
|
var list = new List<(long from, long to)>();
|
|
for (var i = 0L; i < size; i += SIZE)
|
|
{
|
|
long to;
|
|
if (i + SIZE >= size)
|
|
{
|
|
to = size - 1;
|
|
}
|
|
else
|
|
{
|
|
to = i + SIZE - 1;
|
|
}
|
|
list.Add((i, to));
|
|
}
|
|
|
|
var data = new byte[size];
|
|
var task = new TaskCompletionSource<string>();
|
|
|
|
ParallelTask.Start($"download.async.{id}", 0, list.Count, Configs.DownloadIllustThreads, i =>
|
|
{
|
|
var (from, to) = list[i];
|
|
using (var request = new HttpRequestMessage(HttpMethod.Get, url))
|
|
{
|
|
var headers = request.Headers;
|
|
headers.Add("Accept", Configs.AcceptPureImage);
|
|
headers.Add("Accept-Language", Configs.AcceptLanguage);
|
|
headers.Add("Accept-Encoding", "identity");
|
|
headers.Referrer = referer;
|
|
headers.IfRange = new RangeConditionHeaderValue(lastModified);
|
|
headers.Range = new RangeHeaderValue(from, to);
|
|
headers.Add("User-Agent", Configs.UserAgent);
|
|
using (var response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result)
|
|
using (var ms = new MemoryStream(data, (int)from, (int)(to - from + 1)))
|
|
{
|
|
response.Content.CopyToAsync(ms).Wait();
|
|
#if DEBUG
|
|
App.DebugPrint($"downloaded range: from ({from:n0}) to ({to:n0})");
|
|
#endif
|
|
}
|
|
}
|
|
return true;
|
|
},
|
|
complete: () =>
|
|
{
|
|
using (var fs = File.OpenWrite(file))
|
|
{
|
|
fs.Write(data, 0, data.Length);
|
|
}
|
|
task.SetResult(file);
|
|
});
|
|
|
|
return task.Task;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
App.DebugError("image.download.async", ex.Message);
|
|
return Task.FromResult<string>(null);
|
|
}
|
|
}
|
|
|
|
private static HttpResponseMessage Download(string url, Action<HttpRequestHeaders> headerAction, HttpContent post = null)
|
|
{
|
|
#if DEBUG
|
|
var method = post == null ? "GET" : "POST";
|
|
App.DebugPrint($"{method}: {url}");
|
|
#endif
|
|
var uri = new Uri(url);
|
|
var proxy = Configs.Proxy;
|
|
var handler = new HttpClientHandler
|
|
{
|
|
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
|
|
UseCookies = false
|
|
};
|
|
if (proxy != null)
|
|
{
|
|
handler.Proxy = proxy;
|
|
handler.UseProxy = true;
|
|
}
|
|
var client = new HttpClient(handler)
|
|
{
|
|
BaseAddress = new Uri($"{uri.Scheme}://{uri.Host}"),
|
|
Timeout = TimeSpan.FromSeconds(30)
|
|
};
|
|
return TryCount(() =>
|
|
{
|
|
using (var request = new HttpRequestMessage(post == null ? HttpMethod.Get : HttpMethod.Post, uri.PathAndQuery)
|
|
{
|
|
Version = new Version(1, 1)
|
|
})
|
|
{
|
|
var headers = request.Headers;
|
|
headerAction(headers);
|
|
//if (proxy == null)
|
|
//{
|
|
// var time = BitConverter.GetBytes(DateTime.UtcNow.Ticks);
|
|
// headers.Add("X-Reverse-Ticks", Convert.ToBase64String(time));
|
|
// time = time.Concat(Encoding.UTF8.GetBytes("_reverse_for_pixiv_by_tsanie")).ToArray();
|
|
// var reverse = System.Security.Cryptography.SHA256.Create().ComputeHash(time);
|
|
// headers.Add("X-Reverse", Convert.ToBase64String(reverse));
|
|
//}
|
|
headers.Add("Accept-Language", Configs.AcceptLanguage);
|
|
//headers.Add("Accept-Encoding", Configs.AcceptEncoding);
|
|
if (post != null)
|
|
{
|
|
request.Content = post;
|
|
}
|
|
return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result;
|
|
}
|
|
});
|
|
}
|
|
|
|
private static T TryCount<T>(Func<T> func, int tryCount = 2)
|
|
{
|
|
int tries = 0;
|
|
while (tries < tryCount)
|
|
{
|
|
try
|
|
{
|
|
return func();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
tries++;
|
|
Thread.Sleep(1000);
|
|
App.DebugError("try.do", $"tries: {tries}, error: {ex.Message}");
|
|
}
|
|
}
|
|
return default;
|
|
}
|
|
|
|
public static (long Size, DateTimeOffset LastModified, HttpClient Client) GetUgoiraHeader(string url, string id)
|
|
{
|
|
var uri = new Uri(url);
|
|
var proxy = Configs.Proxy;
|
|
var handler = new HttpClientHandler
|
|
{
|
|
UseCookies = false
|
|
};
|
|
if (proxy != null)
|
|
{
|
|
handler.Proxy = proxy;
|
|
handler.UseProxy = true;
|
|
}
|
|
var client = new HttpClient(handler)
|
|
{
|
|
BaseAddress = new Uri($"{uri.Scheme}://{uri.Host}"),
|
|
Timeout = TimeSpan.FromSeconds(30)
|
|
};
|
|
var response = TryCount(() =>
|
|
{
|
|
using (var request = new HttpRequestMessage(HttpMethod.Head, uri.PathAndQuery)
|
|
{
|
|
Version = new Version(1, 1)
|
|
})
|
|
{
|
|
var headers = request.Headers;
|
|
UgoiraHeaderAction(headers, id);
|
|
headers.Add("Accept-Encoding", "gzip, deflate");
|
|
headers.Add("Accept-Language", Configs.AcceptLanguage);
|
|
return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result;
|
|
}
|
|
});
|
|
|
|
var size = response.Content.Headers.ContentLength.Value;
|
|
var lastModified = response.Content.Headers.LastModified.Value;
|
|
|
|
return (size, lastModified, client);
|
|
}
|
|
|
|
public static long DownloadUgoiraImage(HttpClient client, string url, string id, DateTimeOffset lastModified, long from, long to, Stream stream)
|
|
{
|
|
var uri = new Uri(url);
|
|
var response = TryCount(() =>
|
|
{
|
|
using (var request = new HttpRequestMessage(HttpMethod.Get, uri.PathAndQuery)
|
|
{
|
|
Version = new Version(1, 1)
|
|
})
|
|
{
|
|
var headers = request.Headers;
|
|
UgoiraHeaderAction(headers, id);
|
|
headers.Add("Accept-Encoding", "identity");
|
|
headers.IfRange = new RangeConditionHeaderValue(lastModified);
|
|
headers.Range = new RangeHeaderValue(from, to);
|
|
headers.Add("Accept-Language", Configs.AcceptLanguage);
|
|
return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result;
|
|
}
|
|
});
|
|
|
|
var length = response.Content.Headers.ContentLength.Value;
|
|
response.Content.CopyToAsync(stream).Wait();
|
|
return length;
|
|
}
|
|
|
|
private static void UgoiraHeaderAction(HttpRequestHeaders headers, string id)
|
|
{
|
|
headers.Add("Accept", "*/*");
|
|
headers.Add("Origin", Configs.Referer);
|
|
headers.Referrer = new Uri(string.Format(Configs.RefererIllust, id));
|
|
headers.Add("User-Agent", Configs.UserAgent);
|
|
}
|
|
}
|
|
|
|
}
|