using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; namespace Gallery.Utils { public class HttpUtility { public static T LoadObject(string file, string url, string referer, out string error, bool force = false, bool nojson = false, HttpContent post = null, Action header = null, Func namehandler = null, Func action = null, Func @return = null) { string content = null; if (post == null && !force && file != null && File.Exists(file)) { try { content = File.ReadAllText(file); } catch (Exception ex) { App.DebugError("load", $"failed to read file: {file}, error: {ex.Message}"); } } if (content == null) { bool noToken = string.IsNullOrEmpty(Configs.CsrfToken); if (noToken) { post = null; } var response = Download(url, headers => { if (referer != null) { headers.Referrer = new Uri(referer); } headers.Add("User-Agent", Configs.UserAgent); headers.Add("Accept", Configs.AcceptJson); var cookie = Configs.Cookie; if (cookie != null) { headers.Add("Cookie", cookie); } if (post != null && !noToken) { headers.Add("Origin", Configs.Referer); headers.Add("X-Csrf-Token", Configs.CsrfToken); } if (header == null) { var userId = Configs.UserId; if (userId != null) { headers.Add("X-User-Id", userId); } } else { header(headers); } }, post); if (response == null) { error = "response is null"; return default; } if (!response.IsSuccessStatusCode) { App.DebugPrint($"http failed with code: {(int)response.StatusCode} - {response.StatusCode}"); error = response.StatusCode.ToString(); return default; } using (response) { try { content = response.Content.ReadAsStringAsync().Result; if (action != null) { content = action(content); } if (@return != null) { error = null; return @return(content); } } catch (Exception ex) { App.DebugError("load.stream", $"failed to read stream, error: {ex.Message}"); error = ex.Message; return default; } if (content == null) { content = string.Empty; } bool rtn = false; T result = default; if (namehandler != null) { try { result = JsonSerializer.Deserialize(content); file = namehandler(result); rtn = true; } catch (Exception ex) { var memo = content.Length < 20 ? content : content.Substring(0, 20) + "..."; App.DebugError("load", $"failed to parse illust JSON object, content: {memo}, error: {ex.Message}"); error = content; return default; } } if (file != null) { try { var folder = Path.GetDirectoryName(file); if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } File.WriteAllText(file, content, Encoding.UTF8); } catch (Exception ex) { App.DebugError("save", $"failed to save illust JSON object, error: {ex.Message}"); } } if (rtn) { error = null; return result; } } } try { error = null; if (nojson) { return JsonSerializer.Deserialize("{}"); } else { return JsonSerializer.Deserialize(content); } } catch (Exception ex) { var memo = content.Length < 20 ? content : content.Substring(0, 20) + "..."; App.DebugError("load", $"failed to parse illust JSON object, content: {memo}, error: {ex.Message}"); error = content; return default; } } public static string DownloadImage(string url, string working, string folder) { try { var directory = Path.Combine(working, folder); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } var file = Path.Combine(directory, Path.GetFileName(url)); var response = Download(url, headers => { headers.Referrer = new Uri(Configs.Referer); headers.Add("User-Agent", Configs.UserAgent); headers.Add("Accept", Configs.AcceptPureImage); }); if (response == null) { return null; } using (response) using (var fs = File.OpenWrite(file)) { response.Content.CopyToAsync(fs).Wait(); //if (response.Headers.Date != null) //{ // File.SetLastWriteTimeUtc(file, response.Headers.Date.Value.UtcDateTime); //} } return file; } catch (Exception ex) { App.DebugError("image.download", ex.Message); return null; } } public static Task DownloadImageAsync(string url, string id, string working, string folder) { try { var directory = Path.Combine(working, folder); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } var file = Path.Combine(directory, Path.GetFileName(url)); var proxy = Configs.Proxy; var referer = new Uri(string.Format(Configs.RefererIllust, id)); var handler = new HttpClientHandler { UseCookies = false }; if (proxy != null) { handler.Proxy = proxy; handler.UseProxy = true; } var client = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) }; long size; DateTimeOffset lastModified; using (var request = new HttpRequestMessage(HttpMethod.Head, url)) { var headers = request.Headers; headers.Add("Accept", Configs.AcceptPureImage); headers.Add("Accept-Language", Configs.AcceptLanguage); headers.Referrer = referer; headers.Add("User-Agent", Configs.UserAgent); using (var response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result) { size = response.Content.Headers.ContentLength.Value; lastModified = response.Content.Headers.LastModified.Value; #if DEBUG App.DebugPrint($"content length: {size:n0} bytes, last modified: {lastModified}"); #endif } } // segments const int SIZE = 150000; var list = new List<(long from, long to)>(); for (var i = 0L; i < size; i += SIZE) { long to; if (i + SIZE >= size) { to = size - 1; } else { to = i + SIZE - 1; } list.Add((i, to)); } var data = new byte[size]; var task = new TaskCompletionSource(); ParallelTask.Start($"download.async.{id}", 0, list.Count, Configs.DownloadIllustThreads, i => { var (from, to) = list[i]; using (var request = new HttpRequestMessage(HttpMethod.Get, url)) { var headers = request.Headers; headers.Add("Accept", Configs.AcceptPureImage); headers.Add("Accept-Language", Configs.AcceptLanguage); headers.Add("Accept-Encoding", "identity"); headers.Referrer = referer; headers.IfRange = new RangeConditionHeaderValue(lastModified); headers.Range = new RangeHeaderValue(from, to); headers.Add("User-Agent", Configs.UserAgent); using (var response = client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result) using (var ms = new MemoryStream(data, (int)from, (int)(to - from + 1))) { response.Content.CopyToAsync(ms).Wait(); #if DEBUG App.DebugPrint($"downloaded range: from ({from:n0}) to ({to:n0})"); #endif } } return true; }, complete: () => { using (var fs = File.OpenWrite(file)) { fs.Write(data, 0, data.Length); } task.SetResult(file); }); return task.Task; } catch (Exception ex) { App.DebugError("image.download.async", ex.Message); return Task.FromResult(null); } } private static HttpResponseMessage Download(string url, Action headerAction, HttpContent post = null) { #if DEBUG var method = post == null ? "GET" : "POST"; App.DebugPrint($"{method}: {url}"); #endif var uri = new Uri(url); var proxy = Configs.Proxy; var handler = new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, UseCookies = false }; if (proxy != null) { handler.Proxy = proxy; handler.UseProxy = true; } var client = new HttpClient(handler) { BaseAddress = new Uri($"{uri.Scheme}://{uri.Host}"), Timeout = TimeSpan.FromSeconds(30) }; return TryCount(() => { using (var request = new HttpRequestMessage(post == null ? HttpMethod.Get : HttpMethod.Post, uri.PathAndQuery) { Version = new Version(1, 1) }) { var headers = request.Headers; headerAction(headers); //if (proxy == null) //{ // var time = BitConverter.GetBytes(DateTime.UtcNow.Ticks); // headers.Add("X-Reverse-Ticks", Convert.ToBase64String(time)); // time = time.Concat(Encoding.UTF8.GetBytes("_reverse_for_pixiv_by_tsanie")).ToArray(); // var reverse = System.Security.Cryptography.SHA256.Create().ComputeHash(time); // headers.Add("X-Reverse", Convert.ToBase64String(reverse)); //} headers.Add("Accept-Language", Configs.AcceptLanguage); //headers.Add("Accept-Encoding", Configs.AcceptEncoding); if (post != null) { request.Content = post; } return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result; } }); } private static T TryCount(Func func, int tryCount = 2) { int tries = 0; while (tries < tryCount) { try { return func(); } catch (Exception ex) { tries++; Thread.Sleep(1000); App.DebugError("try.do", $"tries: {tries}, error: {ex.Message}"); } } return default; } public static (long Size, DateTimeOffset LastModified, HttpClient Client) GetUgoiraHeader(string url, string id) { var uri = new Uri(url); var proxy = Configs.Proxy; var handler = new HttpClientHandler { UseCookies = false }; if (proxy != null) { handler.Proxy = proxy; handler.UseProxy = true; } var client = new HttpClient(handler) { BaseAddress = new Uri($"{uri.Scheme}://{uri.Host}"), Timeout = TimeSpan.FromSeconds(30) }; var response = TryCount(() => { using (var request = new HttpRequestMessage(HttpMethod.Head, uri.PathAndQuery) { Version = new Version(1, 1) }) { var headers = request.Headers; UgoiraHeaderAction(headers, id); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", Configs.AcceptLanguage); return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result; } }); var size = response.Content.Headers.ContentLength.Value; var lastModified = response.Content.Headers.LastModified.Value; return (size, lastModified, client); } public static long DownloadUgoiraImage(HttpClient client, string url, string id, DateTimeOffset lastModified, long from, long to, Stream stream) { var uri = new Uri(url); var response = TryCount(() => { using (var request = new HttpRequestMessage(HttpMethod.Get, uri.PathAndQuery) { Version = new Version(1, 1) }) { var headers = request.Headers; UgoiraHeaderAction(headers, id); headers.Add("Accept-Encoding", "identity"); headers.IfRange = new RangeConditionHeaderValue(lastModified); headers.Range = new RangeHeaderValue(from, to); headers.Add("Accept-Language", Configs.AcceptLanguage); return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result; } }); var length = response.Content.Headers.ContentLength.Value; response.Content.CopyToAsync(stream).Wait(); return length; } private static void UgoiraHeaderAction(HttpRequestHeaders headers, string id) { headers.Add("Accept", "*/*"); headers.Add("Origin", Configs.Referer); headers.Referrer = new Uri(string.Format(Configs.RefererIllust, id)); headers.Add("User-Agent", Configs.UserAgent); } } }