using System; using System.IO; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; using Newtonsoft.Json; namespace Pixiview.Utils { public class HttpUtility { public static T LoadObject(string file, string url, string referer, bool force = false, Action header = null, Func namehandler = null, Func action = null) { string content = null; if (!force && file != null && File.Exists(file)) { try { content = File.ReadAllText(file); } catch (Exception ex) { App.DebugError("load", $"failed to read file: {file}, error: {ex.Message}"); } } if (content == null) { var response = Download(url, headers => { if (referer != null) { headers.Referrer = new Uri(referer); } headers.Add("User-Agent", Configs.UserAgent); headers.Add("Accept", Configs.AcceptJson); var cookie = Configs.Cookie; if (cookie != null) { headers.Add("Cookie", cookie); } if (header == null) { var userId = Configs.UserId; if (userId != null) { headers.Add("X-User-Id", userId); } } else { header(headers); } }); if (response == null) { return default; } using (response) { try { content = response.Content.ReadAsStringAsync().Result; if (action != null) { content = action(content); } } catch (Exception ex) { App.DebugError("load.strea", $"failed to read stream, error: {ex.Message}"); return default; } bool rtn = false; T result = default; if (namehandler != null) { try { result = JsonConvert.DeserializeObject(content); file = namehandler(result); rtn = true; } catch (Exception ex) { App.DebugError("load", $"failed to parse illust JSON object, error: {ex.Message}"); } } if (file != null) { try { var folder = Path.GetDirectoryName(file); if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } File.WriteAllText(file, content, Encoding.UTF8); } catch (Exception ex) { App.DebugError("save", $"failed to save illust JSON object, error: {ex.Message}"); } } if (rtn) { return result; } } } try { return JsonConvert.DeserializeObject(content); } catch (Exception ex) { App.DebugError("load", $"failed to parse illust JSON object, error: {ex.Message}"); return default; } } public static string DownloadImage(string url, string working, string folder) { try { var directory = Path.Combine(working, folder); if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } var file = Path.Combine(directory, Path.GetFileName(url)); var response = Download(url, headers => { headers.Referrer = new Uri(Configs.Referer); headers.Add("User-Agent", Configs.UserAgent); headers.Add("Accept", Configs.AcceptImage); }); if (response == null) { return null; } using (response) using (var fs = File.OpenWrite(file)) { response.Content.CopyToAsync(fs).Wait(); //if (response.Headers.Date != null) //{ // File.SetLastWriteTimeUtc(file, response.Headers.Date.Value.UtcDateTime); //} } return file; } catch (Exception ex) { App.DebugError("image.download", ex.Message); return null; } } private static HttpResponseMessage Download(string url, Action headerAction) { App.DebugPrint($"GET: {url}"); var uri = new Uri(url); var proxy = Configs.Proxy; var handler = new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, UseCookies = false }; if (proxy != null) { handler.Proxy = proxy; handler.UseProxy = true; } var client = new HttpClient(handler) { BaseAddress = new Uri($"{uri.Scheme}://{uri.Host}"), Timeout = TimeSpan.FromSeconds(30) }; return TryCount(() => { using (var request = new HttpRequestMessage(HttpMethod.Get, uri.PathAndQuery) { Version = new Version(2, 0) }) { var headers = request.Headers; headerAction(headers); if (proxy == null) { headers.Add("x-reverse", "yes"); } headers.Add("Accept-Language", Configs.AcceptLanguage); //headers.Add("Accept-Encoding", Configs.AcceptEncoding); return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result; } }); } private static T TryCount(Func func, int tryCount = 2) { int tries = 0; while (tries < tryCount) { try { return func(); } catch (Exception ex) { tries++; Thread.Sleep(1000); App.DebugError("try.do", $"tries: {tries}, error: {ex.Message}"); } } return default; } public static (long Size, DateTimeOffset LastModified, HttpClient Client) GetUgoiraHeader(string url, string id) { var uri = new Uri(url); var proxy = Configs.Proxy; var handler = new HttpClientHandler { UseCookies = false }; if (proxy != null) { handler.Proxy = proxy; handler.UseProxy = true; } var client = new HttpClient(handler) { BaseAddress = new Uri($"{uri.Scheme}://{uri.Host}"), Timeout = TimeSpan.FromSeconds(30) }; var response = TryCount(() => { using (var request = new HttpRequestMessage(HttpMethod.Head, uri.PathAndQuery) { Version = new Version(2, 0) }) { var headers = request.Headers; UgoiraHeaderAction(headers, id); headers.Add("Accept-Encoding", "gzip, deflate"); headers.Add("Accept-Language", Configs.AcceptLanguage); return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result; } }); var size = response.Content.Headers.ContentLength.Value; var lastModified = response.Content.Headers.LastModified.Value; return (size, lastModified, client); } public static long DownloadUgoiraImage(HttpClient client, string url, string id, DateTimeOffset lastModified, long from, long to, Stream stream) { var uri = new Uri(url); var response = TryCount(() => { using (var request = new HttpRequestMessage(HttpMethod.Get, uri.PathAndQuery) { Version = new Version(2, 0) }) { var headers = request.Headers; UgoiraHeaderAction(headers, id); headers.Add("Accept-Encoding", "identity"); headers.IfRange = new RangeConditionHeaderValue(lastModified); headers.Range = new RangeHeaderValue(from, to); headers.Add("Accept-Language", Configs.AcceptLanguage); return client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result; } }); var length = response.Content.Headers.ContentLength.Value; response.Content.CopyToAsync(stream).Wait(); return length; } private static void UgoiraHeaderAction(HttpRequestHeaders headers, string id) { headers.Add("Accept", "*/*"); headers.Add("Origin", Configs.Referer); headers.Referrer = new Uri(string.Format(Configs.RefererIllust, id)); headers.Add("User-Agent", Configs.UserAgent); } } }