From 8e05324f3629f467ef245cb000ac1db602f109d7 Mon Sep 17 00:00:00 2001 From: Richard Chen Date: Sun, 9 Feb 2020 18:00:25 +0800 Subject: [PATCH 1/3] generate ookla-speedtest via go --- .github/workflows/build.yml | 5 --- data/ookla-speedtest | 11 +++--- main.go | 69 +++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68c2a506..711bb069 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,11 +31,6 @@ jobs: run: | go get -u -v -insecure $REPO_URL - - name: Automatically generate ookla-speedtest sub-list from source - run: | - curl -L -o servers-source.xml "https://c.speedtest.net/speedtest-servers-static.php" -H 'Accept-Encoding: gzip' --compressed - perl -ne '/host="(.+):[0-9]+"/ && print "full:$1\n"' servers-source.xml | perl -ne 'print if not /^(full:([0-9]{1,3}\.){3}[0-9]{1,3})$/' | perl -ne 'print lc' | sort --ignore-case -u >> $GOPATH/src/$REPO_URL/data/ookla-speedtest - - name: Build geosite.dat file run: | domain-list-community diff --git a/data/ookla-speedtest b/data/ookla-speedtest index 2b501dbe..f2df9083 100644 --- a/data/ookla-speedtest +++ b/data/ookla-speedtest @@ -1,10 +1,8 @@ # Source: https://c.speedtest.net/speedtest-servers-static.php # -# Ookla-speedtest hosts are now generated automatically by GitHub Actions. -# For more details, please visit: -# https://github.com/v2ray/domain-list-community/blob/master/.github/workflows/build.yml +# Ookla-speedtest hosts are now generated automatically when generating dlc.dat file. # -# Or you can generate manually with following Javascript code: +# You can also generate manually with following Javascript code: # let servers = []; # document.querySelectorAll('server').forEach(s => { # let v = s.attributes.host.value; @@ -26,6 +24,11 @@ # output += "full:" + s + "\n"; # }); # console.log(output); +# +# Or you can generate manually with following sh code: +# curl -L -o servers-source.xml "https://c.speedtest.net/speedtest-servers-static.php" -H 'Accept-Encoding: gzip' --compressed +# perl -ne '/host="(.+):[0-9]+"/ && print "full:$1\n"' servers-source.xml | perl -ne 'print if not /^(full:([0-9]{1,3}\.){3}[0-9]{1,3})$/' | perl -ne 'print lc' | sort --ignore-case -u >> $GOPATH/src/$REPO_URL/data/ookla-speedtest + # Do not remove the following line include:ookla-speedtest-ads diff --git a/main.go b/main.go index 91544bd4..273b9cf1 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,15 @@ package main import ( "bufio" + "compress/gzip" "errors" "fmt" "io/ioutil" + "net/http" "os" "path/filepath" + "regexp" + "sort" "strconv" "strings" @@ -152,6 +156,65 @@ func DetectPath(path string) (string, error) { return "", err } +func GenerateSpeedtest(path string) error { + req, err := http.NewRequest("GET", "https://c.speedtest.net/speedtest-servers-static.php", nil) + if err != nil { + return err + } + + req.Header.Set("Accept-Encoding", "gzip") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + data, err := gzip.NewReader(resp.Body) + if err != nil { + return err + } + defer data.Close() + + body, err := ioutil.ReadAll(data) + if err != nil { + return err + } + + reg := regexp.MustCompile(`host="(.+):[0-9]+"`) + matchList := reg.FindAllStringSubmatch(string(body), -1) + + exist := make(map[string]bool) + var domainList []string + for _, match := range matchList { + domain := match[1] + if exist[domain] { + continue + } + + ifIP, err := regexp.Match(`^([0-9]{1,3}\.){3}[0-9]{1,3}$`, []byte(domain)) + if err != nil { + return err + } + + if ifIP { + continue + } + + domainList = append(domainList, "full:"+strings.ToLower(domain)) + exist[domain] = true + } + sort.Strings(domainList) + + fPath := filepath.Join(path, "ookla-speedtest") + b := append([]byte("include:ookla-speedtest-ads\n"), []byte(strings.Join(domainList, "\n"))...) + err = ioutil.WriteFile(fPath, b, 0644) + if err != nil { + return err + } + + return nil +} + func Load(path string) (*List, error) { file, err := os.Open(path) if err != nil { @@ -221,6 +284,12 @@ func main() { fmt.Println("Failed: ", err) return } + + if err = GenerateSpeedtest(dir); err != nil { + fmt.Println("Failed: ", err) + return + } + ref := make(map[string]*List) err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { From 157b7292a3f7d7e94f8e4c11ff84d613afa16c34 Mon Sep 17 00:00:00 2001 From: Richard Chen Date: Sun, 9 Feb 2020 18:20:07 +0800 Subject: [PATCH 2/3] add UA --- main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main.go b/main.go index 273b9cf1..b7ea7acd 100644 --- a/main.go +++ b/main.go @@ -163,6 +163,7 @@ func GenerateSpeedtest(path string) error { } req.Header.Set("Accept-Encoding", "gzip") + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36") resp, err := http.DefaultClient.Do(req) if err != nil { return err @@ -183,6 +184,10 @@ func GenerateSpeedtest(path string) error { reg := regexp.MustCompile(`host="(.+):[0-9]+"`) matchList := reg.FindAllStringSubmatch(string(body), -1) + if len(matchList) == 0 { + return errors.New("cannot get ookla-speedtest domains") + } + exist := make(map[string]bool) var domainList []string for _, match := range matchList { @@ -204,6 +209,7 @@ func GenerateSpeedtest(path string) error { exist[domain] = true } sort.Strings(domainList) + fmt.Printf("%ddomains added.\n", len(domainList)) fPath := filepath.Join(path, "ookla-speedtest") b := append([]byte("include:ookla-speedtest-ads\n"), []byte(strings.Join(domainList, "\n"))...) From 94e0310748e91c5352f90b0c508b969a8f96ad89 Mon Sep 17 00:00:00 2001 From: Richard Chen Date: Sun, 9 Feb 2020 18:26:35 +0800 Subject: [PATCH 3/3] Revert "add UA" This reverts commit 157b7292a3f7d7e94f8e4c11ff84d613afa16c34. --- main.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/main.go b/main.go index b7ea7acd..273b9cf1 100644 --- a/main.go +++ b/main.go @@ -163,7 +163,6 @@ func GenerateSpeedtest(path string) error { } req.Header.Set("Accept-Encoding", "gzip") - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36") resp, err := http.DefaultClient.Do(req) if err != nil { return err @@ -184,10 +183,6 @@ func GenerateSpeedtest(path string) error { reg := regexp.MustCompile(`host="(.+):[0-9]+"`) matchList := reg.FindAllStringSubmatch(string(body), -1) - if len(matchList) == 0 { - return errors.New("cannot get ookla-speedtest domains") - } - exist := make(map[string]bool) var domainList []string for _, match := range matchList { @@ -209,7 +204,6 @@ func GenerateSpeedtest(path string) error { exist[domain] = true } sort.Strings(domainList) - fmt.Printf("%ddomains added.\n", len(domainList)) fPath := filepath.Join(path, "ookla-speedtest") b := append([]byte("include:ookla-speedtest-ads\n"), []byte(strings.Join(domainList, "\n"))...)