generate ookla-speedtest via go

.github/workflows/build.yml

@@ -31,11 +31,6 @@ jobs:
         run: |
           go get -u -v -insecure $REPO_URL
 
-      - name: Automatically generate ookla-speedtest sub-list from source
-        run: |
-          curl -L -o servers-source.xml "https://c.speedtest.net/speedtest-servers-static.php" -H 'Accept-Encoding: gzip' --compressed
-          perl -ne '/host="(.+):[0-9]+"/ && print "full:$1\n"' servers-source.xml | perl -ne 'print if not /^(full:([0-9]{1,3}\.){3}[0-9]{1,3})$/' | perl -ne 'print lc' | sort --ignore-case -u >> $GOPATH/src/$REPO_URL/data/ookla-speedtest
-
       - name: Build geosite.dat file
         run: |
           domain-list-community

data/ookla-speedtest

@@ -1,10 +1,8 @@
 # Source: https://c.speedtest.net/speedtest-servers-static.php
 #
-# Ookla-speedtest hosts are now generated automatically by GitHub Actions.
+# Ookla-speedtest hosts are now generated automatically when generating dlc.dat file.
-# For more details, please visit:
-# https://github.com/v2ray/domain-list-community/blob/master/.github/workflows/build.yml
 #
-# Or you can generate manually with following Javascript code:
+# You can also generate manually with following Javascript code:
 # let servers = [];
 # document.querySelectorAll('server').forEach(s => {
 #   let v = s.attributes.host.value;
@@ -26,6 +24,11 @@
 #   output += "full:" + s + "\n";
 # });
 # console.log(output);
+#
+# Or you can generate manually with following sh code:
+# curl -L -o servers-source.xml "https://c.speedtest.net/speedtest-servers-static.php" -H 'Accept-Encoding: gzip' --compressed
+# perl -ne '/host="(.+):[0-9]+"/ && print "full:$1\n"' servers-source.xml | perl -ne 'print if not /^(full:([0-9]{1,3}\.){3}[0-9]{1,3})$/' | perl -ne 'print lc' | sort --ignore-case -u >> $GOPATH/src/$REPO_URL/data/ookla-speedtest
+
 
 # Do not remove the following line
 include:ookla-speedtest-ads

main.go

@@ -2,11 +2,15 @@ package main
 
 import (
 	"bufio"
+	"compress/gzip"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
+	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -152,6 +156,65 @@ func DetectPath(path string) (string, error) {
 	return "", err
 }
 
+func GenerateSpeedtest(path string) error {
+	req, err := http.NewRequest("GET", "https://c.speedtest.net/speedtest-servers-static.php", nil)
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set("Accept-Encoding", "gzip")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	data, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return err
+	}
+	defer data.Close()
+
+	body, err := ioutil.ReadAll(data)
+	if err != nil {
+		return err
+	}
+
+	reg := regexp.MustCompile(`host="(.+):[0-9]+"`)
+	matchList := reg.FindAllStringSubmatch(string(body), -1)
+
+	exist := make(map[string]bool)
+	var domainList []string
+	for _, match := range matchList {
+		domain := match[1]
+		if exist[domain] {
+			continue
+		}
+
+		ifIP, err := regexp.Match(`^([0-9]{1,3}\.){3}[0-9]{1,3}$`, []byte(domain))
+		if err != nil {
+			return err
+		}
+
+		if ifIP {
+			continue
+		}
+
+		domainList = append(domainList, "full:"+strings.ToLower(domain))
+		exist[domain] = true
+	}
+	sort.Strings(domainList)
+
+	fPath := filepath.Join(path, "ookla-speedtest")
+	b := append([]byte("include:ookla-speedtest-ads\n"), []byte(strings.Join(domainList, "\n"))...)
+	err = ioutil.WriteFile(fPath, b, 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func Load(path string) (*List, error) {
 	file, err := os.Open(path)
 	if err != nil {
@@ -221,6 +284,12 @@ func main() {
 		fmt.Println("Failed: ", err)
 		return
 	}
+
+	if err = GenerateSpeedtest(dir); err != nil {
+		fmt.Println("Failed: ", err)
+		return
+	}
+
 	ref := make(map[string]*List)
 	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {