From 670629a255e2790c9e3ed13c0bbfa1dbc9ff81fa Mon Sep 17 00:00:00 2001
From: loyalsoldier <10487845+Loyalsoldier@users.noreply.github.com>
Date: Tue, 28 Jul 2020 00:50:37 +0800
Subject: [PATCH 1/2] Feat: export lists in plaintext format
---
main.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 56 insertions(+), 1 deletion(-)
diff --git a/main.go b/main.go
index 055b6294..134fb391 100644
--- a/main.go
+++ b/main.go
@@ -18,6 +18,7 @@ import (
var (
dataPath = flag.String("datapath", "", "Path to your custom 'data' directory")
+ exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma")
defaultDataPath = filepath.Join("src", "github.com", "v2fly", "domain-list-community", "data")
)
@@ -38,6 +39,25 @@ type ParsedList struct {
Entry []Entry
}
+func (l *ParsedList) toPlainText(listName string) error {
+ var entryBytes []byte
+ for _, entry := range l.Entry {
+ var attrString string
+ if entry.Attrs != nil {
+ for _, attr := range entry.Attrs {
+ attrString += "@" + attr.GetKey() + ","
+ }
+ attrString = strings.TrimRight(attrString, ",")
+ }
+ // Entry output format is: type:domain.tld:@attr1,@attr2
+ entryBytes = append(entryBytes, []byte(entry.Type+":"+entry.Value+":"+attrString+"\n")...)
+ }
+ if err := ioutil.WriteFile(listName+".txt", entryBytes, 0644); err != nil {
+ return fmt.Errorf(err.Error())
+ }
+ return nil
+}
+
func (l *ParsedList) toProto() (*router.GeoSite, error) {
site := &router.GeoSite{
CountryCode: l.Name,
@@ -75,6 +95,18 @@ func (l *ParsedList) toProto() (*router.GeoSite, error) {
return site, nil
}
+func exportPlainTextList(list []string, refName string, pl *ParsedList) {
+ for _, listName := range list {
+ if strings.ToUpper(refName) == strings.ToUpper(listName) {
+ if err := pl.toPlainText(strings.ToLower(refName)); err != nil {
+ fmt.Println("Failed: ", err)
+ continue
+ }
+ fmt.Printf("'%s' has been generated successfully in current directory.\n", listName)
+ }
+ }
+}
+
func removeComment(line string) string {
idx := strings.Index(line, "#")
if idx == -1 {
@@ -106,6 +138,7 @@ func parseAttribute(attr string) (*router.Domain_Attribute, error) {
return &attribute, errors.New("invalid attribute: " + attr)
}
+ // Trim attribute prefix `@` character
attr = attr[1:]
parts := strings.Split(attr, "=")
if len(parts) == 1 {
@@ -307,7 +340,8 @@ func main() {
os.Exit(1)
}
protoList := new(router.GeoSiteList)
- for _, list := range ref {
+ var existList []string
+ for refName, list := range ref {
pl, err := ParseList(list, ref)
if err != nil {
fmt.Println("Failed: ", err)
@@ -319,6 +353,27 @@ func main() {
os.Exit(1)
}
protoList.Entry = append(protoList.Entry, site)
+
+ // Flatten and export plaintext list
+ if *exportLists != "" {
+ if existList != nil {
+ exportPlainTextList(existList, refName, pl)
+ } else {
+ exportedListSlice := strings.Split(*exportLists, ",")
+ for _, exportedListName := range exportedListSlice {
+ fileName := filepath.Join(dir, exportedListName)
+ _, err := os.Stat(fileName)
+ if err == nil || os.IsExist(err) {
+ existList = append(existList, exportedListName)
+ } else {
+ fmt.Printf("'%s' list does not exist in '%s' directory.\n", exportedListName, dir)
+ }
+ }
+ if existList != nil {
+ exportPlainTextList(existList, refName, pl)
+ }
+ }
+ }
}
protoBytes, err := proto.Marshal(protoList)
From 8d76a667dc3efc1d01ae3bafab8467706411224c Mon Sep 17 00:00:00 2001
From: loyalsoldier <10487845+Loyalsoldier@users.noreply.github.com>
Date: Tue, 28 Jul 2020 00:56:30 +0800
Subject: [PATCH 2/2] Export lists by GitHub Actions & push to release branch
---
.github/workflows/build.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4f26eca1..40586563 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -32,7 +32,7 @@ jobs:
- name: Build dlc.dat file
run: |
- domain-list-community --datapath=${{ env.GOPATH }}/src/${{ env.REPO_URL }}/data
+ domain-list-community --datapath=${{ env.GOPATH }}/src/${{ env.REPO_URL }}/data --exportlists=cn,category-ads-all,geolocation-\!cn
- name: Generate dlc.dat sha256 hash
run: |
@@ -44,7 +44,7 @@ jobs:
git config --local user.name "actions"
git config --local user.email "action@github.com"
git checkout -b release
- git add dlc.dat dlc.dat.sha256sum
+ git add dlc.dat dlc.dat.sha256sum *.txt
git commit -m "${{ env.RELEASE_NAME }}"
git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
git push -f -u origin release