Fix panic and rule loss in list parsing/resolution (#3905)

* Fix panic on empty inclusion attribute, rule loss in selective inclusion and other bugs * Exercise domain/list-name validators in tests * Consolidate parsed list state into ParsedList and simplify inclusion parsing Co-authored-by: Loyalsoldier <[email protected]> --------- Co-authored-by: copilot-swe-agent[bot] <[email protected]> Co-authored-by: Loyalsoldier <[email protected]>
2026-08-04 18:22:31 +08:00
1 parent 4eb9b28041
commit b49404a24e
@@ -38,15 +38,17 @@ type Inclusion struct {
 }

 type ParsedList struct {
-	Resolved   bool
 	Inclusions []*Inclusion
-	Entries    []*Entry
+	Entries    []*Entry // Entries parsed from the list itself
+	// The fields below are filled in by resolveList
+	Resolving    bool
+	Resolved     bool
+	RoughEntries map[string]*Entry // Deduplicated direct and included entries
+	FinalEntries []*Entry          // Sorted entries without redundant subdomains
 }

 type Processor struct {
-	plMap     map[string]*ParsedList
-	finalMap  map[string][]*Entry
-	cirIncMap map[string]bool
+	parsedListByName map[string]*ParsedList
 }

 type GeoSites struct {
@@ -136,11 +138,12 @@ func (gs *GeoSites) assembleDat(task DatTask) error {
 				return fmt.Errorf("list %q not found for allowlist task", list)
 			}
 		}
+		slices.Sort(allowedIdxes)
+		allowedIdxes = slices.Compact(allowedIdxes) // Avoid duplicated lists
 		allowedlen := len(allowedIdxes)
 		if allowedlen == 0 {
 			return fmt.Errorf("allowlist needs at least one valid list")
 		}
-		slices.Sort(allowedIdxes)
 		geoSiteList.Entry = make([]*router.GeoSite, allowedlen)
 		for i, idx := range allowedIdxes {
 			geoSiteList.Entry[i] = gs.Sites[idx]
@@ -151,12 +154,12 @@ func (gs *GeoSites) assembleDat(task DatTask) error {
 			if idx, ok := gs.SiteIdx[strings.ToUpper(list)]; ok {
 				deniedMap[idx] = true
 			} else {
-				fmt.Printf("[Warn] list %q not found in denylist task %q", list, task.Name)
+				fmt.Printf("[Warn] list %q not found in denylist task %q\n", list, task.Name)
 			}
 		}
 		deniedlen := len(deniedMap)
 		if deniedlen == 0 {
-			fmt.Printf("[Warn] nothing to deny in task %q", task.Name)
+			fmt.Printf("[Warn] nothing to deny in task %q\n", task.Name)
 			geoSiteList.Entry = gs.Sites
 		} else {
 			geoSiteList.Entry = make([]*router.GeoSite, 0, len(gs.Sites)-deniedlen)
@@ -237,7 +240,8 @@ func parseEntry(typ, rule string) (*Entry, []string, error) {
 		}
 	}

-	slices.Sort(entry.Attrs) // Sort attributes
+	slices.Sort(entry.Attrs)                  // Sort attributes
+	entry.Attrs = slices.Compact(entry.Attrs) // Remove duplicated attributes
 	// Formated plain entry: type:domain.tld:@attr1,@attr2
 	var plain strings.Builder
 	plain.Grow(plen)
@@ -272,8 +276,7 @@ func parseInclusion(rule string) (*Inclusion, error) {
 		switch part[0] {
 		case '@':
 			attr := strings.ToLower(part[1:])
-			if attr[0] == '-' {
-				battr := attr[1:]
+			if battr, ok := strings.CutPrefix(attr, "-"); ok {
 				if !validateAttrChars(battr) {
 					return inc, fmt.Errorf("invalid ban attribute: %q", battr)
 				}
@@ -336,10 +339,10 @@ func validateSiteName(name string) bool {
 }

 func (p *Processor) getOrCreateParsedList(name string) *ParsedList {
-	pl, exist := p.plMap[name]
+	pl, exist := p.parsedListByName[name]
 	if !exist {
-		pl = &ParsedList{Resolved: false}
-		p.plMap[name] = pl
+		pl = new(ParsedList)
+		p.parsedListByName[name] = pl
 	}
 	return pl
 }
@@ -457,49 +460,55 @@ func polishList(roughMap map[string]*Entry) []*Entry {
 	return finalList
 }

-func (p *Processor) resolveList(plname string) error {
-	pl, ok := p.plMap[plname]
+// resolveList resolves the inclusions of the named list and returns it.
+func (p *Processor) resolveList(plname string) (*ParsedList, error) {
+	pl, ok := p.parsedListByName[plname]
 	if !ok {
-		return fmt.Errorf("list %q not found", plname)
+		return nil, fmt.Errorf("list %q not found", plname)
 	}
 	if pl.Resolved {
-		return nil
+		return pl, nil
 	}
-	if p.cirIncMap[plname] {
-		return fmt.Errorf("circular inclusion in: %q", plname)
+	if pl.Resolving {
+		return nil, fmt.Errorf("circular inclusion in: %q", plname)
 	}
-	p.cirIncMap[plname] = true
-	defer delete(p.cirIncMap, plname)
+	pl.Resolving = true
+	defer func() { pl.Resolving = false }()

-	roughMap := make(map[string]*Entry) // Avoid basic duplicates
-	for _, dentry := range pl.Entries { // Add direct entries
-		roughMap[dentry.Plain] = dentry
+	roughEntries := make(map[string]*Entry) // Avoid basic duplicates
+	for _, dentry := range pl.Entries {     // Add direct entries
+		roughEntries[dentry.Plain] = dentry
 	}
 	for _, inc := range pl.Inclusions { // Add included entries
-		if err := p.resolveList(inc.Source); err != nil {
-			return fmt.Errorf("failed to resolve inclusion %q: %w", inc.Source, err)
+		ipl, err := p.resolveList(inc.Source)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve inclusion %q: %w", inc.Source, err)
 		}
 		isFullInc := len(inc.MustAttrs) == 0 && len(inc.BanAttrs) == 0
-		for _, ientry := range p.finalMap[inc.Source] {
+		// Filter the unpolished entries of the source list, otherwise selective
+		// inclusion would lose rules that have been pruned in the source list as
+		// redundant subdomains of a parent rule which is filtered out here.
+		for _, ientry := range ipl.RoughEntries {
 			if isFullInc || isMatchAttrFilters(ientry, inc) {
-				roughMap[ientry.Plain] = ientry
+				roughEntries[ientry.Plain] = ientry
 			}
 		}
 	}
-	if len(roughMap) == 0 {
+	pl.RoughEntries = roughEntries
+	if len(roughEntries) == 0 {
 		fmt.Printf("[Warn] ignore empty list %q\n", plname)
 	} else {
-		p.finalMap[plname] = polishList(roughMap)
+		pl.FinalEntries = polishList(roughEntries)
 	}
 	pl.Resolved = true
-	return nil
+	return pl, nil
 }

 func run() error {
 	fmt.Printf("using domain lists data in %q\n", *dataPath)

-	// Generate plMap
-	processor := &Processor{plMap: make(map[string]*ParsedList)}
+	// Parse all lists in the data directory
+	processor := &Processor{parsedListByName: make(map[string]*ParsedList)}
 	err := filepath.WalkDir(*dataPath, func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return err
@@ -516,30 +525,29 @@ func run() error {
 	if err != nil {
 		return fmt.Errorf("failed to loadData: %w", err)
 	}
-	// Generate finalMap
-	processor.finalMap = make(map[string][]*Entry, len(processor.plMap))
-	processor.cirIncMap = make(map[string]bool)
-	for plname := range processor.plMap {
-		if err := processor.resolveList(plname); err != nil {
+	// Resolve the inclusions of all lists
+	for plname := range processor.parsedListByName {
+		if _, err := processor.resolveList(plname); err != nil {
 			return fmt.Errorf("failed to resolveList %q: %w", plname, err)
 		}
 	}
-	processor.plMap = nil

 	// Make sure output directory exists
 	if err := os.MkdirAll(*outputDir, 0755); err != nil {
 		return fmt.Errorf("failed to create output directory: %w", err)
 	}
 	// Export plaintext lists
+	failedCount := 0
 	for rawEpList := range strings.SplitSeq(*exportLists, ",") {
 		if epList := strings.TrimSpace(rawEpList); epList != "" {
-			entries, exist := processor.finalMap[strings.ToUpper(epList)]
-			if !exist {
-				fmt.Printf("[Warn] list %q does not exist\n", epList)
+			pl, exist := processor.parsedListByName[strings.ToUpper(epList)]
+			if !exist || len(pl.FinalEntries) == 0 {
+				fmt.Printf("[Warn] list %q does not exist or is empty\n", epList)
 				continue
 			}
-			if err := writePlainList(epList, entries); err != nil {
+			if err := writePlainList(epList, pl.FinalEntries); err != nil {
 				fmt.Printf("[Error] failed to write list %q: %v\n", epList, err)
+				failedCount++
 				continue
 			}
 			fmt.Printf("list %q has been generated successfully\n", epList)
@@ -547,20 +555,22 @@ func run() error {
 	}

 	// Generate proto sites
-	sitesCount := len(processor.finalMap)
+	listsCount := len(processor.parsedListByName)
 	gs := &GeoSites{
-		Sites:   make([]*router.GeoSite, 0, sitesCount),
-		SiteIdx: make(map[string]int, sitesCount),
+		Sites:   make([]*router.GeoSite, 0, listsCount),
+		SiteIdx: make(map[string]int, listsCount),
 	}
-	for siteName, siteEntries := range processor.finalMap {
-		gs.Sites = append(gs.Sites, makeProtoList(siteName, siteEntries))
+	for siteName, pl := range processor.parsedListByName {
+		if len(pl.FinalEntries) == 0 { // Skip empty lists
+			continue
+		}
+		gs.Sites = append(gs.Sites, makeProtoList(siteName, pl.FinalEntries))
 	}
-	processor = nil
 	// Sort proto sites so the generated file is reproducible
 	slices.SortFunc(gs.Sites, func(a, b *router.GeoSite) int {
 		return strings.Compare(a.CountryCode, b.CountryCode)
 	})
-	for i := range sitesCount {
+	for i := range gs.Sites {
 		gs.SiteIdx[gs.Sites[i].CountryCode] = i
 	}

@@ -577,9 +587,13 @@ func run() error {
 	}
 	for _, task := range tasks {
 		if err := gs.assembleDat(task); err != nil {
-			fmt.Printf("[Error] failed to assembleDat %q: %v", task.Name, err)
+			fmt.Printf("[Error] failed to assembleDat %q: %v\n", task.Name, err)
+			failedCount++
 		}
 	}
+	if failedCount > 0 {
+		return fmt.Errorf("%d output file(s) failed to be generated", failedCount)
+	}
 	return nil
 }