From d29f7d95b6138030d344d97f5bd42015d5863ada Mon Sep 17 00:00:00 2001 From: unidcml Date: Sun, 2 Feb 2025 19:31:07 +0300 Subject: [PATCH 1/9] Add xray-geosite to convert.py --- convert.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/convert.py b/convert.py index 950de92..5b91bf6 100755 --- a/convert.py +++ b/convert.py @@ -306,6 +306,48 @@ def generate_srs_subnets(input_file, output_json_directory='JSON', compiled_outp except subprocess.CalledProcessError as e: print(f"Compile error {output_file_path}: {e}") +def prepare_dat_domains(domains_or_dirs, output_name): + compiler_directory = 'xray-geosite' + output_lists_directory = os.path.join(compiler_directory, 'data') + + os.makedirs(output_lists_directory, exist_ok=True) + + extracted_domains = [] + + if all(os.path.isdir(os.path.abspath(d)) for d in domains_or_dirs): + for directory in domains_or_dirs: + abs_directory = os.path.abspath(directory) + for filename in os.listdir(abs_directory): + file_path = os.path.join(abs_directory, filename) + + if os.path.isfile(file_path): + with open(file_path, 'r', encoding='utf-8') as file: + attribute = os.path.splitext(filename)[0] + extracted_domains.extend(f"{line.strip()} @{attribute}" for line in file if line.strip()) + else: + extracted_domains = domains_or_dirs + + output_file_path = os.path.join(output_lists_directory, output_name) + with open(output_file_path, 'w', encoding='utf-8') as file: + file.writelines(f"{name}\n" for name in extracted_domains) + +def generate_dat_domains(compiled_output_directory='DAT'): + working_directory = os.getcwd() + compiler_directory = os.path.join(working_directory, 'xray-geosite') + data_path = os.path.join(compiler_directory, 'data') + output_directory = os.path.abspath(compiled_output_directory) + + os.makedirs(compiled_output_directory, exist_ok=True) + + try: + subprocess.run( + ["go", "run", compiler_directory, f"--datapath={data_path}", "--outputname=geosite.dat", f"--outputdir={output_directory}"], + check=True, + cwd=compiler_directory + ) + except subprocess.CalledProcessError as e: + print(f"Compile error geosite.dat: {e}") + if __name__ == '__main__': # Russia inside Path("Russia").mkdir(parents=True, exist_ok=True) @@ -362,4 +404,10 @@ if __name__ == '__main__': # Sing-box subnets generate_srs_subnets(DiscordSubnets) generate_srs_subnets(TwitterSubnets) - generate_srs_subnets(MetaSubnets) \ No newline at end of file + generate_srs_subnets(MetaSubnets) + + # Xray domains + prepare_dat_domains(directories, 'russia_inside') + prepare_dat_domains(russia_outside, 'russia_outside') + prepare_dat_domains(ukraine_inside, 'ukraine_inside') + generate_dat_domains() \ No newline at end of file From f448f195a54753fffbf5af3e448850ee2b9df913 Mon Sep 17 00:00:00 2001 From: unidcml Date: Sun, 2 Feb 2025 19:31:31 +0300 Subject: [PATCH 2/9] Add xray-geosite to Dockerfile --- Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0c35b51..c15466d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,25 @@ FROM ghcr.io/sagernet/sing-box:v1.10.7 AS sing-box +FROM golang:1.23.1-alpine3.20 AS golang + FROM python:3.10.16-alpine3.21 COPY --from=sing-box /usr/local/bin/sing-box /bin/sing-box +COPY --from=golang /usr/local/go /usr/local/go + +ENV GOROOT=/usr/local/go + +ENV PATH="/usr/local/go/bin:${PATH}" + RUN pip install --no-cache-dir tldextract +COPY src/xray-geosite /app/xray-geosite + +WORKDIR /app/xray-geosite + +RUN go mod download + WORKDIR /app COPY convert.py /app/convert.py From 13478d72d51bb70148bbf0a6a7e113cf5d017d51 Mon Sep 17 00:00:00 2001 From: unidcml Date: Sun, 2 Feb 2025 19:31:58 +0300 Subject: [PATCH 3/9] Add xray-geosite to src --- src/xray-geosite/go.mod | 16 ++ src/xray-geosite/go.sum | 20 ++ src/xray-geosite/main.go | 391 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 427 insertions(+) create mode 100644 src/xray-geosite/go.mod create mode 100644 src/xray-geosite/go.sum create mode 100644 src/xray-geosite/main.go diff --git a/src/xray-geosite/go.mod b/src/xray-geosite/go.mod new file mode 100644 index 0000000..72fceab --- /dev/null +++ b/src/xray-geosite/go.mod @@ -0,0 +1,16 @@ +module github.com/v2fly/domain-list-community + +go 1.22 + +toolchain go1.23.1 + +require ( + github.com/v2fly/v2ray-core/v5 v5.19.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + github.com/adrg/xdg v0.5.0 // indirect + github.com/golang/protobuf v1.5.4 // indirect + golang.org/x/sys v0.25.0 // indirect +) diff --git a/src/xray-geosite/go.sum b/src/xray-geosite/go.sum new file mode 100644 index 0000000..8326946 --- /dev/null +++ b/src/xray-geosite/go.sum @@ -0,0 +1,20 @@ +github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY= +github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/v2fly/v2ray-core/v5 v5.19.0 h1:TF2noX1c1npgSg98TASHLdYDWDRhi97gBLpid1cwMUY= +github.com/v2fly/v2ray-core/v5 v5.19.0/go.mod h1:iRydCoQWwE8mhaf/VOWe5jKB8r7LkZfHsbOvwRfJWUo= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/xray-geosite/main.go b/src/xray-geosite/main.go new file mode 100644 index 0000000..d874270 --- /dev/null +++ b/src/xray-geosite/main.go @@ -0,0 +1,391 @@ +package main + +import ( + "bufio" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" + "google.golang.org/protobuf/proto" +) + +var ( + dataPath = flag.String("datapath", "./data", "Path to your custom 'data' directory") + outputName = flag.String("outputname", "dlc.dat", "Name of the generated dat file") + outputDir = flag.String("outputdir", "./", "Directory to place all generated files") + exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma") +) + +type Entry struct { + Type string + Value string + Attrs []*router.Domain_Attribute +} + +type List struct { + Name string + Entry []Entry +} + +type ParsedList struct { + Name string + Inclusion map[string]bool + Entry []Entry +} + +func (l *ParsedList) toPlainText(listName string) error { + var entryBytes []byte + for _, entry := range l.Entry { + var attrString string + if entry.Attrs != nil { + for _, attr := range entry.Attrs { + attrString += "@" + attr.GetKey() + "," + } + attrString = strings.TrimRight(":"+attrString, ",") + } + // Entry output format is: type:domain.tld:@attr1,@attr2 + entryBytes = append(entryBytes, []byte(entry.Type+":"+entry.Value+attrString+"\n")...) + } + if err := os.WriteFile(filepath.Join(*outputDir, listName+".txt"), entryBytes, 0644); err != nil { + return fmt.Errorf(err.Error()) + } + return nil +} + +func (l *ParsedList) toProto() (*router.GeoSite, error) { + site := &router.GeoSite{ + CountryCode: l.Name, + } + for _, entry := range l.Entry { + switch entry.Type { + case "domain": + site.Domain = append(site.Domain, &router.Domain{ + Type: router.Domain_RootDomain, + Value: entry.Value, + Attribute: entry.Attrs, + }) + case "regexp": + site.Domain = append(site.Domain, &router.Domain{ + Type: router.Domain_Regex, + Value: entry.Value, + Attribute: entry.Attrs, + }) + case "keyword": + site.Domain = append(site.Domain, &router.Domain{ + Type: router.Domain_Plain, + Value: entry.Value, + Attribute: entry.Attrs, + }) + case "full": + site.Domain = append(site.Domain, &router.Domain{ + Type: router.Domain_Full, + Value: entry.Value, + Attribute: entry.Attrs, + }) + default: + return nil, errors.New("unknown domain type: " + entry.Type) + } + } + return site, nil +} + +func exportPlainTextList(list []string, refName string, pl *ParsedList) { + for _, listName := range list { + if strings.EqualFold(refName, listName) { + if err := pl.toPlainText(strings.ToLower(refName)); err != nil { + fmt.Println("Failed: ", err) + continue + } + fmt.Printf("'%s' has been generated successfully.\n", listName) + } + } +} + +func removeComment(line string) string { + idx := strings.Index(line, "#") + if idx == -1 { + return line + } + return strings.TrimSpace(line[:idx]) +} + +func parseDomain(domain string, entry *Entry) error { + kv := strings.Split(domain, ":") + if len(kv) == 1 { + entry.Type = "domain" + entry.Value = strings.ToLower(kv[0]) + return nil + } + + if len(kv) == 2 { + entry.Type = strings.ToLower(kv[0]) + entry.Value = strings.ToLower(kv[1]) + return nil + } + + return errors.New("Invalid format: " + domain) +} + +func parseAttribute(attr string) (*router.Domain_Attribute, error) { + var attribute router.Domain_Attribute + if len(attr) == 0 || attr[0] != '@' { + return &attribute, errors.New("invalid attribute: " + attr) + } + + // Trim attribute prefix `@` character + attr = attr[1:] + parts := strings.Split(attr, "=") + if len(parts) == 1 { + attribute.Key = strings.ToLower(parts[0]) + attribute.TypedValue = &router.Domain_Attribute_BoolValue{BoolValue: true} + } else { + attribute.Key = strings.ToLower(parts[0]) + intv, err := strconv.Atoi(parts[1]) + if err != nil { + return &attribute, errors.New("invalid attribute: " + attr + ": " + err.Error()) + } + attribute.TypedValue = &router.Domain_Attribute_IntValue{IntValue: int64(intv)} + } + return &attribute, nil +} + +func parseEntry(line string) (Entry, error) { + line = strings.TrimSpace(line) + parts := strings.Split(line, " ") + + var entry Entry + if len(parts) == 0 { + return entry, errors.New("empty entry") + } + + if err := parseDomain(parts[0], &entry); err != nil { + return entry, err + } + + for i := 1; i < len(parts); i++ { + attr, err := parseAttribute(parts[i]) + if err != nil { + return entry, err + } + entry.Attrs = append(entry.Attrs, attr) + } + + return entry, nil +} + +func Load(path string) (*List, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + + list := &List{ + Name: strings.ToUpper(filepath.Base(path)), + } + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + line = removeComment(line) + if len(line) == 0 { + continue + } + entry, err := parseEntry(line) + if err != nil { + return nil, err + } + list.Entry = append(list.Entry, entry) + } + + return list, nil +} + +func isMatchAttr(Attrs []*router.Domain_Attribute, includeKey string) bool { + isMatch := false + mustMatch := true + matchName := includeKey + if strings.HasPrefix(includeKey, "!") { + isMatch = true + mustMatch = false + matchName = strings.TrimLeft(includeKey, "!") + } + + for _, Attr := range Attrs { + attrName := Attr.Key + if mustMatch { + if matchName == attrName { + isMatch = true + break + } + } else { + if matchName == attrName { + isMatch = false + break + } + } + } + return isMatch +} + +func createIncludeAttrEntrys(list *List, matchAttr *router.Domain_Attribute) []Entry { + newEntryList := make([]Entry, 0, len(list.Entry)) + matchName := matchAttr.Key + for _, entry := range list.Entry { + matched := isMatchAttr(entry.Attrs, matchName) + if matched { + newEntryList = append(newEntryList, entry) + } + } + return newEntryList +} + +func ParseList(list *List, ref map[string]*List) (*ParsedList, error) { + pl := &ParsedList{ + Name: list.Name, + Inclusion: make(map[string]bool), + } + entryList := list.Entry + for { + newEntryList := make([]Entry, 0, len(entryList)) + hasInclude := false + for _, entry := range entryList { + if entry.Type == "include" { + refName := strings.ToUpper(entry.Value) + if entry.Attrs != nil { + for _, attr := range entry.Attrs { + InclusionName := strings.ToUpper(refName + "@" + attr.Key) + if pl.Inclusion[InclusionName] { + continue + } + pl.Inclusion[InclusionName] = true + + refList := ref[refName] + if refList == nil { + return nil, errors.New(entry.Value + " not found.") + } + attrEntrys := createIncludeAttrEntrys(refList, attr) + if len(attrEntrys) != 0 { + newEntryList = append(newEntryList, attrEntrys...) + } + } + } else { + InclusionName := refName + if pl.Inclusion[InclusionName] { + continue + } + pl.Inclusion[InclusionName] = true + refList := ref[refName] + if refList == nil { + return nil, errors.New(entry.Value + " not found.") + } + newEntryList = append(newEntryList, refList.Entry...) + } + hasInclude = true + } else { + newEntryList = append(newEntryList, entry) + } + } + entryList = newEntryList + if !hasInclude { + break + } + } + pl.Entry = entryList + + return pl, nil +} + +func main() { + flag.Parse() + + dir := *dataPath + fmt.Println("Use domain lists in", dir) + + ref := make(map[string]*List) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + list, err := Load(path) + if err != nil { + return err + } + ref[list.Name] = list + return nil + }) + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + + // Create output directory if not exist + if _, err := os.Stat(*outputDir); os.IsNotExist(err) { + if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil { + fmt.Println("Failed: ", mkErr) + os.Exit(1) + } + } + + protoList := new(router.GeoSiteList) + var existList []string + for refName, list := range ref { + pl, err := ParseList(list, ref) + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + site, err := pl.toProto() + if err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } + protoList.Entry = append(protoList.Entry, site) + + // Flatten and export plaintext list + if *exportLists != "" { + if existList != nil { + exportPlainTextList(existList, refName, pl) + } else { + exportedListSlice := strings.Split(*exportLists, ",") + for _, exportedListName := range exportedListSlice { + fileName := filepath.Join(dir, exportedListName) + _, err := os.Stat(fileName) + if err == nil || os.IsExist(err) { + existList = append(existList, exportedListName) + } else { + fmt.Printf("'%s' list does not exist in '%s' directory.\n", exportedListName, dir) + } + } + if existList != nil { + exportPlainTextList(existList, refName, pl) + } + } + } + } + + // Sort protoList so the marshaled list is reproducible + sort.SliceStable(protoList.Entry, func(i, j int) bool { + return protoList.Entry[i].CountryCode < protoList.Entry[j].CountryCode + }) + + protoBytes, err := proto.Marshal(protoList) + if err != nil { + fmt.Println("Failed:", err) + os.Exit(1) + } + if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil { + fmt.Println("Failed: ", err) + os.Exit(1) + } else { + fmt.Println(*outputName, "has been generated successfully.") + } +} From e7eb3addd8a5b6bbb1e087b375367f4c8b691109 Mon Sep 17 00:00:00 2001 From: unidcml Date: Sun, 2 Feb 2025 19:45:02 +0300 Subject: [PATCH 4/9] Update misc files --- .github/workflows/create-lists.yml | 7 +++++-- .gitignore | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-lists.yml b/.github/workflows/create-lists.yml index 100589f..ac182e1 100644 --- a/.github/workflows/create-lists.yml +++ b/.github/workflows/create-lists.yml @@ -30,7 +30,8 @@ jobs: -v ${{ github.workspace }}/Categories:/app/Categories \ -v ${{ github.workspace }}/Services:/app/Services \ -v ${{ github.workspace }}/SRS:/app/SRS \ - itdoginfo/compilesrs:0.1.4 + -v ${{ github.workspace }}/DAT:/app/DAT \ + itdoginfo/compilesrs:0.1.5 - name: Check Russia/inside-dnsmasq-ipset uses: itdoginfo/dnsmasq-action@0.1 @@ -78,5 +79,7 @@ jobs: - name: Release uses: softprops/action-gh-release@v2.1.0 with: - files: "${{ github.workspace }}/SRS/*.srs" + files: | + "${{ github.workspace }}/SRS/*.srs" + "${{ github.workspace }}/DAT/*.dat" tag_name: ${{ env.TAG_NAME }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 602f3be..e218d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ antifilter-domains.lst uablacklist-domains.lst zaboronahelp-domains.lst SRS -JSON \ No newline at end of file +JSON +DAT \ No newline at end of file From 6dae1b91272e8e4fdd59902508debef65ddc1a47 Mon Sep 17 00:00:00 2001 From: unidcml Date: Sun, 2 Feb 2025 20:41:54 +0300 Subject: [PATCH 5/9] Move xray-geosite to root --- Dockerfile | 2 +- {src/xray-geosite => xray-geosite}/go.mod | 0 {src/xray-geosite => xray-geosite}/go.sum | 0 {src/xray-geosite => xray-geosite}/main.go | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename {src/xray-geosite => xray-geosite}/go.mod (100%) rename {src/xray-geosite => xray-geosite}/go.sum (100%) rename {src/xray-geosite => xray-geosite}/main.go (100%) diff --git a/Dockerfile b/Dockerfile index c15466d..fa1f918 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ENV PATH="/usr/local/go/bin:${PATH}" RUN pip install --no-cache-dir tldextract -COPY src/xray-geosite /app/xray-geosite +COPY xray-geosite /app/xray-geosite WORKDIR /app/xray-geosite diff --git a/src/xray-geosite/go.mod b/xray-geosite/go.mod similarity index 100% rename from src/xray-geosite/go.mod rename to xray-geosite/go.mod diff --git a/src/xray-geosite/go.sum b/xray-geosite/go.sum similarity index 100% rename from src/xray-geosite/go.sum rename to xray-geosite/go.sum diff --git a/src/xray-geosite/main.go b/xray-geosite/main.go similarity index 100% rename from src/xray-geosite/main.go rename to xray-geosite/main.go From 694df08c6c59638690c4793f2f96dc80c3a06466 Mon Sep 17 00:00:00 2001 From: unidcml Date: Sun, 2 Feb 2025 20:42:50 +0300 Subject: [PATCH 6/9] Change geosite categories separator --- convert.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/convert.py b/convert.py index 5b91bf6..6a76877 100755 --- a/convert.py +++ b/convert.py @@ -407,7 +407,7 @@ if __name__ == '__main__': generate_srs_subnets(MetaSubnets) # Xray domains - prepare_dat_domains(directories, 'russia_inside') - prepare_dat_domains(russia_outside, 'russia_outside') - prepare_dat_domains(ukraine_inside, 'ukraine_inside') + prepare_dat_domains(directories, 'russia-inside') + prepare_dat_domains(russia_outside, 'russia-outside') + prepare_dat_domains(ukraine_inside, 'ukraine-inside') generate_dat_domains() \ No newline at end of file From 7d859ba5211e9abbe8fe6089706ed702f98a78ec Mon Sep 17 00:00:00 2001 From: unidcml Date: Sun, 2 Feb 2025 21:13:58 +0300 Subject: [PATCH 7/9] Update convert.py --- convert.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/convert.py b/convert.py index 6a76877..44af615 100755 --- a/convert.py +++ b/convert.py @@ -314,11 +314,10 @@ def prepare_dat_domains(domains_or_dirs, output_name): extracted_domains = [] - if all(os.path.isdir(os.path.abspath(d)) for d in domains_or_dirs): + if all(os.path.isdir(d) for d in domains_or_dirs): for directory in domains_or_dirs: - abs_directory = os.path.abspath(directory) - for filename in os.listdir(abs_directory): - file_path = os.path.join(abs_directory, filename) + for filename in os.listdir(directory): + file_path = os.path.join(directory, filename) if os.path.isfile(file_path): with open(file_path, 'r', encoding='utf-8') as file: From 08bfc1cc06061274a2ad80b81832c659a07aa8f2 Mon Sep 17 00:00:00 2001 From: unidcml Date: Mon, 3 Feb 2025 00:58:28 +0300 Subject: [PATCH 8/9] Properly use multi-stage build --- Dockerfile | 20 ++++++++------------ convert.py | 17 +++++------------ 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index fa1f918..a00d7b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,21 @@ FROM ghcr.io/sagernet/sing-box:v1.10.7 AS sing-box -FROM golang:1.23.1-alpine3.20 AS golang +FROM golang:1.23.5-alpine3.21 AS go-builder + +WORKDIR /app + +COPY xray-geosite/. /app + +RUN go build -o geosite-compiler FROM python:3.10.16-alpine3.21 COPY --from=sing-box /usr/local/bin/sing-box /bin/sing-box -COPY --from=golang /usr/local/go /usr/local/go - -ENV GOROOT=/usr/local/go - -ENV PATH="/usr/local/go/bin:${PATH}" +COPY --from=go-builder /app/geosite-compiler /bin/geosite-compiler RUN pip install --no-cache-dir tldextract -COPY xray-geosite /app/xray-geosite - -WORKDIR /app/xray-geosite - -RUN go mod download - WORKDIR /app COPY convert.py /app/convert.py diff --git a/convert.py b/convert.py index 44af615..38b66b4 100755 --- a/convert.py +++ b/convert.py @@ -307,8 +307,7 @@ def generate_srs_subnets(input_file, output_json_directory='JSON', compiled_outp print(f"Compile error {output_file_path}: {e}") def prepare_dat_domains(domains_or_dirs, output_name): - compiler_directory = 'xray-geosite' - output_lists_directory = os.path.join(compiler_directory, 'data') + output_lists_directory = 'data' os.makedirs(output_lists_directory, exist_ok=True) @@ -330,19 +329,13 @@ def prepare_dat_domains(domains_or_dirs, output_name): with open(output_file_path, 'w', encoding='utf-8') as file: file.writelines(f"{name}\n" for name in extracted_domains) -def generate_dat_domains(compiled_output_directory='DAT'): - working_directory = os.getcwd() - compiler_directory = os.path.join(working_directory, 'xray-geosite') - data_path = os.path.join(compiler_directory, 'data') - output_directory = os.path.abspath(compiled_output_directory) - - os.makedirs(compiled_output_directory, exist_ok=True) +def generate_dat_domains(output_name='geosite.dat', output_directory='DAT'): + os.makedirs(output_directory, exist_ok=True) try: subprocess.run( - ["go", "run", compiler_directory, f"--datapath={data_path}", "--outputname=geosite.dat", f"--outputdir={output_directory}"], - check=True, - cwd=compiler_directory + ["geosite-compiler", f"-outputname={output_name}", f"-outputdir={output_directory}"], + check=True ) except subprocess.CalledProcessError as e: print(f"Compile error geosite.dat: {e}") From 38cb76b82716604870104deead1372e24db2d428 Mon Sep 17 00:00:00 2001 From: unidcml Date: Sat, 8 Feb 2025 18:31:44 +0300 Subject: [PATCH 9/9] Get rid of dependencies --- Dockerfile | 9 +- convert.py | 6 +- xray-geosite/go.mod | 16 -- xray-geosite/go.sum | 20 --- xray-geosite/main.go | 391 ------------------------------------------- 5 files changed, 7 insertions(+), 435 deletions(-) delete mode 100644 xray-geosite/go.mod delete mode 100644 xray-geosite/go.sum delete mode 100644 xray-geosite/main.go diff --git a/Dockerfile b/Dockerfile index a00d7b3..7372553 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,17 @@ FROM ghcr.io/sagernet/sing-box:v1.10.7 AS sing-box -FROM golang:1.23.5-alpine3.21 AS go-builder +FROM golang:1.22.12-alpine3.21 AS go-builder WORKDIR /app -COPY xray-geosite/. /app - -RUN go build -o geosite-compiler +RUN CGO_ENABLED=0 GOOS=linux go install -ldflags="-s -w" \ + github.com/v2fly/domain-list-community@20250207120917 FROM python:3.10.16-alpine3.21 COPY --from=sing-box /usr/local/bin/sing-box /bin/sing-box -COPY --from=go-builder /app/geosite-compiler /bin/geosite-compiler +COPY --from=go-builder /go/bin/domain-list-community /bin/domain-list-community RUN pip install --no-cache-dir tldextract diff --git a/convert.py b/convert.py index 38b66b4..e4a77e5 100755 --- a/convert.py +++ b/convert.py @@ -307,7 +307,7 @@ def generate_srs_subnets(input_file, output_json_directory='JSON', compiled_outp print(f"Compile error {output_file_path}: {e}") def prepare_dat_domains(domains_or_dirs, output_name): - output_lists_directory = 'data' + output_lists_directory = 'geosite_data' os.makedirs(output_lists_directory, exist_ok=True) @@ -329,12 +329,12 @@ def prepare_dat_domains(domains_or_dirs, output_name): with open(output_file_path, 'w', encoding='utf-8') as file: file.writelines(f"{name}\n" for name in extracted_domains) -def generate_dat_domains(output_name='geosite.dat', output_directory='DAT'): +def generate_dat_domains(data_path='geosite_data', output_name='geosite.dat', output_directory='DAT'): os.makedirs(output_directory, exist_ok=True) try: subprocess.run( - ["geosite-compiler", f"-outputname={output_name}", f"-outputdir={output_directory}"], + ["domain-list-community", f"-datapath={data_path}", f"-outputname={output_name}", f"-outputdir={output_directory}"], check=True ) except subprocess.CalledProcessError as e: diff --git a/xray-geosite/go.mod b/xray-geosite/go.mod deleted file mode 100644 index 72fceab..0000000 --- a/xray-geosite/go.mod +++ /dev/null @@ -1,16 +0,0 @@ -module github.com/v2fly/domain-list-community - -go 1.22 - -toolchain go1.23.1 - -require ( - github.com/v2fly/v2ray-core/v5 v5.19.0 - google.golang.org/protobuf v1.34.2 -) - -require ( - github.com/adrg/xdg v0.5.0 // indirect - github.com/golang/protobuf v1.5.4 // indirect - golang.org/x/sys v0.25.0 // indirect -) diff --git a/xray-geosite/go.sum b/xray-geosite/go.sum deleted file mode 100644 index 8326946..0000000 --- a/xray-geosite/go.sum +++ /dev/null @@ -1,20 +0,0 @@ -github.com/adrg/xdg v0.5.0 h1:dDaZvhMXatArP1NPHhnfaQUqWBLBsmx1h1HXQdMoFCY= -github.com/adrg/xdg v0.5.0/go.mod h1:dDdY4M4DF9Rjy4kHPeNL+ilVF+p2lK8IdM9/rTSGcI4= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/v2fly/v2ray-core/v5 v5.19.0 h1:TF2noX1c1npgSg98TASHLdYDWDRhi97gBLpid1cwMUY= -github.com/v2fly/v2ray-core/v5 v5.19.0/go.mod h1:iRydCoQWwE8mhaf/VOWe5jKB8r7LkZfHsbOvwRfJWUo= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/xray-geosite/main.go b/xray-geosite/main.go deleted file mode 100644 index d874270..0000000 --- a/xray-geosite/main.go +++ /dev/null @@ -1,391 +0,0 @@ -package main - -import ( - "bufio" - "errors" - "flag" - "fmt" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - - router "github.com/v2fly/v2ray-core/v5/app/router/routercommon" - "google.golang.org/protobuf/proto" -) - -var ( - dataPath = flag.String("datapath", "./data", "Path to your custom 'data' directory") - outputName = flag.String("outputname", "dlc.dat", "Name of the generated dat file") - outputDir = flag.String("outputdir", "./", "Directory to place all generated files") - exportLists = flag.String("exportlists", "", "Lists to be flattened and exported in plaintext format, separated by ',' comma") -) - -type Entry struct { - Type string - Value string - Attrs []*router.Domain_Attribute -} - -type List struct { - Name string - Entry []Entry -} - -type ParsedList struct { - Name string - Inclusion map[string]bool - Entry []Entry -} - -func (l *ParsedList) toPlainText(listName string) error { - var entryBytes []byte - for _, entry := range l.Entry { - var attrString string - if entry.Attrs != nil { - for _, attr := range entry.Attrs { - attrString += "@" + attr.GetKey() + "," - } - attrString = strings.TrimRight(":"+attrString, ",") - } - // Entry output format is: type:domain.tld:@attr1,@attr2 - entryBytes = append(entryBytes, []byte(entry.Type+":"+entry.Value+attrString+"\n")...) - } - if err := os.WriteFile(filepath.Join(*outputDir, listName+".txt"), entryBytes, 0644); err != nil { - return fmt.Errorf(err.Error()) - } - return nil -} - -func (l *ParsedList) toProto() (*router.GeoSite, error) { - site := &router.GeoSite{ - CountryCode: l.Name, - } - for _, entry := range l.Entry { - switch entry.Type { - case "domain": - site.Domain = append(site.Domain, &router.Domain{ - Type: router.Domain_RootDomain, - Value: entry.Value, - Attribute: entry.Attrs, - }) - case "regexp": - site.Domain = append(site.Domain, &router.Domain{ - Type: router.Domain_Regex, - Value: entry.Value, - Attribute: entry.Attrs, - }) - case "keyword": - site.Domain = append(site.Domain, &router.Domain{ - Type: router.Domain_Plain, - Value: entry.Value, - Attribute: entry.Attrs, - }) - case "full": - site.Domain = append(site.Domain, &router.Domain{ - Type: router.Domain_Full, - Value: entry.Value, - Attribute: entry.Attrs, - }) - default: - return nil, errors.New("unknown domain type: " + entry.Type) - } - } - return site, nil -} - -func exportPlainTextList(list []string, refName string, pl *ParsedList) { - for _, listName := range list { - if strings.EqualFold(refName, listName) { - if err := pl.toPlainText(strings.ToLower(refName)); err != nil { - fmt.Println("Failed: ", err) - continue - } - fmt.Printf("'%s' has been generated successfully.\n", listName) - } - } -} - -func removeComment(line string) string { - idx := strings.Index(line, "#") - if idx == -1 { - return line - } - return strings.TrimSpace(line[:idx]) -} - -func parseDomain(domain string, entry *Entry) error { - kv := strings.Split(domain, ":") - if len(kv) == 1 { - entry.Type = "domain" - entry.Value = strings.ToLower(kv[0]) - return nil - } - - if len(kv) == 2 { - entry.Type = strings.ToLower(kv[0]) - entry.Value = strings.ToLower(kv[1]) - return nil - } - - return errors.New("Invalid format: " + domain) -} - -func parseAttribute(attr string) (*router.Domain_Attribute, error) { - var attribute router.Domain_Attribute - if len(attr) == 0 || attr[0] != '@' { - return &attribute, errors.New("invalid attribute: " + attr) - } - - // Trim attribute prefix `@` character - attr = attr[1:] - parts := strings.Split(attr, "=") - if len(parts) == 1 { - attribute.Key = strings.ToLower(parts[0]) - attribute.TypedValue = &router.Domain_Attribute_BoolValue{BoolValue: true} - } else { - attribute.Key = strings.ToLower(parts[0]) - intv, err := strconv.Atoi(parts[1]) - if err != nil { - return &attribute, errors.New("invalid attribute: " + attr + ": " + err.Error()) - } - attribute.TypedValue = &router.Domain_Attribute_IntValue{IntValue: int64(intv)} - } - return &attribute, nil -} - -func parseEntry(line string) (Entry, error) { - line = strings.TrimSpace(line) - parts := strings.Split(line, " ") - - var entry Entry - if len(parts) == 0 { - return entry, errors.New("empty entry") - } - - if err := parseDomain(parts[0], &entry); err != nil { - return entry, err - } - - for i := 1; i < len(parts); i++ { - attr, err := parseAttribute(parts[i]) - if err != nil { - return entry, err - } - entry.Attrs = append(entry.Attrs, attr) - } - - return entry, nil -} - -func Load(path string) (*List, error) { - file, err := os.Open(path) - if err != nil { - return nil, err - } - defer file.Close() - - list := &List{ - Name: strings.ToUpper(filepath.Base(path)), - } - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - line = removeComment(line) - if len(line) == 0 { - continue - } - entry, err := parseEntry(line) - if err != nil { - return nil, err - } - list.Entry = append(list.Entry, entry) - } - - return list, nil -} - -func isMatchAttr(Attrs []*router.Domain_Attribute, includeKey string) bool { - isMatch := false - mustMatch := true - matchName := includeKey - if strings.HasPrefix(includeKey, "!") { - isMatch = true - mustMatch = false - matchName = strings.TrimLeft(includeKey, "!") - } - - for _, Attr := range Attrs { - attrName := Attr.Key - if mustMatch { - if matchName == attrName { - isMatch = true - break - } - } else { - if matchName == attrName { - isMatch = false - break - } - } - } - return isMatch -} - -func createIncludeAttrEntrys(list *List, matchAttr *router.Domain_Attribute) []Entry { - newEntryList := make([]Entry, 0, len(list.Entry)) - matchName := matchAttr.Key - for _, entry := range list.Entry { - matched := isMatchAttr(entry.Attrs, matchName) - if matched { - newEntryList = append(newEntryList, entry) - } - } - return newEntryList -} - -func ParseList(list *List, ref map[string]*List) (*ParsedList, error) { - pl := &ParsedList{ - Name: list.Name, - Inclusion: make(map[string]bool), - } - entryList := list.Entry - for { - newEntryList := make([]Entry, 0, len(entryList)) - hasInclude := false - for _, entry := range entryList { - if entry.Type == "include" { - refName := strings.ToUpper(entry.Value) - if entry.Attrs != nil { - for _, attr := range entry.Attrs { - InclusionName := strings.ToUpper(refName + "@" + attr.Key) - if pl.Inclusion[InclusionName] { - continue - } - pl.Inclusion[InclusionName] = true - - refList := ref[refName] - if refList == nil { - return nil, errors.New(entry.Value + " not found.") - } - attrEntrys := createIncludeAttrEntrys(refList, attr) - if len(attrEntrys) != 0 { - newEntryList = append(newEntryList, attrEntrys...) - } - } - } else { - InclusionName := refName - if pl.Inclusion[InclusionName] { - continue - } - pl.Inclusion[InclusionName] = true - refList := ref[refName] - if refList == nil { - return nil, errors.New(entry.Value + " not found.") - } - newEntryList = append(newEntryList, refList.Entry...) - } - hasInclude = true - } else { - newEntryList = append(newEntryList, entry) - } - } - entryList = newEntryList - if !hasInclude { - break - } - } - pl.Entry = entryList - - return pl, nil -} - -func main() { - flag.Parse() - - dir := *dataPath - fmt.Println("Use domain lists in", dir) - - ref := make(map[string]*List) - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - list, err := Load(path) - if err != nil { - return err - } - ref[list.Name] = list - return nil - }) - if err != nil { - fmt.Println("Failed: ", err) - os.Exit(1) - } - - // Create output directory if not exist - if _, err := os.Stat(*outputDir); os.IsNotExist(err) { - if mkErr := os.MkdirAll(*outputDir, 0755); mkErr != nil { - fmt.Println("Failed: ", mkErr) - os.Exit(1) - } - } - - protoList := new(router.GeoSiteList) - var existList []string - for refName, list := range ref { - pl, err := ParseList(list, ref) - if err != nil { - fmt.Println("Failed: ", err) - os.Exit(1) - } - site, err := pl.toProto() - if err != nil { - fmt.Println("Failed: ", err) - os.Exit(1) - } - protoList.Entry = append(protoList.Entry, site) - - // Flatten and export plaintext list - if *exportLists != "" { - if existList != nil { - exportPlainTextList(existList, refName, pl) - } else { - exportedListSlice := strings.Split(*exportLists, ",") - for _, exportedListName := range exportedListSlice { - fileName := filepath.Join(dir, exportedListName) - _, err := os.Stat(fileName) - if err == nil || os.IsExist(err) { - existList = append(existList, exportedListName) - } else { - fmt.Printf("'%s' list does not exist in '%s' directory.\n", exportedListName, dir) - } - } - if existList != nil { - exportPlainTextList(existList, refName, pl) - } - } - } - } - - // Sort protoList so the marshaled list is reproducible - sort.SliceStable(protoList.Entry, func(i, j int) bool { - return protoList.Entry[i].CountryCode < protoList.Entry[j].CountryCode - }) - - protoBytes, err := proto.Marshal(protoList) - if err != nil { - fmt.Println("Failed:", err) - os.Exit(1) - } - if err := os.WriteFile(filepath.Join(*outputDir, *outputName), protoBytes, 0644); err != nil { - fmt.Println("Failed: ", err) - os.Exit(1) - } else { - fmt.Println(*outputName, "has been generated successfully.") - } -}