Release all domain lists

This reverts commit b735c06461.

.github/workflows/build.yml

@@ -24,12 +24,18 @@ jobs:
           echo "::set-env name=TAG_NAME::$(date +%Y%m%d%H%M)"
           echo "::set-env name=GEOIP_REPO::github.com/v2ray/geoip"
           echo "::set-env name=GEOSITE_REPO::github.com/v2ray/domain-list-community"
-          echo "::set-env name=GOOGLE_URL::https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf"
-          echo "::set-env name=APPLE_URL::https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf"
-          echo "::set-env name=GFWLIST_URL::https://cokebar.github.io/gfwlist2dnsmasq/gfwlist_domain.txt"
-          echo "::set-env name=Blocked_DOMAINS_URL::https://raw.githubusercontent.com/wongsyrone/domain-block-list/master/domains.txt"
           echo "::set-env name=CHINA_DOMAINS_URL::https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf"
-          echo "::set-env name=Profiles_URL::https://raw.githubusercontent.com/ConnersHua/Profiles/master/Shadow/Pro.conf"
+          echo "::set-env name=GOOGLE_DOMAINS_URL::https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf"
+          echo "::set-env name=APPLE_DOMAINS_URL::https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf"
+          echo "::set-env name=GFWLIST_DOMAINS_URL::https://cokebar.github.io/gfwlist2dnsmasq/gfwlist_domain.txt"
+          echo "::set-env name=GREATFIRE_DOMAINS_URL::https://raw.githubusercontent.com/wongsyrone/domain-block-list/master/domains.txt"
+          echo "::set-env name=PROFILES_URL::https://raw.githubusercontent.com/ConnersHua/Profiles/master/Shadow/Pro.conf"
+          echo "::set-env name=GEQ1AN_RULES_APPLE_URL::https://raw.githubusercontent.com/GeQ1an/Rules/master/QuantumultX/Filter/Apple.list"
+          echo "::set-env name=GEQ1AN_RULES_MICROSOFT_URL::https://raw.githubusercontent.com/GeQ1an/Rules/master/QuantumultX/Filter/Microsoft.list"
+          echo "::set-env name=GEQ1AN_RULES_GLOBAL_MEDIA_URL::https://raw.githubusercontent.com/GeQ1an/Rules/master/QuantumultX/Filter/GMedia.list"
+          echo "::set-env name=GEQ1AN_RULES_OUTSIDE_URL::https://raw.githubusercontent.com/GeQ1an/Rules/master/QuantumultX/Filter/Outside.list"
+          echo "::set-env name=GEQ1AN_RULES_REJECT_URL::https://raw.githubusercontent.com/GeQ1an/Rules/master/QuantumultX/Filter/AdBlock.list"
+          echo "::set-env name=LHIE1_RULES_REJECT_URL::https://raw.githubusercontent.com/lhie1/Rules/master/Auto/REJECT.conf"
           echo "::set-env name=GOPATH::$(dirname $GITHUB_WORKSPACE)"
           echo "::add-path::$(dirname $GITHUB_WORKSPACE)/bin"
         shell: bash
@@ -39,23 +45,17 @@ jobs:
         with:
           ref: hidden
 
-#       - name: Get GeoLite2 file
-#         run: |
-#           curl -sSL -O https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip
-#           unzip GeoLite2-Country-CSV.zip
-#           rm -f GeoLite2-Country-CSV.zip
-#           mv GeoLite2* geoip
-
-#       - name: Generate geoip.dat file
-#         run: |
-#           go get -u -v -insecure $GEOIP_REPO
-#           geoip --country=./geoip/GeoLite2-Country-Locations-en.csv --ipv4=./geoip/GeoLite2-Country-Blocks-IPv4.csv --ipv6=./geoip/GeoLite2-Country-Blocks-IPv6.csv
-#           mkdir -p ./publish
-#           mv ./geoip.dat ./publish/
-
-      - name: Copy geoip.dat
+      - name: Get GeoLite2 file
         run: |
-          curl -sSL -O https://github.com/Loyalsoldier/v2ray-rules-dat/raw/master/geoip.dat
+          curl -L -o GeoLite2-Country-CSV.zip "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key=JvbzLLx7qBZT&suffix=zip"
+          unzip GeoLite2-Country-CSV.zip
+          rm -f GeoLite2-Country-CSV.zip
+          mv GeoLite2* geoip
+
+      - name: Generate geoip.dat file
+        run: |
+          go get -u -v -insecure $GEOIP_REPO
+          geoip --country=./geoip/GeoLite2-Country-Locations-en.csv --ipv4=./geoip/GeoLite2-Country-Blocks-IPv4.csv --ipv6=./geoip/GeoLite2-Country-Blocks-IPv6.csv
           mkdir -p ./publish
           mv ./geoip.dat ./publish/
 
@@ -63,60 +63,61 @@ jobs:
         run: |
           go get -u -v -insecure $GEOSITE_REPO
 
-      - name: Get and add gfwlist domains into temp-proxy.txt file
+      - name: Get and add direct domains into temp-direct.txt file
         run: |
-          curl -sSL $GFWLIST_URL > temp-proxy.txt
-      
-      - name: Get and add google domains into temp-proxy.txt file
-        run: |
-          curl -sSL $GOOGLE_URL | awk -F '/' '{print $2}' >> temp-proxy.txt
+          curl -sSL $CHINA_DOMAINS_URL | awk -F '/' '{print $2}' > temp-direct.txt
 
-      - name: Get and add apple domains into temp-proxy.txt file
+      - name: Get and add proxy domains into temp-proxy.txt file
         run: |
-          curl -sSL $APPLE_URL | awk -F '/' '{print $2}' >> temp-proxy.txt
-
-      - name: Get and add proxy domains from @ConnersHua/Profiles into temp-proxy.txt file
-        run: |
-          curl -sSL $Profiles_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+PROXY/ {print $2}' >> temp-proxy.txt
+          curl -sSL $GFWLIST_DOMAINS_URL > temp-proxy.txt
+          curl -sSL $GREATFIRE_DOMAINS_URL >> temp-proxy.txt
+          curl -sSL $GOOGLE_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-proxy.txt
+          curl -sSL $APPLE_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-proxy.txt
+          curl -sSL $PROFILES_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+PROXY/ {print $2}' >> temp-proxy.txt
+          curl -sSL $GEQ1AN_RULES_APPLE_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Apple/ {print $2}' >> temp-proxy.txt
+          curl -sSL $GEQ1AN_RULES_MICROSOFT_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Microsoft/ {print $2}' >> temp-proxy.txt
+          curl -sSL $GEQ1AN_RULES_GLOBAL_MEDIA_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+GMedia/ {print $2}' >> temp-proxy.txt
+          curl -sSL $GEQ1AN_RULES_OUTSIDE_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+Outside/ {print $2}' >> temp-proxy.txt
           
-      - name: Get and add blocked domains from @wongsyrone/domain-block-list into temp-proxy.txt file
+      - name: Get and add reject domains into temp-reject.txt file
         run: |
-          curl -sSL $Blocked_DOMAINS_URL >> temp-proxy.txt
+          curl -sSL $PROFILES_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+REJECT/ {print $2}' > temp-reject.txt
+          curl -sSL $GEQ1AN_RULES_REJECT_URL | awk -F ',' '/^(HOST|DOMAIN)(,|\-SUFFIX,).+AdBlock/ {print $2}' >> temp-reject.txt
+          curl -sSL $LHIE1_RULES_REJECT_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+REJECT/ {print $2}' >> temp-reject.txt
 
-      - name: Get and add direct domains from @ConnersHua/Profiles into temp-direct.txt file
-        run: |
-          curl -sSL $Profiles_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+DIRECT/ {print $2}' > temp-direct.txt
-
-      - name: Get and add chinalist domains into temp-direct.txt file
-        run: |
-          curl -sSL $CHINA_DOMAINS_URL | awk -F '/' '{print $2}' >> temp-direct.txt
-
-      - name: Get and add rejected domains from @ConnersHua/Profiles into category-ads-all
-        run: |
-          cd $GOPATH/src/$GEOSITE_REPO/data
-          curl -sSL $Profiles_URL | awk -F ',' '/^DOMAIN(,|\-SUFFIX,).+REJECT/ {print $2}' > profilereject
-          echo "include:profilereject" >> category-ads-all
-
-      - name: Add proxy and direct domains from the branch named hidden of this repo to appropriate temp files
+      - name: Add proxy and direct domains from "hidden" branch to appropriate temp files
         run: |
           cat proxy.txt >> temp-proxy.txt
           cat direct.txt >> temp-direct.txt
 
-      - name: Remove repeated domains and write domains to appropriate list
+      - name: Remove repeated domains, sort domains and write domains to appropriate list
         run: |
-          cat temp-proxy.txt | sort --ignore-case -u > $GOPATH/src/$GEOSITE_REPO/data/proxylist
-          cat temp-direct.txt | sort --ignore-case -u > $GOPATH/src/$GEOSITE_REPO/data/directlist
+          cat temp-proxy.txt | sort --ignore-case -u > proxy-sort.txt
+          cat temp-direct.txt | sort --ignore-case -u > direct-sort.txt
+          cat temp-reject.txt | sort --ignore-case -u > reject-sort.txt
 
-      - name: Add lists into appropriate category
+      - name: Filter valid domains
+        run: |
+          cat proxy-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/proxylist
+          cat proxy-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > proxy-filter.txt
+          cat direct-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/directlist
+          cat direct-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > direct-filter.txt
+          cat reject-sort.txt | awk '/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > $GOPATH/src/$GEOSITE_REPO/data/rejectlist
+          cat reject-sort.txt | awk '!/^([a-zA-Z0-9[.-.][._.]]+\.)+[a-zA-Z0-9[.-.]]+$/{print $0}' > reject-filter.txt
+
+      - name: Add list into appropriate category file
         run: |
           cd $GOPATH/src/$GEOSITE_REPO/data
           echo "include:proxylist" >> geolocation-\!cn
           echo "include:directlist" >> cn
+          echo "include:rejectlist" >> category-ads-all
 
       - name: Build geosite.dat file
         run: |
           domain-list-community
           mv ./dlc.dat ./publish/geosite.dat
+          mv ./*-filter.txt ./publish/
+          cp -f $GOPATH/src/$GEOSITE_REPO/data/{proxy,direct,reject}list ./publish/
 
       - name: Release dat files
         uses: Ricky-Hao/action-release@master

README.md

@@ -8,19 +8,15 @@
 
 - 通过仓库 [@v2ray/geoip](https://github.com/v2ray/geoip) 生成
 - 其中 IP 地址来源于 [MaxMind 免费 IP](https://dev.maxmind.com/geoip/geoip2/geolite2/)
-- **优点**：由于项目每天自动构建，所以更新速度比官方 `geoip.dat` 要快得多
 
 ### geosite.dat
 
 - 通过仓库 [@v2ray/domain-list-community](https://github.com/v2ray/domain-list-community) 生成
 - **加入大量中国大陆域名**：通过仓库 [@felixonmars/dnsmasq-china-list](https://github.com/felixonmars/dnsmasq-china-list) 生成并加入到 `geosite:cn` 类别中
 - **加入最新 GFWList 域名**：通过仓库 [@cokebar/gfwlist2dnsmasq](https://github.com/cokebar/gfwlist2dnsmasq) 生成并加入到 `geosite:geolocation-!cn` 类别中
-- **加入 Greatfire.org 检测到的屏蔽域名**：通过仓库 [@wongsyrone/domain-block-list](https://github.com/wongsyrone/domain-block-list) 获取 greatfire.org 检测到的屏蔽域名，并加入到 `geosite:geolocation-!cn` 类别中
-- **加入更多直连、代理、广告域名**：通过仓库 [@ConnersHua/Profiles](https://github.com/ConnersHua/Profiles/tree/master) 获取更多直连、代理、广告域名，并分别加入到 `geosite:cn`、`geosite:geolocation-!cn` 和 `geosite:category-ads-all` 类别中
+- **加入 Greatfire Analyzer 检测到的屏蔽域名**：通过仓库 [@wongsyrone/domain-block-list](https://github.com/wongsyrone/domain-block-list) 获取 [Greatfire Analyzer](https://zh.greatfire.org/analyzer) 检测到的屏蔽域名，并加入到 `geosite:geolocation-!cn` 类别中
+- **加入更多代理域名和广告域名**：通过仓库 [@ConnersHua/Profiles](https://github.com/ConnersHua/Profiles/tree/master)、[@GeQ1an/Rules](https://github.com/GeQ1an/Rules/tree/master/QuantumultX) 和 [@lhie1/Rules](https://github.com/lhie1/Rules/tree/master) 获取更多代理域名、广告域名，并分别加入到 `geosite:geolocation-!cn` 和 `geosite:category-ads-all` 类别中
 - **加入自定义直连和代理域名**：由于上游域名列表更新缓慢或缺失某些被屏蔽的域名，所以引入自定义域名列表，主要为了解决在 DNS 解析 `A` 和 `AAAA` 记录时的 DNS 泄漏问题。[`hidden` 分支](https://github.com/Loyalsoldier/v2ray-rules-dat/tree/hidden)里有两个文件 `direct.txt` 和 `proxy.txt`，分别放置自定义的直连、代理域名，最终分别加入到 `geosite:cn` 和 `geosite:geolocation-!cn` 类别中
-- **优点**：
-  - 由于项目每天自动构建，所以更新速度比官方 `geosite.dat` 要快得多
-  - 对从上述仓库和文件中获取的直连、代理域名进行简单去重，尽量减小规则文件体积
 
 ## 规则文件下载及使用方式
 
@@ -131,4 +127,6 @@
 - [@cokebar/gfwlist2dnsmasq](https://github.com/cokebar/gfwlist2dnsmasq)
 - [@wongsyrone/domain-block-list](https://github.com/wongsyrone/domain-block-list)
 - [@ConnersHua/Profiles](https://github.com/ConnersHua/Profiles/tree/master)
+- [@GeQ1an/Rules](https://github.com/GeQ1an/Rules/tree/master/QuantumultX)
+- [@lhie1/Rules](https://github.com/lhie1/Rules/tree/master)
 - [MaxMind 免费 IP](https://dev.maxmind.com/geoip/geoip2/geolite2/)