自行定制geoip.dat和h2y.dat

Home / 系统运维 / 自行定制geoip.dat和h2y.dat

下载Loyalsoldier/geoip：

git clone https://github.com/Loyalsoldier/geoip.git
cd geoip

config.json:

{
  "input": [
    {
      "type": "maxmindGeoLite2CountryCSV",
      "action": "add",
      "args": {
        "country": "./geolite2/GeoLite2-Country-Locations-en.csv",
        "ipv4": "./geolite2/GeoLite2-Country-Blocks-IPv4.csv",
        "ipv6": "./geolite2/GeoLite2-Country-Blocks-IPv6.csv"
      }
    },
    {
      "type": "cutter",
      "action": "remove",
      "args": {
        "wantedList": ["cn"],
        "onlyIPType": "ipv4"
      }
    },
    {
      "type": "text",
      "action": "add",
      "args": {
        "name": "cn",
        "uri": "https://raw.githubusercontent.com/17mon/china_ip_list/master/china_ip_list.txt",
        "onlyIPType": "ipv4"
      }
    },
    {
      "type": "text",
      "action": "add",
      "args": {
        "name": "cn",
        "uri": "https://raw.githubusercontent.com/gaoyifan/china-operator-ip/ip-lists/china.txt",
        "onlyIPType": "ipv4"
      }
    },
    {
      "type": "text",
      "action": "add",
      "args": {
        "name": "cn",
        "uri": "https://raw.githubusercontent.com/gaoyifan/china-operator-ip/ip-lists/china6.txt",
        "onlyIPType": "ipv6"
      }
    },
    {
      "type": "text",
      "action": "add",
      "args": {
        "name": "tor",
        "uri": "https://check.torproject.org/torbulkexitlist"
      }
    },
    {
      "type": "text",
      "action": "add",
      "args": {
        "name": "cloudflare",
        "uri": "https://www.cloudflare.com/ips-v4"
      }
    },
    {
      "type": "text",
      "action": "add",
      "args": {
        "name": "cloudflare",
        "uri": "https://www.cloudflare.com/ips-v6"
      }
    },
    {
      "type": "text",
      "action": "add",
      "args": {
        "name": "telegram",
        "uri": "https://core.telegram.org/resources/cidr.txt"
      }
    },
    {
      "type": "json",
      "action": "add",
      "args": {
        "name": "google",
        "uri": "https://www.gstatic.com/ipranges/goog.json",
        "jsonPath": ["prefixes.#.ipv4Prefix", "prefixes.#.ipv6Prefix"]
      }
    },
    {
      "type": "json",
      "action": "add",
      "args": {
        "name": "google",
        "uri": "https://www.gstatic.com/ipranges/cloud.json",
        "jsonPath": ["prefixes.#.ipv4Prefix", "prefixes.#.ipv6Prefix"]
      }
    },
    {
      "type": "json",
      "action": "add",
      "args": {
        "name": "fastly",
        "uri": "https://api.fastly.com/public-ip-list",
        "jsonPath": ["addresses", "ipv6_addresses"]
      }
    },
    {
      "type": "json",
      "action": "add",
      "args": {
        "name": "cloudfront",
        "uri": "https://ip-ranges.amazonaws.com/ip-ranges.json",
        "jsonPath": [
          "prefixes.#(service==\"CLOUDFRONT\")#.ip_prefix",
          "ipv6_prefixes.#(service==\"CLOUDFRONT\")#.ipv6_prefix"
        ]
      }
    },
    {
      "type": "maxmindGeoLite2ASNCSV",
      "action": "add",
      "args": {
        "ipv4": "./geolite2/GeoLite2-ASN-Blocks-IPv4.csv",
        "ipv6": "./geolite2/GeoLite2-ASN-Blocks-IPv6.csv",
        "wantedList": {
          "cloudflare": [
            "AS395747",
            "AS394536",
            "AS209242",
            "AS203898",
            "AS202623",
            "AS14789",
            "AS139242",
            "AS133877",
            "AS13335",
            "AS132892"
          ],
          "facebook": ["AS63293", "AS54115", "AS32934"],
          "fastly": ["AS54113", "AS394192"],
          "google": [
            "AS6432",
            "AS55023",
            "AS45566",
            "AS43515",
            "AS41264",
            "AS40873",
            "AS396982",
            "AS395973",
            "AS394699",
            "AS394639",
            "AS394507",
            "AS36987",
            "AS36492",
            "AS36385",
            "AS36384",
            "AS36040",
            "AS36039",
            "AS26910",
            "AS26684",
            "AS22859",
            "AS22577",
            "AS19527",
            "AS19448",
            "AS19425",
            "AS16591",
            "AS16550",
            "AS15169",
            "AS13949",
            "AS139190",
            "AS139070"
          ],
          "netflix": ["AS55095", "AS40027", "AS394406", "AS2906"],
          "telegram": ["AS62041", "AS62014", "AS59930", "AS44907", "AS211157"],
          "twitter": ["AS8945", "AS63179", "AS54888", "AS35995", "AS13414"]
        }
      }
    },
    {
      "type": "private",
      "action": "add"
    }
  ],
  "output": [
    {
      "type": "v2rayGeoIPDat",
      "action": "output",
      "args": {
        "outputDir": "./output",
        "outputName": "geoip.dat"
      }
    },
    {
      "type": "v2rayGeoIPDat",
      "action": "output",
      "args": {
        "outputDir": "./output",
        "outputName": "geoip-only-cn-private.dat",
        "wantedList": ["cn", "private"]
      }
    },
    {
      "type": "v2rayGeoIPDat",
      "action": "output",
      "args": {
        "outputDir": "./output",
        "outputName": "geoip-asn.dat",
        "wantedList": [
          "cloudflare",
          "cloudfront",
          "facebook",
          "fastly",
          "google",
          "netflix",
          "telegram",
          "twitter",
          "tor"
        ]
      }
    },
    {
      "type": "v2rayGeoIPDat",
      "action": "output",
      "args": {
        "outputDir": "./output",
        "oneFilePerList": true,
        "wantedList": ["cn", "private"]
      }
    },
    {
      "type": "v2rayGeoIPDat",
      "action": "output",
      "args": {
        "oneFilePerList": true
      }
    },
    {
      "type": "maxmindMMDB",
      "action": "output",
      "args": {
        "outputDir": "./output",
        "outputName": "Country.mmdb",
        "sourceMMDBURI": "./geolite2/GeoLite2-Country.mmdb",
        "overwriteList": [
          "cn",
          "cloudflare",
          "cloudfront",
          "facebook",
          "fastly",
          "google",
          "netflix",
          "telegram",
          "twitter",
          "tor",
          "private"
        ]
      }
    },
    {
      "type": "maxmindMMDB",
      "action": "output",
      "args": {
        "outputDir": "./output",
        "outputName": "Country-without-asn.mmdb",
        "sourceMMDBURI": "./geolite2/GeoLite2-Country.mmdb",
        "overwriteList": ["cn", "private"],
        "excludedList": [
          "cloudflare",
          "cloudfront",
          "facebook",
          "fastly",
          "google",
          "netflix",
          "telegram",
          "twitter",
          "tor"
        ]
      }
    },
    {
      "type": "maxmindMMDB",
      "action": "output",
      "args": {
        "outputDir": "./output",
        "outputName": "Country-only-cn-private.mmdb",
        "sourceMMDBURI": "./geolite2/GeoLite2-Country.mmdb",
        "wantedList": ["cn", "private"]
      }
    },
    {
      "type": "maxmindMMDB",
      "action": "output",
      "args": {
        "outputDir": "./output",
        "outputName": "Country-asn.mmdb",
        "wantedList": [
          "cloudflare",
          "cloudfront",
          "facebook",
          "fastly",
          "google",
          "netflix",
          "telegram",
          "twitter",
          "tor"
        ]
      }
    },
    {
      "type": "singboxSRS",
      "action": "output"
    },
    {
      "type": "mihomoMRS",
      "action": "output"
    },
    {
      "type": "text",
      "action": "output"
    },
    {
      "type": "text",
      "action": "output",
      "args": {
        "outputDir": "./output/nginx/allow",
        "outputExtension": ".conf",
        "addPrefixInLine": "allow ",
        "addSuffixInLine": ";"
      }
    },
    {
      "type": "text",
      "action": "output",
      "args": {
        "outputDir": "./output/nginx/deny",
        "outputExtension": ".conf",
        "addPrefixInLine": "deny ",
        "addSuffixInLine": ";"
      }
    },
    {
      "type": "clashRuleSetClassical",
      "action": "output"
    },
    {
      "type": "clashRuleSet",
      "action": "output"
    },
    {
      "type": "surgeRuleSet",
      "action": "output",
      "args": {
        "addSuffixInLine": ",no-resolve"
      }
    }
  ]
}

运行代码：

DownloadFile.sh：

#!/bin/bash
RequiredFile=(GeoLite2-ASN-Blocks-IPv4.csv GeoLite2-ASN-Blocks-IPv6.csv GeoLite2-ASN.mmdb GeoLite2-Country-Blocks-IPv4.csv GeoLite2-Country-Blocks-IPv6.csv GeoLite2-Country-Locations-en.csv GeoLite2-Country-Locations-zh-CN.csv GeoLite2-Country.mmdb)
Agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36"
for ((i = 0; i < ${#RequiredFile[@]}; i++)); do
    Header="X-Forwarded-For: $(echo | awk -v A="$(date +%S%N)" '{srand();a=int(A*rand())%254+1;srand();b=int(A*rand())%254+1;srand();c=int(A*rand())%254+1;srand();d=int(A*rand())%253+1;printf"%s.%s.%s.%s",a,b,c,d}')"
    curl -H "$Header" -A "$Agent" -e - -m 300 --retry 6 -x 127.0.0.1:3801 -sLk -o - "https://raw.githubusercontent.com/Loyalsoldier/geoip/refs/heads/release/${RequiredFile[$i]}" >${RequiredFile[$i]}
    if [ $? -ne 0 ]; then
        echo "Download ${RequiredFile[$i]} failed!"
        exit 1
    fi
    if [ ! -f ${RequiredFile[$i]} ]; then
        echo "Download ${RequiredFile[$i]} failed!"
        exit 1
    else
        echo "Download ${RequiredFile[$i]} success!"
    fi
done
mkdir geolite2
cd geolite2
bash ./DownloadFile.sh
go run ./ convert -c ./config.json


如果遇到EOF问题报错，解决方法：
/bin/rm -rf /root/.cache/go-build/
go clean -modcache

下载：https://github.com/onplus/v2ray-SiteDAT/releases/download/0.1/v2sitedat_linux_amd64.zip

解压zip包时候如果遇到类似：need PK compat. v6.3 (can do v4.6) 错误，解决方法：

apt-get install p7zip-full

7z x v2sitedat_linux_amd64.zip

cd v2sitedat_linux_amd64/sites
/bin/rm -f cn direct proxy reject
wget -q -O ./gfw https://raw.githubusercontent.com/yunhaoz659/h2y/refs/heads/main/gfw
wget -q -O ./ad  https://raw.githubusercontent.com/yunhaoz659/h2y/refs/heads/main/ad
wget -q -O ./cn  https://raw.githubusercontent.com/yunhaoz659/h2y/refs/heads/main/cn
cd ..
mkdir output
./v2sitedat -dat ./output/h2y.dat -dir ./sites/
"domain": [
  "ext:h2y.dat:gfw"
]

gfw、cn、ad域名库的更新脚本

#!/bin/bash
export LANG=C
#从gfwlist原生文件中提取出”域名”
curl -4sLk -o - "https://raw.githubusercontent.com/gfwlist/gfwlist/refs/heads/master/gfwlist.txt" |base64 -d |grep -E "^(\||\|\||[a-zA-Z0-9.-])" |grep -E -o "[a-zA-Z0-9.-]+\.[a-zA-Z0-9-]{2,}" |sed 's/^[^a-zA-Z0-9-]//g' |sort -u >/tmp/gfwlist.tmp
curl -4sLk -o - "https://raw.githubusercontent.com/gfwlist/gfwlist/refs/heads/master/gfwlist.txt" |base64 -d |grep -E "^/" |grep -Eo "google\\\..*\)" |grep -Eo "\(.*\)" |sed 's/)\|(//g'|awk '{b=split($0,a,"|")}END{for(i=1;i<=b;i++){printf"%s.%s\n","google",a[i]}}' >>/tmp/gfwlist.tmp
#从pexcn项目的gfwlist文件中提取”域名”并与原生追加
curl -4sLk -o - "https://raw.githubusercontent.com/pexcn/daily/gh-pages/shadowrocket/gfwlist.conf" |awk -F',' '{if($1=="DOMAIN-SUFFIX" && $3=="PROXY"){print $2}}' >>/tmp/gfwlist.tmp
#去重性过滤
sed -i 's/^www\.//g' /tmp/gfwlist.tmp
sort -u /tmp/gfwlist.tmp >/tmp/gfwlist.sort
/bin/mv -f /tmp/gfwlist.sort /tmp/gfwlist.tmp
#有效性过滤，筛出”真域名”
mapfile -t TOPDS < <(awk -F. '{print $NF}' /tmp/gfwlist.tmp|sort -u |awk '{printf"%s ",$1}')
cat /dev/null >/tmp/gfwlist.valid
threads=3
fifofile=/tmp/$$.fifo
mkfifo $fifofile
exec 6<>$fifofile
/bin/rm $fifofile
for ((i=0;i<$threads;i++));do
  echo >&6
done
for i in ${TOPDS[@]};do
  [ -z "$i" ] && continue
  read -u6
  {
    IFVALID=$(dig ${i}. |awk '/IN.*SOA/{if($1==a){print 1}else{print 0}}/^[^;].*IN.*A/{if($1==a){print 1}else{print 0}}' a="${i}." |tail -1)
    if [ "$IFVALID" -eq "1" ];then
      grep -E "\.${i}$" /tmp/gfwlist.tmp >>/tmp/gfwlist.valid
    fi
    echo >&6
  }&
done
wait
exec 6>&- 6<&-
sort -u /tmp/gfwlist.valid |grep -E -o "[a-zA-Z0-9.-]+\.[a-zA-Z0-9-]{2,}" >/root/gfw
#删除临时文件
/bin/rm -f /tmp/gfwlist.*








#从privacy-protection-tools项目的anti-AD文件中提取出”域名”
curl -4sLk -o - "https://raw.githubusercontent.com/privacy-protection-tools/anti-AD/refs/heads/master/anti-ad-surge2.txt" |awk '/^\./{gsub("^\.","",$1);print $1}' >>/tmp/ad.tmp
#从pexcn项目的gfwlist文件中提取”域名”并与上面项目追加
curl -4sLk -o - "https://raw.githubusercontent.com/pexcn/daily/gh-pages/adblock/adblock.conf" |awk -F'/' '{if($1=="server="){print $2}}' >>/tmp/ad.tmp
#去重性过滤
sed -i 's/^\.//g' /tmp/ad.tmp
sort -u /tmp/ad.tmp >/tmp/ad.sort
/bin/mv -f /tmp/ad.sort /tmp/ad.tmp
#有效性过滤，筛出”真域名”
mapfile -t TOPDS < <(awk -F. '{print $NF}' /tmp/ad.tmp|sort -u |awk '{printf"%s ",$1}')
cat /dev/null >/tmp/ad.valid
threads=3
fifofile=/tmp/$$.fifo
mkfifo $fifofile
exec 6<>$fifofile
/bin/rm $fifofile
for ((i=0;i<$threads;i++));do
  echo >&6
done
for i in ${TOPDS[@]};do
  [ -z "$i" ] && continue
  read -u6
  {
    IFVALID=$(dig ${i}. |awk '/IN.*SOA/{if($1==a){print 1}else{print 0}}/^[^;].*IN.*A/{if($1==a){print 1}else{print 0}}' a="${i}." |tail -1)
    if [ "$IFVALID" -eq "1" ];then
      grep -E "\.${i}$" /tmp/ad.tmp >>/tmp/ad.valid
    fi
    echo >&6
  }&
done
wait
exec 6>&- 6<&-
sort -u /tmp/ad.valid |grep -E -o "[a-zA-Z0-9.-]+\.[a-zA-Z0-9-]{2,}" >/root/ad
#删除临时文件
/bin/rm -f /tmp/ad.*








#从pexcn项目的chinalist文件中提取”域名”
curl -4sLk -o - "https://raw.githubusercontent.com/pexcn/daily/gh-pages/chinalist/chinalist.txt" >/root/cn








if [ -d "/data/github/h2y/.git" ];then
  /bin/mv -f /root/ad /root/cn /root/gfw /data/github/h2y
  cd /data/github/h2y/
  git add .
  git commit -m "$(date +%Y%m%d) 更新文件"
  git push origin main
fi
Categories: 系统运维

Related Posts:-
zhangyunhao 0 comments

wordpress批量关闭文章的评论功能

mysql> SET sql_mode = 'ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION'; mysql> alter table wp_posts alter…
Read More
zhangyunhao 0 comments

给 WordPress 配置 Nginx HTTPS 反向代理

给 WordPress 配置 Nginx HTTPS 反向代理后，网站的 CSS、JS 等静态资源链接都变成了 HTTP 协议，导致浏览器报混合内容错误，资源加载异常…
Read More
zhangyunhao 0 comments

nodejs打包项目

1.安装node https://nodejs.org/dist/v18.20.4/node-v18.20.4-x64.msi 2.安装vscode https://vscode.download.prss.microsoft.com/dbazure/download/stable/f1e16e1e6214d7c44d078b1f0607b2388f29d729/VSCodeUserSetup-x64-1.91.1.exe 3.项目目录里空白地方右击 通过Code打开 4.Terminal New Terminal 5.删除node_modules、package-lock.json…
Read More