diff --git a/.github/workflows/create-lists.yml b/.github/workflows/create-lists.yml index fc62d13..b113860 100644 --- a/.github/workflows/create-lists.yml +++ b/.github/workflows/create-lists.yml @@ -4,9 +4,9 @@ on: push: branches: [ "main" ] paths: - - .github/** + - .github/workflows/create-lists.yml - src/** - - '*.py' + - convert.py schedule: - cron: '29 */8 * * *' diff --git a/.github/workflows/create-subnets.yml b/.github/workflows/create-subnets.yml new file mode 100644 index 0000000..f4cacf5 --- /dev/null +++ b/.github/workflows/create-subnets.yml @@ -0,0 +1,33 @@ +name: Create subnets + +on: + push: + branches: [ "subnets" ] + paths: + - .github/workflows/create-subnets.yml + - get-subnets.py + schedule: + - cron: '16 7 * * 1' + +permissions: + contents: write + +jobs: + generate: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4.1.7 + - name: Generate subnets + uses: actions/setup-python@v5.1.0 + with: + python-version: '3.10' + - run: | + python get-subnets.py + - name: Push subnets + uses: EndBug/add-and-commit@v9.1.4 + with: + add: 'Subnets' + author_name: GitHub Action + author_email: githubaction@githubaction.com + message: 'Update subnet' + push: true \ No newline at end of file diff --git a/Subnets/IPv4/Meta.lst b/Subnets/IPv4/Meta.lst new file mode 100644 index 0000000..27a6b54 --- /dev/null +++ b/Subnets/IPv4/Meta.lst @@ -0,0 +1,26 @@ +31.13.24.0/21 +31.13.64.0/18 +45.64.40.0/22 +57.141.0.0/24 +57.141.3.0/24 +57.141.5.0/24 +57.141.7.0/24 +57.141.8.0/24 +57.141.10.0/24 +57.141.13.0/24 +57.144.0.0/14 +66.220.144.0/20 +69.63.176.0/20 +69.171.224.0/19 +74.119.76.0/22 +102.132.96.0/20 +103.4.96.0/22 +129.134.0.0/17 +157.240.0.0/17 +157.240.192.0/18 +163.70.128.0/17 +173.252.64.0/18 +179.60.192.0/22 +185.60.216.0/22 +185.89.216.0/22 +204.15.20.0/22 diff --git a/Subnets/IPv4/Twitter.lst b/Subnets/IPv4/Twitter.lst new file mode 100644 index 0000000..b01ac87 --- /dev/null +++ b/Subnets/IPv4/Twitter.lst @@ -0,0 +1,13 @@ +64.63.0.0/18 +69.195.160.0/19 +103.252.112.0/22 +104.244.40.0/23 +104.244.42.0/24 +104.244.44.0/22 +188.64.224.0/21 +192.133.76.0/22 +199.16.156.0/22 +199.59.148.0/22 +199.96.56.0/23 +202.160.128.0/22 +209.237.192.0/19 diff --git a/Subnets/IPv6/Meta.lst b/Subnets/IPv6/Meta.lst new file mode 100644 index 0000000..454fd2e --- /dev/null +++ b/Subnets/IPv6/Meta.lst @@ -0,0 +1,2 @@ +2620:0:1c00::/40 +2a03:2880::/32 diff --git a/Subnets/IPv6/Twitter.lst b/Subnets/IPv6/Twitter.lst new file mode 100644 index 0000000..ae3eba2 --- /dev/null +++ b/Subnets/IPv6/Twitter.lst @@ -0,0 +1,3 @@ +2400:6680:f000::/36 +2606:1f80:f000::/36 +2a04:9d40:f000::/36 diff --git a/get-subnets.py b/get-subnets.py new file mode 100755 index 0000000..f84be58 --- /dev/null +++ b/get-subnets.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3.10 + +import ipaddress +import urllib.request + +BGP_TOOLS_URL = 'https://bgp.tools/table.txt' +USER_AGENT = 'itdog.info - hi@itdog.info' +AS_FILE = 'AS.lst' +IPv4_DIR = 'Subnets/IPv4' +IPv6_DIR = 'Subnets/IPv6' + +AS_META = '32934' +AS_TWITTER = '13414' +META = 'Meta.lst' +TWITTER = 'Twitter.lst' + +subnet_list = [] + +def subnet_summarization(subnet_list): + subnets = [ipaddress.ip_network(subnet) for subnet in subnet_list] + return list(ipaddress.collapse_addresses(subnets)) + +def process_subnets(subnet_list, target_as): + ipv4_subnets = [] + ipv6_subnets = [] + + for subnet_str, as_number in subnet_list: + try: + subnet = ipaddress.ip_network(subnet_str) + if as_number == target_as: + if subnet.version == 4: + ipv4_subnets.append(subnet_str) + elif subnet.version == 6: + ipv6_subnets.append(subnet_str) + except ValueError: + print(f"Invalid subnet: {subnet_str}") + sys.exit(1) + + ipv4_merged = subnet_summarization(ipv4_subnets) + ipv6_merged = subnet_summarization(ipv6_subnets) + + return ipv4_merged, ipv6_merged + +def write_subnets_to_file(subnets, filename): + with open(filename, 'w') as file: + for subnet in subnets: + file.write(f'{subnet}\n') + +if __name__ == '__main__': + request = urllib.request.Request(BGP_TOOLS_URL, headers={'User-Agent': USER_AGENT}) + + with urllib.request.urlopen(request) as response: + for line in response: + decoded_line = line.decode('utf-8').strip() + subnet, as_number = decoded_line.split() + subnet_list.append((subnet, as_number)) + + # Meta + ipv4_merged_meta, ipv6_merged_meta = process_subnets(subnet_list, AS_META) + write_subnets_to_file(ipv4_merged_meta, f'{IPv4_DIR}/{META}') + write_subnets_to_file(ipv6_merged_meta, f'{IPv6_DIR}/{META}') + + # Twitter + ipv4_merged_twitter, ipv6_merged_twitter = process_subnets(subnet_list, AS_TWITTER) + write_subnets_to_file(ipv4_merged_twitter, f'{IPv4_DIR}/{TWITTER}') + write_subnets_to_file(ipv6_merged_twitter, f'{IPv6_DIR}/{TWITTER}') \ No newline at end of file