From bfe83a73058d7f176abddeae0ee6167897aa1d83 Mon Sep 17 00:00:00 2001 From: itdoginfo Date: Fri, 21 Jun 2024 11:14:35 +0300 Subject: [PATCH 1/4] Get subnets logic --- .github/workflows/create-lists.yml | 4 +- .github/workflows/create-subnets.yml | 33 ++++++++++++++ get-subnets.py | 66 ++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/create-subnets.yml create mode 100755 get-subnets.py diff --git a/.github/workflows/create-lists.yml b/.github/workflows/create-lists.yml index fc62d13..afe873a 100644 --- a/.github/workflows/create-lists.yml +++ b/.github/workflows/create-lists.yml @@ -4,9 +4,9 @@ on: push: branches: [ "main" ] paths: - - .github/** + - .github/create-lists.yml - src/** - - '*.py' + - convert.py schedule: - cron: '29 */8 * * *' diff --git a/.github/workflows/create-subnets.yml b/.github/workflows/create-subnets.yml new file mode 100644 index 0000000..832cc49 --- /dev/null +++ b/.github/workflows/create-subnets.yml @@ -0,0 +1,33 @@ +name: Create subnets + +on: + push: + branches: [ "subnets" ] + paths: + - .github/create-subnets.yml + - get-subnets.py + schedule: + - cron: '15 7 * * 1' + +permissions: + contents: write + +jobs: + generate: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4.1.7 + - name: Generate subnets + uses: actions/setup-python@v5.1.0 + with: + python-version: '3.10' + - run: | + python get-subnets.py + - name: Push subnets + uses: EndBug/add-and-commit@v9.1.4 + with: + add: 'Subnets' + author_name: GitHub Action + author_email: githubaction@githubaction.com + message: 'Update subnet' + push: true \ No newline at end of file diff --git a/get-subnets.py b/get-subnets.py new file mode 100755 index 0000000..f84be58 --- /dev/null +++ b/get-subnets.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3.10 + +import ipaddress +import urllib.request + +BGP_TOOLS_URL = 'https://bgp.tools/table.txt' +USER_AGENT = 'itdog.info - hi@itdog.info' +AS_FILE = 'AS.lst' +IPv4_DIR = 'Subnets/IPv4' +IPv6_DIR = 'Subnets/IPv6' + +AS_META = '32934' +AS_TWITTER = '13414' +META = 'Meta.lst' +TWITTER = 'Twitter.lst' + +subnet_list = [] + +def subnet_summarization(subnet_list): + subnets = [ipaddress.ip_network(subnet) for subnet in subnet_list] + return list(ipaddress.collapse_addresses(subnets)) + +def process_subnets(subnet_list, target_as): + ipv4_subnets = [] + ipv6_subnets = [] + + for subnet_str, as_number in subnet_list: + try: + subnet = ipaddress.ip_network(subnet_str) + if as_number == target_as: + if subnet.version == 4: + ipv4_subnets.append(subnet_str) + elif subnet.version == 6: + ipv6_subnets.append(subnet_str) + except ValueError: + print(f"Invalid subnet: {subnet_str}") + sys.exit(1) + + ipv4_merged = subnet_summarization(ipv4_subnets) + ipv6_merged = subnet_summarization(ipv6_subnets) + + return ipv4_merged, ipv6_merged + +def write_subnets_to_file(subnets, filename): + with open(filename, 'w') as file: + for subnet in subnets: + file.write(f'{subnet}\n') + +if __name__ == '__main__': + request = urllib.request.Request(BGP_TOOLS_URL, headers={'User-Agent': USER_AGENT}) + + with urllib.request.urlopen(request) as response: + for line in response: + decoded_line = line.decode('utf-8').strip() + subnet, as_number = decoded_line.split() + subnet_list.append((subnet, as_number)) + + # Meta + ipv4_merged_meta, ipv6_merged_meta = process_subnets(subnet_list, AS_META) + write_subnets_to_file(ipv4_merged_meta, f'{IPv4_DIR}/{META}') + write_subnets_to_file(ipv6_merged_meta, f'{IPv6_DIR}/{META}') + + # Twitter + ipv4_merged_twitter, ipv6_merged_twitter = process_subnets(subnet_list, AS_TWITTER) + write_subnets_to_file(ipv4_merged_twitter, f'{IPv4_DIR}/{TWITTER}') + write_subnets_to_file(ipv6_merged_twitter, f'{IPv6_DIR}/{TWITTER}') \ No newline at end of file From 7d41e7b9589fb8aec4c92be8a85243cbaa8fbaeb Mon Sep 17 00:00:00 2001 From: itdoginfo Date: Fri, 21 Jun 2024 11:17:33 +0300 Subject: [PATCH 2/4] Subnets dir --- Subnets/IPv4/Meta.lst | 26 ++++++++++++++++++++++++++ Subnets/IPv4/Twitter.lst | 13 +++++++++++++ Subnets/IPv6/Meta.lst | 2 ++ Subnets/IPv6/Twitter.lst | 3 +++ 4 files changed, 44 insertions(+) create mode 100644 Subnets/IPv4/Meta.lst create mode 100644 Subnets/IPv4/Twitter.lst create mode 100644 Subnets/IPv6/Meta.lst create mode 100644 Subnets/IPv6/Twitter.lst diff --git a/Subnets/IPv4/Meta.lst b/Subnets/IPv4/Meta.lst new file mode 100644 index 0000000..27a6b54 --- /dev/null +++ b/Subnets/IPv4/Meta.lst @@ -0,0 +1,26 @@ +31.13.24.0/21 +31.13.64.0/18 +45.64.40.0/22 +57.141.0.0/24 +57.141.3.0/24 +57.141.5.0/24 +57.141.7.0/24 +57.141.8.0/24 +57.141.10.0/24 +57.141.13.0/24 +57.144.0.0/14 +66.220.144.0/20 +69.63.176.0/20 +69.171.224.0/19 +74.119.76.0/22 +102.132.96.0/20 +103.4.96.0/22 +129.134.0.0/17 +157.240.0.0/17 +157.240.192.0/18 +163.70.128.0/17 +173.252.64.0/18 +179.60.192.0/22 +185.60.216.0/22 +185.89.216.0/22 +204.15.20.0/22 diff --git a/Subnets/IPv4/Twitter.lst b/Subnets/IPv4/Twitter.lst new file mode 100644 index 0000000..b01ac87 --- /dev/null +++ b/Subnets/IPv4/Twitter.lst @@ -0,0 +1,13 @@ +64.63.0.0/18 +69.195.160.0/19 +103.252.112.0/22 +104.244.40.0/23 +104.244.42.0/24 +104.244.44.0/22 +188.64.224.0/21 +192.133.76.0/22 +199.16.156.0/22 +199.59.148.0/22 +199.96.56.0/23 +202.160.128.0/22 +209.237.192.0/19 diff --git a/Subnets/IPv6/Meta.lst b/Subnets/IPv6/Meta.lst new file mode 100644 index 0000000..454fd2e --- /dev/null +++ b/Subnets/IPv6/Meta.lst @@ -0,0 +1,2 @@ +2620:0:1c00::/40 +2a03:2880::/32 diff --git a/Subnets/IPv6/Twitter.lst b/Subnets/IPv6/Twitter.lst new file mode 100644 index 0000000..ae3eba2 --- /dev/null +++ b/Subnets/IPv6/Twitter.lst @@ -0,0 +1,3 @@ +2400:6680:f000::/36 +2606:1f80:f000::/36 +2a04:9d40:f000::/36 From 424e08dee8a2a5aefb52f19e625df90743f41475 Mon Sep 17 00:00:00 2001 From: itdoginfo Date: Fri, 21 Jun 2024 11:19:18 +0300 Subject: [PATCH 3/4] Test action --- .github/workflows/create-subnets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-subnets.yml b/.github/workflows/create-subnets.yml index 832cc49..7be3647 100644 --- a/.github/workflows/create-subnets.yml +++ b/.github/workflows/create-subnets.yml @@ -7,7 +7,7 @@ on: - .github/create-subnets.yml - get-subnets.py schedule: - - cron: '15 7 * * 1' + - cron: '16 7 * * 1' permissions: contents: write From d6d38337c35974fdad7754b55ae8f21b426bd6d2 Mon Sep 17 00:00:00 2001 From: itdoginfo Date: Fri, 21 Jun 2024 11:21:26 +0300 Subject: [PATCH 4/4] Fix path --- .github/workflows/create-lists.yml | 2 +- .github/workflows/create-subnets.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-lists.yml b/.github/workflows/create-lists.yml index afe873a..b113860 100644 --- a/.github/workflows/create-lists.yml +++ b/.github/workflows/create-lists.yml @@ -4,7 +4,7 @@ on: push: branches: [ "main" ] paths: - - .github/create-lists.yml + - .github/workflows/create-lists.yml - src/** - convert.py schedule: diff --git a/.github/workflows/create-subnets.yml b/.github/workflows/create-subnets.yml index 7be3647..f4cacf5 100644 --- a/.github/workflows/create-subnets.yml +++ b/.github/workflows/create-subnets.yml @@ -4,7 +4,7 @@ on: push: branches: [ "subnets" ] paths: - - .github/create-subnets.yml + - .github/workflows/create-subnets.yml - get-subnets.py schedule: - cron: '16 7 * * 1'