Merge pull request #17 from itdoginfo/subnets

Adding subnets

.github/workflows/create-lists.yml

@@ -4,9 +4,9 @@ on:
   push:
     branches: [ "main" ]
     paths:
-      - .github/**
+      - .github/workflows/create-lists.yml
       - src/**
-      - '*.py'
+      - convert.py
   schedule:
     - cron: '29 */8 * * *'
 

.github/workflows/create-subnets.yml

@@ -0,0 +1,33 @@
+name: Create subnets
+
+on:
+  push:
+    branches: [ "subnets" ]
+    paths:
+      - .github/workflows/create-subnets.yml
+      - get-subnets.py
+  schedule:
+    - cron: '16 7 * * 1'
+
+permissions:
+  contents: write
+
+jobs:
+  generate:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - name: Generate subnets
+      uses: actions/setup-python@v5.1.0
+      with:
+        python-version: '3.10'
+    - run: |
+        python get-subnets.py
+    - name: Push subnets
+      uses: EndBug/add-and-commit@v9.1.4
+      with:
+        add: 'Subnets'
+        author_name: GitHub Action
+        author_email: githubaction@githubaction.com
+        message: 'Update subnet'
+        push: true

Subnets/IPv4/Meta.lst

@@ -0,0 +1,26 @@
+31.13.24.0/21
+31.13.64.0/18
+45.64.40.0/22
+57.141.0.0/24
+57.141.3.0/24
+57.141.5.0/24
+57.141.7.0/24
+57.141.8.0/24
+57.141.10.0/24
+57.141.13.0/24
+57.144.0.0/14
+66.220.144.0/20
+69.63.176.0/20
+69.171.224.0/19
+74.119.76.0/22
+102.132.96.0/20
+103.4.96.0/22
+129.134.0.0/17
+157.240.0.0/17
+157.240.192.0/18
+163.70.128.0/17
+173.252.64.0/18
+179.60.192.0/22
+185.60.216.0/22
+185.89.216.0/22
+204.15.20.0/22

Subnets/IPv4/Twitter.lst

@@ -0,0 +1,13 @@
+64.63.0.0/18
+69.195.160.0/19
+103.252.112.0/22
+104.244.40.0/23
+104.244.42.0/24
+104.244.44.0/22
+188.64.224.0/21
+192.133.76.0/22
+199.16.156.0/22
+199.59.148.0/22
+199.96.56.0/23
+202.160.128.0/22
+209.237.192.0/19

Subnets/IPv6/Meta.lst

@@ -0,0 +1,2 @@
+2620:0:1c00::/40
+2a03:2880::/32

Subnets/IPv6/Twitter.lst

@@ -0,0 +1,3 @@
+2400:6680:f000::/36
+2606:1f80:f000::/36
+2a04:9d40:f000::/36

get-subnets.py

@@ -0,0 +1,66 @@
+#!/usr/bin/python3.10
+
+import ipaddress
+import urllib.request
+
+BGP_TOOLS_URL = 'https://bgp.tools/table.txt'
+USER_AGENT = 'itdog.info - hi@itdog.info'
+AS_FILE = 'AS.lst'
+IPv4_DIR = 'Subnets/IPv4'
+IPv6_DIR = 'Subnets/IPv6'
+
+AS_META = '32934'
+AS_TWITTER = '13414'
+META = 'Meta.lst'
+TWITTER = 'Twitter.lst'
+
+subnet_list = []
+
+def subnet_summarization(subnet_list):
+    subnets = [ipaddress.ip_network(subnet) for subnet in subnet_list]
+    return list(ipaddress.collapse_addresses(subnets))
+
+def process_subnets(subnet_list, target_as):
+    ipv4_subnets = []
+    ipv6_subnets = []
+
+    for subnet_str, as_number in subnet_list:
+        try:
+            subnet = ipaddress.ip_network(subnet_str)
+            if as_number == target_as:
+                if subnet.version == 4:
+                    ipv4_subnets.append(subnet_str)
+                elif subnet.version == 6:
+                    ipv6_subnets.append(subnet_str)
+        except ValueError:
+            print(f"Invalid subnet: {subnet_str}")
+            sys.exit(1)
+
+    ipv4_merged = subnet_summarization(ipv4_subnets)
+    ipv6_merged = subnet_summarization(ipv6_subnets)
+
+    return ipv4_merged, ipv6_merged
+
+def write_subnets_to_file(subnets, filename):
+    with open(filename, 'w') as file:
+        for subnet in subnets:
+            file.write(f'{subnet}\n')
+
+if __name__ == '__main__':
+    request = urllib.request.Request(BGP_TOOLS_URL, headers={'User-Agent': USER_AGENT})
+    
+    with urllib.request.urlopen(request) as response:
+        for line in response:
+            decoded_line = line.decode('utf-8').strip()
+            subnet, as_number = decoded_line.split()
+            subnet_list.append((subnet, as_number))
+
+    # Meta
+    ipv4_merged_meta, ipv6_merged_meta = process_subnets(subnet_list, AS_META)
+    write_subnets_to_file(ipv4_merged_meta, f'{IPv4_DIR}/{META}')
+    write_subnets_to_file(ipv6_merged_meta, f'{IPv6_DIR}/{META}')
+
+    # Twitter
+    ipv4_merged_twitter, ipv6_merged_twitter = process_subnets(subnet_list, AS_TWITTER)
+    write_subnets_to_file(ipv4_merged_twitter, f'{IPv4_DIR}/{TWITTER}')
+    write_subnets_to_file(ipv6_merged_twitter, f'{IPv6_DIR}/{TWITTER}')