Update scripts and lists

README.md

@@ -2,10 +2,15 @@
 
 This repository contains two Python scripts that allow you to retrieve network lists based on either an Autonomous System (AS) name or a Network name.
 
-## Features
+## Files and features
 
 - `network_list_from_as.py`: Retrieves a list of networks associated with a given AS name.
 - `network_list_from_netname.py`: Retrieves a list of networks associated with a given Network name.
+- `lists/ru-gov-netnames.txt`: A list of network names associated with the Russian government. **Contributors are welcome!**
+- `lists/ru-gov-ass.txt`: A list of AS numbers associated with the Russian government. **Contributors are welcome!**
+- `blacklist_updater.sh`: Static blacklist updater.
+- `blacklist.txt`: Static blacklist generated from both lists! **Will be periodically updated!**
+- `blacklist_comments.txt`: Static blacklist generated from both lists with comments! **Will be periodically updated!**
 
 <img width="240" alt="image" src="https://github.com/C24Be/AS_Network_List/assets/153936414/2ec89fa9-b39a-416d-b1a1-20ddc89377ed">
 
@@ -45,10 +50,16 @@ These scripts have been tested on MacOS, FreeBSD and Linux.
     ./network_list_from_as.py AS61280
     ```
 
-2. To suppress all output except the prefixes, use the `--quiet` or `-q` switch:
+2. Run the script with a URL to a file in a GitHub repository as an argument:
 
     ```bash
-    ./network_list_from_as.py AS61280 -q
+    ./network_list_from_as.py https://github.com/C24Be/AS_Network_List/blob/main/lists/ru-gov-ass.txt
+    ```
+
+    Or better use the raw file link:
+
+    ```bash
+    ./network_list_from_as.py https://raw.githubusercontent.com/C24Be/AS_Network_List/main/lists/ru-gov-ass.txt
     ```
 
 3. To display a help message, use the `-h` or `--help` switch:
@@ -62,19 +73,19 @@ These scripts have been tested on MacOS, FreeBSD and Linux.
 1. Run the script with a file containing a list of network names as an argument:
 
     ```bash
-    ./network_list_from_netname.py lists/run-gov-netnames.txt
+    ./network_list_from_netname.py lists/ru-gov-netnames.txt
     ```
 
 2. Run the script with a URL to a file in a GitHub repository as an argument:
 
     ```bash
-    ./network_list_from_netname.py https://github.com/AntiZapret/antizapret/blob/master/blacklist4.txt
+    ./network_list_from_netname.py https://github.com/C24Be/AS_Network_List/blob/main/lists/ru-gov-netnames.txt
     ```
 
     Or better use the raw file link:
 
     ```bash
-    ./network_list_from_netname.py https://raw.githubusercontent.com/AntiZapret/antizapret/master/blacklist4.txt
+    ./network_list_from_netname.py https://raw.githubusercontent.com/C24Be/AS_Network_List/main/lists/ru-gov-netnames.txt
     ```
 
 3. To display a help message, use the `-h` or `--help` switch:

blacklists_updater.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+outfile_wo_comments="blacklist.txt"
+outfile_w_comments="blacklist_comments.txt"
+
+url_ass="https://raw.githubusercontent.com/C24Be/AS_Network_List/main/lists/ru-gov-ass.txt"
+url_nets="https://raw.githubusercontent.com/C24Be/AS_Network_List/main/lists/ru-gov-netnames.txt"
+
+./network_list_from_as.py ${url_ass} -q > ${outfile_wo_comments}
+./network_list_from_netname.py ${url_nets} -q >> ${outfile_wo_comments}
+
+./network_list_from_as.py ${url_ass} > ${outfile_w_comments}
+./network_list_from_netname.py ${url_nets} >> ${outfile_w_comments}

lists/ru-gov-ass.txt

@@ -1,10 +1,11 @@
+# ----------
 #
 # Just list of strings started with AS...
 #
-#  AS123
-#  AS1234
+#  AS31430
+#  AS61280
 #
+# ----------
+
 # Номер автономки у ГРЧЦ и подконтрольному ему ДЦОА
 AS61280
-# Test
-AS31430

network_list_from_as.py

@@ -2,6 +2,7 @@
 
 import requests
 import argparse
+import re
 from cymruwhois import Client
 
 def get_as_prefixes(asn):
@@ -14,27 +15,41 @@ def get_as_prefixes(asn):
     else:
         return []
 
-def get_as_whois(asn):
-    c = Client()
-    r = c.lookup(asn)
-    return r.owner
-
-def main():
-    parser = argparse.ArgumentParser(description='./as_network_list.py -q AS61280')
-    parser.add_argument('asn', help='The AS number to get prefixes for.')
-    parser.add_argument('-q', '--quiet', action='store_true', help='Disable all output except prefixes.')
-    args = parser.parse_args()
-
-    asn = args.asn
-    prefixes = get_as_prefixes(asn)
-    whois_info = get_as_whois(asn)
+def convert_to_raw_github_url(url):
+    return url.replace("https://github.com/", "https://raw.githubusercontent.com/").replace("/blob", "")
 
+def print_prefixes(asn):
+    line = re.sub(r'[^AS0-9]', '', asn)
     if not args.quiet:
-        print(f"Prefixes announced by {asn}:")
-        print(f"Whois info for {asn}: {whois_info}")
-
+        print(f"# Networks announced by {line}")
+    prefixes = get_as_prefixes(line)
     for prefix in prefixes:
         print(prefix)
 
-if __name__ == "__main__":
-    main()
+def extract_asses(asn_filename_or_url):
+    if asn_filename_or_url.startswith('AS'):
+        print_prefixes(asn_filename_or_url)
+
+        return None
+
+    if asn_filename_or_url.startswith('http://') or asn_filename_or_url.startswith('https://'):
+        if 'github.com' in asn_filename_or_url:
+            asn_filename_or_url = convert_to_raw_github_url(asn_filename_or_url)
+        response = requests.get(asn_filename_or_url)
+        lines = response.text.split('\n')
+    else:
+        with open(asn_filename_or_url, 'r') as file:
+            lines = file.readlines()
+
+    for line in lines:
+        if re.match(r'^AS.*', line):
+            print_prefixes(line)
+
+    return None
+
+parser = argparse.ArgumentParser(description='./as_network_list.py -q AS61280')
+parser.add_argument('asn_filename_or_url', help='The AS number to get networks / The file or URL to extract AS numbers from.')
+parser.add_argument('-q', '--quiet', action='store_true', help='Disable all output except prefixes.')
+args = parser.parse_args()
+
+extract_asses(args.asn_filename_or_url)

network_list_from_netname.py

@@ -57,8 +57,11 @@ def extract_netname(filename_or_url):
 
     for line in lines:
         if re.match(r'.*netname:', line):
-            response = whois_query(whois_server, line.split(':')[1].strip())
+            netname = line.split(':')[1].strip()
+            response = whois_query(whois_server, netname)
             if response is not None:
+                if not args.quiet:
+                    print(f"# Network name: {netname}")
                 ip_range = response.split(':')[1].strip()
                 cidrs = convert_to_cidr(ip_range)
                 for cidr in cidrs:
@@ -66,12 +69,9 @@ def extract_netname(filename_or_url):
 
     return None
 
-def main():
 parser = argparse.ArgumentParser(description='Extract netname from file.')
-    parser.add_argument('filename_or_url', help='The file to extract netname from.')
+parser.add_argument('filename_or_url', help='The file or URL to extract netnames from.')
+parser.add_argument('-q', '--quiet', action='store_true', help='Disable all output except prefixes.')
 args = parser.parse_args()
 
 extract_netname(args.filename_or_url)
-
-if __name__ == "__main__":
-    main()