diff --git a/.github/workflows/parse_ripe_database.yml b/.github/workflows/parse_ripe_database.yml new file mode 100644 index 0000000..38dabb3 --- /dev/null +++ b/.github/workflows/parse_ripe_database.yml @@ -0,0 +1,37 @@ +name: All RU Networks and ASN updater + +env: + GH_PAT: ${{ secrets.GH_PAT }} + REPO_NAME: AS_Network_List + REPO_OWNER: C24Be + RIPE_DB_URL: https://ftp.ripe.net/ripe/dbase/split/ + RIPE_DB_FILE: ripe.db.inetnum + +on: + workflow_dispatch: + schedule: + - cron: '0 12 * * 0' + +jobs: + get_lists: + name: 'Get/Lists' + + runs-on: ubuntu-22.04 + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # this is required to fetch all history for all branches and tags + token: ${{ env.GH_PAT }} + - uses: ./.github/actions/pyInstall + - run: | + wget ${{ env.RIPE_DB_URL }}${{ env.RIPE_DB_FILE }}.gz + gzip -d ${{ env.RIPE_DB_FILE }}.gz + - run: ./parse_ripe_db.py ripe.db.inetnum auto/ripe-ru-ipv4.txt auto/ripe-ru-ipv4.json + - uses: ./.github/actions/gitPush + env: + PUSH_FILES: auto/*txt diff --git a/network_list_from_netname.py b/network_list_from_netname.py index ff11a09..ba3e953 100755 --- a/network_list_from_netname.py +++ b/network_list_from_netname.py @@ -2,20 +2,13 @@ import argparse import requests -import ipaddress import re -from pylib.whois import whois_query +from pylib.whois import whois_query +from pylib.ip import convert_to_cidr def convert_to_raw_github_url(url): return url.replace("https://github.com/", "https://raw.githubusercontent.com/").replace("/blob", "") -def convert_to_cidr(ip_range): - start_ip, end_ip = ip_range.split(' - ') - start_ip = ipaddress.IPv4Address(start_ip) - end_ip = ipaddress.IPv4Address(end_ip) - cidrs = ipaddress.summarize_address_range(start_ip, end_ip) - return [str(cidr) for cidr in cidrs] - def extract_netname(filename_or_url): if filename_or_url.startswith('http://') or filename_or_url.startswith('https://'): if 'github.com' in filename_or_url: diff --git a/parse_ripe_db.py b/parse_ripe_db.py new file mode 100755 index 0000000..1b30f99 --- /dev/null +++ b/parse_ripe_db.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import argparse +import re +import json +from pylib.ip import convert_to_cidr + +country = "RU" + +def parse(filename, output_text, output_json): + cList = [] + record = {} + with open(filename, 'r', encoding='latin-1') as f: + lines = f.readlines() + f.close() + for line in lines: + if re.match(r'^inetnum:', line): + if record: + record['inetnum'] = convert_to_cidr(record['inetnum']) + if record['country'] == country: + print(record) + cList.append(record) + record = {} + record['inetnum'] = line.split('inetnum:', 1)[1].strip() + record['descr'] = '' + record['netname'] = '' + record['country'] = '' + record['org'] = '' + if re.match(r'^netname:', line): + record['netname'] = line.split('netname:', 1)[1].strip() + if re.match(r'^descr:', line): + record['descr'] = str(record['descr'].strip() + ' ' + line.split('descr:', 1)[1].strip()).strip() + if re.match(r'^mnt-by:', line): + record['netname'] = str(record['netname'].strip() + ' ' + line.split('mnt-by:', 1)[1].strip()).strip() + if re.match(r'^country:', line): + record['country'] = line.split('country:', 1)[1].strip() + if re.match(r'^org:', line): + record['org'] = line.split('org:', 1)[1].strip() + if record: + cList.append(record) + + with open(output_json, 'w') as f: + json.dump(cList, f, indent=4) + f.close() + + with open(output_text, 'w') as f: + for record in cList: + for net in record['inetnum']: + f.write(net + ' ' + record['netname'] + ' (' + record['org'] + ') [' + record['descr'] + ']\n') + f.close() + +parser = argparse.ArgumentParser(description='Parse RIPE DB for getting a list of RU networks.') +parser.add_argument('filename', help='ripe.db.inetnum file to parse.') +parser.add_argument('output_text', help='write text db to...') +parser.add_argument('output_json', help='write json do to...') +args = parser.parse_args() + +if not (args.filename): + parser.print_help() + exit() + +parse(args.filename, args.output_text, args.output_json) diff --git a/pylib/ip.py b/pylib/ip.py new file mode 100755 index 0000000..5c47a3b --- /dev/null +++ b/pylib/ip.py @@ -0,0 +1,8 @@ +import ipaddress + +def convert_to_cidr(ip_range): + start_ip, end_ip = ip_range.split(' - ') + start_ip = ipaddress.IPv4Address(start_ip) + end_ip = ipaddress.IPv4Address(end_ip) + cidrs = ipaddress.summarize_address_range(start_ip, end_ip) + return [str(cidr) for cidr in cidrs]