From 41e7d55598f7c22a8f84ef82a97d1be1378de607 Mon Sep 17 00:00:00 2001 From: Be Date: Sun, 3 Mar 2024 12:51:03 +0100 Subject: [PATCH] whois --- .github/workflows/update_ru_all_lists.yml | 124 +++++++++++++--------- get_info_from_ripe.py | 36 +------ 2 files changed, 76 insertions(+), 84 deletions(-) diff --git a/.github/workflows/update_ru_all_lists.yml b/.github/workflows/update_ru_all_lists.yml index dc94a6d..9c2fc74 100644 --- a/.github/workflows/update_ru_all_lists.yml +++ b/.github/workflows/update_ru_all_lists.yml @@ -11,8 +11,8 @@ on: - cron: '0 0 1 * *' jobs: - get_asn: - name: 'Get/ASN' + get_lists: + name: 'Get/Lists' runs-on: ubuntu-22.04 @@ -25,58 +25,78 @@ jobs: with: fetch-depth: 0 # this is required to fetch all history for all branches and tags token: ${{ env.GH_PAT }} - ref: ${{ github.branch }} - - uses: ./.github/actions/gitReset - uses: ./.github/actions/pyInstall - - run: ./get_info_from_ripe.py --asn --limit 4000 + - run: ./get_info_from_ripe.py - uses: ./.github/actions/gitPush env: PUSH_FILES: auto/*txt - get_ipv4: - name: 'Get/IPv4' - - runs-on: ubuntu-22.04 - - defaults: - run: - shell: bash - - needs: get_asn - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # this is required to fetch all history for all branches and tags - token: ${{ env.GH_PAT }} - ref: ${{ github.branch }} - - uses: ./.github/actions/gitReset - - uses: ./.github/actions/pyInstall - - run: ./get_info_from_ripe.py --ipv4 --limit 4000 - - uses: ./.github/actions/gitPush - env: - PUSH_FILES: auto/*txt - - get_ipv6: - name: 'Get/IPv6' - - runs-on: ubuntu-22.04 - - defaults: - run: - shell: bash - - needs: get_ipv4 - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # this is required to fetch all history for all branches and tags - token: ${{ env.GH_PAT }} - ref: ${{ github.branch }} - - uses: ./.github/actions/gitReset - - uses: ./.github/actions/pyInstall - - run: ./get_info_from_ripe.py --ipv6 --limit 4000 - - uses: ./.github/actions/gitPush - env: - PUSH_FILES: auto/*txt +# get_asn: +# name: 'Get/ASN' +# +# runs-on: ubuntu-22.04 +# +# defaults: +# run: +# shell: bash +# +# steps: +# - uses: actions/checkout@v4 +# with: +# fetch-depth: 0 # this is required to fetch all history for all branches and tags +# token: ${{ env.GH_PAT }} +# ref: ${{ github.branch }} +# - uses: ./.github/actions/gitReset +# - uses: ./.github/actions/pyInstall +# - run: ./get_info_from_ripe.py --asn --limit 4000 +# - uses: ./.github/actions/gitPush +# env: +# PUSH_FILES: auto/*txt +# +# get_ipv4: +# name: 'Get/IPv4' +# +# runs-on: ubuntu-22.04 +# +# defaults: +# run: +# shell: bash +# +# needs: get_asn +# +# steps: +# - uses: actions/checkout@v4 +# with: +# fetch-depth: 0 # this is required to fetch all history for all branches and tags +# token: ${{ env.GH_PAT }} +# ref: ${{ github.branch }} +# - uses: ./.github/actions/gitReset +# - uses: ./.github/actions/pyInstall +# - run: ./get_info_from_ripe.py --ipv4 --limit 4000 +# - uses: ./.github/actions/gitPush +# env: +# PUSH_FILES: auto/*txt +# +# get_ipv6: +# name: 'Get/IPv6' +# +# runs-on: ubuntu-22.04 +# +# defaults: +# run: +# shell: bash +# +# needs: get_ipv4 +# +# steps: +# - uses: actions/checkout@v4 +# with: +# fetch-depth: 0 # this is required to fetch all history for all branches and tags +# token: ${{ env.GH_PAT }} +# ref: ${{ github.branch }} +# - uses: ./.github/actions/gitReset +# - uses: ./.github/actions/pyInstall +# - run: ./get_info_from_ripe.py --ipv6 --limit 4000 +# - uses: ./.github/actions/gitPush +# env: +# PUSH_FILES: auto/*txt diff --git a/get_info_from_ripe.py b/get_info_from_ripe.py index 111ee57..002282c 100755 --- a/get_info_from_ripe.py +++ b/get_info_from_ripe.py @@ -1,51 +1,23 @@ #!/usr/bin/env python3 -import argparse import requests from pylib.whois import whois_query url = "https://stat.ripe.net/data/country-resource-list/data.json?resource=RU&v4_format=prefix" def get_data(json, file, attr, field, prefix=""): - limit = args.limit - count = 0 with open(file, 'w') as f: for x in json['data']['resources'][attr]: - count += 1 x = prefix+x.strip() - if count > limit: - response = None - else: - response = whois_query(x, field) - if response is None: - name = "-no-description-" - else: - name = response.split(':')[1].strip() + name = "-no-description-" print(f"{x} {name}") f.write(str(x+" "+name) + '\n') -parser = argparse.ArgumentParser() -parser.add_argument('--asn', action='store_true', help='Run the ASN query') -parser.add_argument('--ipv4', action='store_true', help='Run the IPv4 query') -parser.add_argument('--ipv6', action='store_true', help='Run the IPv6 query') -parser.add_argument('--all', action='store_true', help='Run all queries') -parser.add_argument('--limit', type=int, help='Limit the number of whois queries to prevent blacklisting from whois servers', default=2500) -args = parser.parse_args() - -if not (args.asn or args.ipv4 or args.ipv6 or args.all): - parser.print_help() - exit() - response = requests.get(url) response.raise_for_status() data = response.json() -if args.asn or args.all: - get_data(data, 'auto/all-ru-asn.txt', 'asn', 'as-name', "AS") - -if args.ipv4 or args.all: - get_data(data, 'auto/all-ru-ipv4.txt', 'ipv4', 'netname') - -if args.ipv6 or args.all: - get_data(data, 'auto/all-ru-ipv6.txt', 'ipv6', 'netname') +get_data(data, 'auto/all-ru-asn.txt', 'asn', 'as-name', "AS") +get_data(data, 'auto/all-ru-ipv4.txt', 'ipv4', 'netname') +get_data(data, 'auto/all-ru-ipv6.txt', 'ipv6', 'netname')