diff --git a/.github/actions/checkout/action.yaml b/.github/actions/checkout/action.yaml new file mode 100644 index 0000000..03d5058 --- /dev/null +++ b/.github/actions/checkout/action.yaml @@ -0,0 +1,7 @@ +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # this is required to fetch all history for all branches and tags + token: ${{ env.GH_PAT }} diff --git a/.github/actions/pyInstall/action.yaml b/.github/actions/pyInstall/action.yaml new file mode 100644 index 0000000..c4dbff0 --- /dev/null +++ b/.github/actions/pyInstall/action.yaml @@ -0,0 +1,8 @@ +runs: + using: "composite" + steps: + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: ./requirements.sh + shell: /bin/bash diff --git a/.github/workflows/update_blacklists.yml b/.github/workflows/update_blacklists.yml index 4fec395..6c15190 100644 --- a/.github/workflows/update_blacklists.yml +++ b/.github/workflows/update_blacklists.yml @@ -21,15 +21,9 @@ jobs: shell: bash steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # this is required to fetch all history for all branches and tags - token: ${{ env.GH_PAT }} - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - run: ./requirements.sh + - uses: ./.github/actions/checkout + - uses: ./.github/actions/pyInstall - run: ./blacklists_updater.sh - uses: ./.github/actions/gitPush env: - PUSH_FILES: auto/blacklist*txt + PUSH_FILES: auto/*txt diff --git a/.github/workflows/update_ru_all_lists.yml b/.github/workflows/update_ru_all_lists.yml index 8fe826f..839b5b2 100644 --- a/.github/workflows/update_ru_all_lists.yml +++ b/.github/workflows/update_ru_all_lists.yml @@ -14,7 +14,7 @@ on: - main jobs: - get_as: + get_asn: name: 'Get/ASN' runs-on: ubuntu-22.04 @@ -24,7 +24,12 @@ jobs: shell: bash steps: - - run: curl -s ifconfig.io + - uses: ./.github/actions/checkout + - uses: ./.github/actions/pyInstall + - run: ./get_info_from_ripe.py --asn --limit 4000 + - uses: ./.github/actions/gitPush + env: + PUSH_FILES: auto/*txt get_ipv4: name: 'Get/IPv4' @@ -35,10 +40,15 @@ jobs: run: shell: bash - needs: get_as + needs: get_asn steps: - - run: curl -s ifconfig.io + - uses: ./.github/actions/checkout + - uses: ./.github/actions/pyInstall + - run: ./get_info_from_ripe.py --ipv4 --limit 4000 + - uses: ./.github/actions/gitPush + env: + PUSH_FILES: auto/*txt get_ipv6: name: 'Get/IPv6' @@ -52,4 +62,9 @@ jobs: needs: get_ipv4 steps: - - run: curl -s ifconfig.io + - uses: ./.github/actions/checkout + - uses: ./.github/actions/pyInstall + - run: ./get_info_from_ripe.py --ipv6 --limit 4000 + - uses: ./.github/actions/gitPush + env: + PUSH_FILES: auto/*txt diff --git a/get_info_from_ripe.py b/get_info_from_ripe.py index 7d5c2b2..a360616 100755 --- a/get_info_from_ripe.py +++ b/get_info_from_ripe.py @@ -32,12 +32,18 @@ def whois_query(whois_server, query, get_field="netname"): return None def get_data(json, file, attr, field, prefix=""): + limit = args.limit + count = 0 with open(file, 'w') as f: for x in json['data']['resources'][attr]: + count += 1 x = prefix+x.strip() - response = whois_query(whois_server, x, field) + if count > limit: + response = None + else: + response = whois_query(whois_server, x, field) if response is None: - name = "None" + name = "-no-description-" else: name = response.split(':')[1].strip() print(f"{x} {name}") @@ -48,6 +54,7 @@ parser.add_argument('--asn', action='store_true', help='Run the ASN query') parser.add_argument('--ipv4', action='store_true', help='Run the IPv4 query') parser.add_argument('--ipv6', action='store_true', help='Run the IPv6 query') parser.add_argument('--all', action='store_true', help='Run all queries') +parser.add_argument('--limit', type=int, help='Limit the number of whois queries to prevent blacklisting from whois servers', default=2500) args = parser.parse_args() if not (args.asn or args.ipv4 or args.ipv6 or args.all):