Add whois limit, actions, etc

.github/actions/checkout/action.yaml

@@ -0,0 +1,7 @@
+runs:
+    using: "composite"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0                         # this is required to fetch all history for all branches and tags
+          token: ${{ env.GH_PAT }}

.github/actions/pyInstall/action.yaml

@@ -0,0 +1,8 @@
+runs:
+    using: "composite"
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: ./requirements.sh
+        shell: /bin/bash

.github/workflows/update_blacklists.yml

@@ -21,15 +21,9 @@ jobs:
         shell: bash
 
     steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0                         # this is required to fetch all history for all branches and tags
-          token: ${{ env.GH_PAT }}
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-      - run: ./requirements.sh
+      - uses: ./.github/actions/checkout
+      - uses: ./.github/actions/pyInstall
       - run: ./blacklists_updater.sh
       - uses: ./.github/actions/gitPush
         env:
-          PUSH_FILES: auto/blacklist*txt
+          PUSH_FILES: auto/*txt

.github/workflows/update_ru_all_lists.yml

@@ -14,7 +14,7 @@ on:
       - main
 
 jobs:
-  get_as:
+  get_asn:
     name: 'Get/ASN'
 
     runs-on: ubuntu-22.04
@@ -24,7 +24,12 @@ jobs:
         shell: bash
 
     steps:
-      - run: curl -s ifconfig.io
+      - uses: ./.github/actions/checkout
+      - uses: ./.github/actions/pyInstall
+      - run: ./get_info_from_ripe.py --asn --limit 4000
+      - uses: ./.github/actions/gitPush
+        env:
+          PUSH_FILES: auto/*txt
 
   get_ipv4:
     name: 'Get/IPv4'
@@ -35,10 +40,15 @@ jobs:
       run:
         shell: bash
 
-    needs: get_as
+    needs: get_asn
 
     steps:
-      - run: curl -s ifconfig.io
+      - uses: ./.github/actions/checkout
+      - uses: ./.github/actions/pyInstall
+      - run: ./get_info_from_ripe.py --ipv4 --limit 4000
+      - uses: ./.github/actions/gitPush
+        env:
+          PUSH_FILES: auto/*txt
 
   get_ipv6:
     name: 'Get/IPv6'
@@ -52,4 +62,9 @@ jobs:
     needs: get_ipv4
 
     steps:
-      - run: curl -s ifconfig.io
+      - uses: ./.github/actions/checkout
+      - uses: ./.github/actions/pyInstall
+      - run: ./get_info_from_ripe.py --ipv6 --limit 4000
+      - uses: ./.github/actions/gitPush
+        env:
+          PUSH_FILES: auto/*txt

get_info_from_ripe.py

@@ -32,12 +32,18 @@ def whois_query(whois_server, query, get_field="netname"):
     return None
 
 def get_data(json, file, attr, field, prefix=""):
+    limit = args.limit
+    count = 0
     with open(file, 'w') as f:
         for x in json['data']['resources'][attr]:
+            count += 1
             x = prefix+x.strip()
-            response = whois_query(whois_server, x, field)
+            if count > limit:
+                response = None
+            else:
+                response = whois_query(whois_server, x, field)
             if response is None:
-                name = "None"
+                name = "-no-description-"
             else:
                 name = response.split(':')[1].strip()
             print(f"{x} {name}")
@@ -48,6 +54,7 @@ parser.add_argument('--asn', action='store_true', help='Run the ASN query')
 parser.add_argument('--ipv4', action='store_true', help='Run the IPv4 query')
 parser.add_argument('--ipv6', action='store_true', help='Run the IPv6 query')
 parser.add_argument('--all', action='store_true', help='Run all queries')
+parser.add_argument('--limit', type=int, help='Limit the number of whois queries to prevent blacklisting from whois servers', default=2500)
 args = parser.parse_args()
 
 if not (args.asn or args.ipv4 or args.ipv6 or args.all):