New VK Exclude rules

blacklists_updater_common.subr

@@ -0,0 +1,56 @@
+#!/bin/sh
+
+# Shared config for blacklists_updater_*.sh scripts.
+# Scripts are expected to define SCRIPT_DIR before sourcing this file.
+
+: "${SCRIPT_DIR:?SCRIPT_DIR must be set before sourcing blacklists_updater_common.subr}"
+
+BLACKLISTS_DIR="${SCRIPT_DIR}/blacklists"
+AUTO_DIR="${SCRIPT_DIR}/auto"
+
+BLACKLIST_FILE="${BLACKLISTS_DIR}/blacklist.txt"
+BLACKLIST_WITH_COMMENTS_FILE="${BLACKLISTS_DIR}/blacklist_with_comments.txt"
+BLACKLIST_V4_FILE="${BLACKLISTS_DIR}/blacklist-v4.txt"
+BLACKLIST_V6_FILE="${BLACKLISTS_DIR}/blacklist-v6.txt"
+BLACKLIST_VK_FILE="${BLACKLISTS_DIR}/blacklist-vk.txt"
+BLACKLIST_VK_V4_FILE="${BLACKLISTS_DIR}/blacklist-vk-v4.txt"
+BLACKLIST_VK_V6_FILE="${BLACKLISTS_DIR}/blacklist-vk-v6.txt"
+
+AUTO_ALL_ASN_FILE="${AUTO_DIR}/all-ru-asn.txt"
+AUTO_ALL_V4_FILE="${AUTO_DIR}/all-ru-ipv4.txt"
+AUTO_ALL_V6_FILE="${AUTO_DIR}/all-ru-ipv6.txt"
+AUTO_RIPE_V4_FILE="${AUTO_DIR}/ripe-ru-ipv4.txt"
+AUTO_BLACK_ASS_FILE="${AUTO_DIR}/black_ass.txt"
+
+NETWORK_LIST_FROM_AS="${SCRIPT_DIR}/network_list_from_as.py"
+NETWORK_LIST_FROM_NETNAME="${SCRIPT_DIR}/network_list_from_netname.py"
+RU_GOV_NETNAMES_FILE="${SCRIPT_DIR}/lists/ru-gov-netnames.txt"
+
+BLACK_NAMES='uvd|umvd|fgup|grchc|roskomnad|federalnaya sluzhba|ufsb|zonatelecom|llc vk|vkontakte|ODNOKLASSNIKI|VKCOMPANY|mail.ru|mail-ru|mail_ru|VK-AS|M100'
+WHITE_NAMES='ruvds'
+#VK_NAME_PATTERN='ru-netbridge-(19911202|20061117)|ru-odnoklassniki-(20100830|20120307|20120626)|odnoklassniki-front'
+#VK_NAME_PATTERN='ru-netbridge-(19911202|20061117)|odnoklassniki'
+VK_NAME_PATTERN='ru-netbridge-(19911202|20061117)|odnoklassniki|vkcompany|vkontakte|llc vk'
+VK_NAME_EXCLUDE_PATTERN='VK Cloud Solutions VK Hosting'
+
+ensure_blacklist_base_dirs() {
+    mkdir -p "${BLACKLISTS_DIR}" "${AUTO_DIR}"
+}
+
+build_vk_name_blacklists() {
+    ensure_blacklist_base_dirs
+
+    tmp_vk_file="$(mktemp "${BLACKLISTS_DIR}/.blacklist-vk.XXXXXX")" || return 1
+
+    for source_file in "${AUTO_ALL_V4_FILE}" "${AUTO_ALL_V6_FILE}" "${AUTO_RIPE_V4_FILE}"; do
+        [ -f "${source_file}" ] || continue
+        awk -v pattern="${VK_NAME_PATTERN}" 'tolower($0) ~ pattern { print }' "${source_file}" \
+            | grep -viF "${VK_NAME_EXCLUDE_PATTERN}" \
+            | awk '{ print $1 }' >> "${tmp_vk_file}"
+    done
+
+    sort -u "${tmp_vk_file}" > "${BLACKLIST_VK_FILE}"
+    grep ':' "${BLACKLIST_VK_FILE}" | sort -u > "${BLACKLIST_VK_V6_FILE}" || true
+    grep -v ':' "${BLACKLIST_VK_FILE}" | sort -u > "${BLACKLIST_VK_V4_FILE}" || true
+    rm -f "${tmp_vk_file}"
+}

blacklists_updater_iptables.sh

@@ -1,22 +1,7 @@
 #!/bin/sh
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-
+. "${SCRIPT_DIR}/blacklists_updater_common.subr"
-# Input files (generated by blacklists_updater_txt.sh)
-blacklist_file="${SCRIPT_DIR}/blacklists/blacklist.txt"
-blacklist_v4_file="${SCRIPT_DIR}/blacklists/blacklist-v4.txt"
-blacklist_v6_file="${SCRIPT_DIR}/blacklists/blacklist-v6.txt"
-
-# Source files for name-based VK filtering
-auto_all_v4_file="${SCRIPT_DIR}/auto/all-ru-ipv4.txt"
-auto_all_v6_file="${SCRIPT_DIR}/auto/all-ru-ipv6.txt"
-auto_ripe_v4_file="${SCRIPT_DIR}/auto/ripe-ru-ipv4.txt"
-vk_name_pattern='vk[[:space:]-]*cloud|vkcompany|vkontakte'
-
-# Additional VK-only text blacklists
-blacklist_vk_file="${SCRIPT_DIR}/blacklists/blacklist-vk.txt"
-blacklist_vk_v4_file="${SCRIPT_DIR}/blacklists/blacklist-vk-v4.txt"
-blacklist_vk_v6_file="${SCRIPT_DIR}/blacklists/blacklist-vk-v6.txt"
 
 # Output directory and files
 iptables_output_dir="${SCRIPT_DIR}/blacklists_iptables"
@@ -26,18 +11,8 @@ iptables_vk_v4_output_file="${iptables_output_dir}/blacklist-vk-v4.ipset"
 iptables_vk_v6_output_file="${iptables_output_dir}/blacklist-vk-v6.ipset"
 
 # Create required directories if they don't exist
-mkdir -p "${iptables_output_dir}" "${SCRIPT_DIR}/blacklists"
+mkdir -p "${iptables_output_dir}" "${BLACKLISTS_DIR}"
-
+build_vk_name_blacklists
-# Build additional VK-only blacklist from network names in auto/*.txt files
-tmp_vk_file="$(mktemp "${SCRIPT_DIR}/blacklists/.blacklist-vk.XXXXXX")"
-for source_file in "${auto_all_v4_file}" "${auto_all_v6_file}" "${auto_ripe_v4_file}"; do
-    [ -f "${source_file}" ] || continue
-    awk -v pattern="${vk_name_pattern}" 'tolower($0) ~ pattern { print $1 }' "${source_file}" >> "${tmp_vk_file}"
-done
-sort -u "${tmp_vk_file}" > "${blacklist_vk_file}"
-grep ':' "${blacklist_vk_file}" | sort -u > "${blacklist_vk_v6_file}" || true
-grep -v ':' "${blacklist_vk_file}" | sort -u > "${blacklist_vk_v4_file}" || true
-rm -f "${tmp_vk_file}"
 
 # Function to generate ipset config from input file
 generate_ipset_config() {
@@ -99,10 +74,10 @@ EOF
 }
 
 # Generate ipset configurations from blacklist files
-generate_ipset_config "${blacklist_v4_file}" "${iptables_v4_output_file}" "(IPv4 only)" "blacklist-v4" "inet"
+generate_ipset_config "${BLACKLIST_V4_FILE}" "${iptables_v4_output_file}" "(IPv4 only)" "blacklist-v4" "inet"
-generate_ipset_config "${blacklist_v6_file}" "${iptables_v6_output_file}" "(IPv6 only)" "blacklist-v6" "inet6"
+generate_ipset_config "${BLACKLIST_V6_FILE}" "${iptables_v6_output_file}" "(IPv6 only)" "blacklist-v6" "inet6"
-generate_ipset_config "${blacklist_vk_v4_file}" "${iptables_vk_v4_output_file}" "(VK names, IPv4 only)" "blacklist-vk-v4" "inet"
+generate_ipset_config "${BLACKLIST_VK_V4_FILE}" "${iptables_vk_v4_output_file}" "(VK names, IPv4 only)" "blacklist-vk-v4" "inet"
-generate_ipset_config "${blacklist_vk_v6_file}" "${iptables_vk_v6_output_file}" "(VK names, IPv6 only)" "blacklist-vk-v6" "inet6"
+generate_ipset_config "${BLACKLIST_VK_V6_FILE}" "${iptables_vk_v6_output_file}" "(VK names, IPv6 only)" "blacklist-vk-v6" "inet6"
 
 echo ""
 echo "VK outgoing block examples (iptables/ipset):"

blacklists_updater_nftables.sh

@@ -4,35 +4,16 @@
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-INPUT_FILE="$SCRIPT_DIR/blacklists/blacklist.txt"
+. "${SCRIPT_DIR}/blacklists_updater_common.subr"
+INPUT_FILE="${BLACKLIST_FILE}"
 OUTPUT_DIR="$SCRIPT_DIR/blacklists_nftables"
 
-# Source files for name-based VK filtering
-AUTO_ALL_V4_FILE="$SCRIPT_DIR/auto/all-ru-ipv4.txt"
-AUTO_ALL_V6_FILE="$SCRIPT_DIR/auto/all-ru-ipv6.txt"
-AUTO_RIPE_V4_FILE="$SCRIPT_DIR/auto/ripe-ru-ipv4.txt"
-VK_NAME_PATTERN='vk[[:space:]-]*cloud|vkcompany|vkontakte'
-
-# Additional VK-only text blacklists
-VK_INPUT_FILE="$SCRIPT_DIR/blacklists/blacklist-vk.txt"
-VK_INPUT_V4_FILE="$SCRIPT_DIR/blacklists/blacklist-vk-v4.txt"
-VK_INPUT_V6_FILE="$SCRIPT_DIR/blacklists/blacklist-vk-v6.txt"
-
 # Create required directories if they don't exist
-mkdir -p "$OUTPUT_DIR" "$SCRIPT_DIR/blacklists"
+mkdir -p "$OUTPUT_DIR" "${BLACKLISTS_DIR}"
 
 echo "Generating nftables blacklists..."
 
-# Build additional VK-only blacklist from network names in auto/*.txt files
+build_vk_name_blacklists
-TMP_VK_FILE="$(mktemp "$SCRIPT_DIR/blacklists/.blacklist-vk.XXXXXX")"
-for source_file in "$AUTO_ALL_V4_FILE" "$AUTO_ALL_V6_FILE" "$AUTO_RIPE_V4_FILE"; do
-    [[ -f "$source_file" ]] || continue
-    awk -v pattern="$VK_NAME_PATTERN" 'tolower($0) ~ pattern { print $1 }' "$source_file" >> "$TMP_VK_FILE"
-done
-sort -u "$TMP_VK_FILE" > "$VK_INPUT_FILE"
-grep ':' "$VK_INPUT_FILE" | sort -u > "$VK_INPUT_V6_FILE" || true
-grep -v ':' "$VK_INPUT_FILE" | sort -u > "$VK_INPUT_V4_FILE" || true
-rm -f "$TMP_VK_FILE"
 
 # Generate mixed IPv4/IPv6 blacklist (recommended single-file load)
 python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \
@@ -53,15 +34,15 @@ python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \
     "$TMP_V6_FILE" \
     "$OUTPUT_DIR/blacklist-v6.nft"
 
-# Generate VK-only blacklists (network names: VK Cloud / VKCOMPANY / VKONTAKTE)
+# Generate VK-only blacklists from the narrowed MAX/VK service name filter
 python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \
-    "$VK_INPUT_FILE" \
+    "${BLACKLIST_VK_FILE}" \
     "$OUTPUT_DIR/blacklist-vk.nft"
 python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \
-    "$VK_INPUT_V4_FILE" \
+    "${BLACKLIST_VK_V4_FILE}" \
     "$OUTPUT_DIR/blacklist-vk-v4.nft"
 python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \
-    "$VK_INPUT_V6_FILE" \
+    "${BLACKLIST_VK_V6_FILE}" \
     "$OUTPUT_DIR/blacklist-vk-v6.nft"
 
 # Clean up temp files

blacklists_updater_nginx.sh

@@ -1,18 +1,16 @@
 #!/bin/sh
 
-# Input files (generated by blacklists_updater_txt.sh)
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-blacklist_file="blacklists/blacklist.txt"
+. "${SCRIPT_DIR}/blacklists_updater_common.subr"
-blacklist_v4_file="blacklists/blacklist-v4.txt"
-blacklist_v6_file="blacklists/blacklist-v6.txt"
 
 # Output directory and files
-nginx_output_dir="blacklists_nginx"
+nginx_output_dir="${SCRIPT_DIR}/blacklists_nginx"
 nginx_output_file="${nginx_output_dir}/blacklist.conf"
 nginx_v4_output_file="${nginx_output_dir}/blacklist-v4.conf"
 nginx_v6_output_file="${nginx_output_dir}/blacklist-v6.conf"
 
 # Create required directories if they don't exist
-mkdir -p "${nginx_output_dir}" "blacklists"
+mkdir -p "${nginx_output_dir}" "${BLACKLISTS_DIR}"
 
 # Function to generate nginx config from input file
 generate_nginx_config() {
@@ -47,6 +45,6 @@ EOF
 }
 
 # Generate nginx configurations from blacklist files
-generate_nginx_config "${blacklist_file}" "${nginx_output_file}" "(mixed IPv4/IPv6)"
+generate_nginx_config "${BLACKLIST_FILE}" "${nginx_output_file}" "(mixed IPv4/IPv6)"
-generate_nginx_config "${blacklist_v4_file}" "${nginx_v4_output_file}" "(IPv4 only)"
+generate_nginx_config "${BLACKLIST_V4_FILE}" "${nginx_v4_output_file}" "(IPv4 only)"
-generate_nginx_config "${blacklist_v6_file}" "${nginx_v6_output_file}" "(IPv6 only)"
+generate_nginx_config "${BLACKLIST_V6_FILE}" "${nginx_v6_output_file}" "(IPv6 only)"

blacklists_updater_routes.sh

@@ -3,38 +3,18 @@
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-
+. "${SCRIPT_DIR}/blacklists_updater_common.subr"
-# Source files for name-based VK filtering
-AUTO_ALL_V4_FILE="${SCRIPT_DIR}/auto/all-ru-ipv4.txt"
-AUTO_ALL_V6_FILE="${SCRIPT_DIR}/auto/all-ru-ipv6.txt"
-AUTO_RIPE_V4_FILE="${SCRIPT_DIR}/auto/ripe-ru-ipv4.txt"
-VK_NAME_PATTERN='vk[[:space:]-]*cloud|vkcompany|vkontakte'
-
-# Additional VK-only text blacklists
-VK_INPUT_FILE="${SCRIPT_DIR}/blacklists/blacklist-vk.txt"
-VK_INPUT_V4_FILE="${SCRIPT_DIR}/blacklists/blacklist-vk-v4.txt"
-VK_INPUT_V6_FILE="${SCRIPT_DIR}/blacklists/blacklist-vk-v6.txt"
 
 # Output directory and files
 ROUTES_OUTPUT_DIR="${SCRIPT_DIR}/blacklists_route"
 ROUTES_V4_FILE="${ROUTES_OUTPUT_DIR}/blacklist-vk-v4.routes"
 ROUTES_V6_FILE="${ROUTES_OUTPUT_DIR}/blacklist-vk-v6.routes"
 
-mkdir -p "${ROUTES_OUTPUT_DIR}" "${SCRIPT_DIR}/blacklists"
+mkdir -p "${ROUTES_OUTPUT_DIR}" "${BLACKLISTS_DIR}"
 
 echo "Generating VK route blacklists..."
 
-# Build additional VK-only blacklist from network names in auto/*.txt files
+build_vk_name_blacklists
-TMP_VK_FILE="$(mktemp "${SCRIPT_DIR}/blacklists/.blacklist-vk.XXXXXX")"
-for source_file in "${AUTO_ALL_V4_FILE}" "${AUTO_ALL_V6_FILE}" "${AUTO_RIPE_V4_FILE}"; do
-	[ -f "${source_file}" ] || continue
-	awk -v pattern="${VK_NAME_PATTERN}" 'tolower($0) ~ pattern { print $1 }' "${source_file}" >> "${TMP_VK_FILE}"
-done
-
-sort -u "${TMP_VK_FILE}" > "${VK_INPUT_FILE}"
-grep ':' "${VK_INPUT_FILE}" | sort -u > "${VK_INPUT_V6_FILE}" || true
-grep -v ':' "${VK_INPUT_FILE}" | sort -u > "${VK_INPUT_V4_FILE}" || true
-rm -f "${TMP_VK_FILE}"
 
 # Generate IPv4 routes file (route VK prefixes to loopback via 127.0.0.1)
 cat > "${ROUTES_V4_FILE}" << EOF
@@ -51,7 +31,7 @@ EOF
 while IFS= read -r network; do
 	[ -n "${network}" ] || continue
 	printf 'ip route replace %s via 127.0.0.1 dev lo onlink\n' "${network}" >> "${ROUTES_V4_FILE}"
-done < "${VK_INPUT_V4_FILE}"
+done < "${BLACKLIST_VK_V4_FILE}"
 
 # Generate IPv6 routes file (route VK prefixes to loopback via ::1)
 cat > "${ROUTES_V6_FILE}" << EOF
@@ -68,10 +48,10 @@ EOF
 while IFS= read -r network; do
 	[ -n "${network}" ] || continue
 	printf 'ip -6 route replace %s via ::1 dev lo\n' "${network}" >> "${ROUTES_V6_FILE}"
-done < "${VK_INPUT_V6_FILE}"
+done < "${BLACKLIST_VK_V6_FILE}"
 
-echo "✓ Generated: ${ROUTES_V4_FILE} (entries: $(wc -l < "${VK_INPUT_V4_FILE}" | tr -d ' '))"
+echo "✓ Generated: ${ROUTES_V4_FILE} (entries: $(wc -l < "${BLACKLIST_VK_V4_FILE}" | tr -d ' '))"
-echo "✓ Generated: ${ROUTES_V6_FILE} (entries: $(wc -l < "${VK_INPUT_V6_FILE}" | tr -d ' '))"
+echo "✓ Generated: ${ROUTES_V6_FILE} (entries: $(wc -l < "${BLACKLIST_VK_V6_FILE}" | tr -d ' '))"
 echo ""
 echo "Examples:"
 echo "  sudo sh ${ROUTES_V4_FILE}"

blacklists_updater_txt.sh

@@ -1,32 +1,24 @@
 #!/bin/sh
 
-outfile_wo_comments="blacklists/blacklist.txt"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-outfile_w_comments="blacklists/blacklist_with_comments.txt"
+. "${SCRIPT_DIR}/blacklists_updater_common.subr"
-blacklist_v4_file="blacklists/blacklist-v4.txt"
-blacklist_v6_file="blacklists/blacklist-v6.txt"
 
-auto_black_ass="auto/black_ass.txt"
+ensure_blacklist_base_dirs
 
-black_names="uvd|umvd|fgup|grchc|roskomnad|federalnaya sluzhba|ufsb|zonatelecom|llc vk|vkontakte|ODNOKLASSNIKI|VKCOMPANY|mail.ru|mail-ru|mail_ru|VK-AS|M100"
+grep -iE "${BLACK_NAMES}" "${AUTO_ALL_ASN_FILE}" | grep -viE "${WHITE_NAMES}" | awk '{ print "# AS-Name: " $0 "\n" $1}' > "${AUTO_BLACK_ASS_FILE}"
-# M100 - mail.ru
+"${NETWORK_LIST_FROM_AS}" "${AUTO_BLACK_ASS_FILE}" > "${BLACKLIST_WITH_COMMENTS_FILE}"
-white_names="ruvds"
+"${NETWORK_LIST_FROM_NETNAME}" "${RU_GOV_NETNAMES_FILE}" >> "${BLACKLIST_WITH_COMMENTS_FILE}"
-
+grep -iE "${BLACK_NAMES}" "${AUTO_ALL_V4_FILE}"  | grep -viE "${WHITE_NAMES}" | awk '{ print "# NET-Name: " $0 "\n" $1}' >> "${BLACKLIST_WITH_COMMENTS_FILE}"
-mkdir -p blacklists auto
+grep -iE "${BLACK_NAMES}" "${AUTO_RIPE_V4_FILE}" | grep -viE "${WHITE_NAMES}" | awk '{ print "# NET-Name: " $0 "\n" $1}' >> "${BLACKLIST_WITH_COMMENTS_FILE}"
-
-grep -iE "${black_names}" auto/all-ru-asn.txt   | grep -viE "${white_names}" | awk '{ print "# AS-Name: " $0 "\n" $1}' > ${auto_black_ass}
-./network_list_from_as.py ${auto_black_ass} > ${outfile_w_comments}
-./network_list_from_netname.py lists/ru-gov-netnames.txt >> ${outfile_w_comments}
-grep -iE "${black_names}" auto/all-ru-ipv4.txt  | grep -viE "${white_names}" | awk '{ print "# NET-Name: " $0 "\n" $1}' >> ${outfile_w_comments}
-grep -iE "${black_names}" auto/ripe-ru-ipv4.txt | grep -viE "${white_names}" | awk '{ print "# NET-Name: " $0 "\n" $1}' >> ${outfile_w_comments}
 
 # Remove comments by sed to avoid flooding WHOIS servers
-grep -v "#" ${outfile_w_comments} | sort | uniq > ${outfile_wo_comments}
+grep -v "#" "${BLACKLIST_WITH_COMMENTS_FILE}" | sort | uniq > "${BLACKLIST_FILE}"
 
 # Split blacklist into IPv4 and IPv6
-grep ':' "${outfile_wo_comments}" | sort | uniq > "${blacklist_v6_file}"
+grep ':' "${BLACKLIST_FILE}" | sort | uniq > "${BLACKLIST_V6_FILE}"
-grep -v ':' "${outfile_wo_comments}" | sort | uniq > "${blacklist_v4_file}"
+grep -v ':' "${BLACKLIST_FILE}" | sort | uniq > "${BLACKLIST_V4_FILE}"
 
 echo "✓ Generated blacklist files"
-echo "  Mixed (IPv4/IPv6): ${outfile_wo_comments} ($(wc -l < "${outfile_wo_comments}" | tr -d ' ') entries)"
+echo "  Mixed (IPv4/IPv6): ${BLACKLIST_FILE} ($(wc -l < "${BLACKLIST_FILE}" | tr -d ' ') entries)"
-echo "  IPv4 only: ${blacklist_v4_file} ($(wc -l < "${blacklist_v4_file}" | tr -d ' ') entries)"
+echo "  IPv4 only: ${BLACKLIST_V4_FILE} ($(wc -l < "${BLACKLIST_V4_FILE}" | tr -d ' ') entries)"
-echo "  IPv6 only: ${blacklist_v6_file} ($(wc -l < "${blacklist_v6_file}" | tr -d ' ') entries)"
+echo "  IPv6 only: ${BLACKLIST_V6_FILE} ($(wc -l < "${BLACKLIST_V6_FILE}" | tr -d ' ') entries)"