From 2be4631e02f0512a04d5f85444d9df8514321a38 Mon Sep 17 00:00:00 2001 From: C24Be Date: Mon, 6 Apr 2026 17:55:22 +0200 Subject: [PATCH] New VK Exclude rules --- blacklists_updater_common.subr | 56 ++++++++++++++++++++++++++++++++++ blacklists_updater_iptables.sh | 39 +++++------------------ blacklists_updater_nftables.sh | 35 +++++---------------- blacklists_updater_nginx.sh | 16 +++++----- blacklists_updater_routes.sh | 34 +++++---------------- blacklists_updater_txt.sh | 36 +++++++++------------- 6 files changed, 99 insertions(+), 117 deletions(-) create mode 100644 blacklists_updater_common.subr diff --git a/blacklists_updater_common.subr b/blacklists_updater_common.subr new file mode 100644 index 0000000..8d90676 --- /dev/null +++ b/blacklists_updater_common.subr @@ -0,0 +1,56 @@ +#!/bin/sh + +# Shared config for blacklists_updater_*.sh scripts. +# Scripts are expected to define SCRIPT_DIR before sourcing this file. + +: "${SCRIPT_DIR:?SCRIPT_DIR must be set before sourcing blacklists_updater_common.subr}" + +BLACKLISTS_DIR="${SCRIPT_DIR}/blacklists" +AUTO_DIR="${SCRIPT_DIR}/auto" + +BLACKLIST_FILE="${BLACKLISTS_DIR}/blacklist.txt" +BLACKLIST_WITH_COMMENTS_FILE="${BLACKLISTS_DIR}/blacklist_with_comments.txt" +BLACKLIST_V4_FILE="${BLACKLISTS_DIR}/blacklist-v4.txt" +BLACKLIST_V6_FILE="${BLACKLISTS_DIR}/blacklist-v6.txt" +BLACKLIST_VK_FILE="${BLACKLISTS_DIR}/blacklist-vk.txt" +BLACKLIST_VK_V4_FILE="${BLACKLISTS_DIR}/blacklist-vk-v4.txt" +BLACKLIST_VK_V6_FILE="${BLACKLISTS_DIR}/blacklist-vk-v6.txt" + +AUTO_ALL_ASN_FILE="${AUTO_DIR}/all-ru-asn.txt" +AUTO_ALL_V4_FILE="${AUTO_DIR}/all-ru-ipv4.txt" +AUTO_ALL_V6_FILE="${AUTO_DIR}/all-ru-ipv6.txt" +AUTO_RIPE_V4_FILE="${AUTO_DIR}/ripe-ru-ipv4.txt" +AUTO_BLACK_ASS_FILE="${AUTO_DIR}/black_ass.txt" + +NETWORK_LIST_FROM_AS="${SCRIPT_DIR}/network_list_from_as.py" +NETWORK_LIST_FROM_NETNAME="${SCRIPT_DIR}/network_list_from_netname.py" +RU_GOV_NETNAMES_FILE="${SCRIPT_DIR}/lists/ru-gov-netnames.txt" + +BLACK_NAMES='uvd|umvd|fgup|grchc|roskomnad|federalnaya sluzhba|ufsb|zonatelecom|llc vk|vkontakte|ODNOKLASSNIKI|VKCOMPANY|mail.ru|mail-ru|mail_ru|VK-AS|M100' +WHITE_NAMES='ruvds' +#VK_NAME_PATTERN='ru-netbridge-(19911202|20061117)|ru-odnoklassniki-(20100830|20120307|20120626)|odnoklassniki-front' +#VK_NAME_PATTERN='ru-netbridge-(19911202|20061117)|odnoklassniki' +VK_NAME_PATTERN='ru-netbridge-(19911202|20061117)|odnoklassniki|vkcompany|vkontakte|llc vk' +VK_NAME_EXCLUDE_PATTERN='VK Cloud Solutions VK Hosting' + +ensure_blacklist_base_dirs() { + mkdir -p "${BLACKLISTS_DIR}" "${AUTO_DIR}" +} + +build_vk_name_blacklists() { + ensure_blacklist_base_dirs + + tmp_vk_file="$(mktemp "${BLACKLISTS_DIR}/.blacklist-vk.XXXXXX")" || return 1 + + for source_file in "${AUTO_ALL_V4_FILE}" "${AUTO_ALL_V6_FILE}" "${AUTO_RIPE_V4_FILE}"; do + [ -f "${source_file}" ] || continue + awk -v pattern="${VK_NAME_PATTERN}" 'tolower($0) ~ pattern { print }' "${source_file}" \ + | grep -viF "${VK_NAME_EXCLUDE_PATTERN}" \ + | awk '{ print $1 }' >> "${tmp_vk_file}" + done + + sort -u "${tmp_vk_file}" > "${BLACKLIST_VK_FILE}" + grep ':' "${BLACKLIST_VK_FILE}" | sort -u > "${BLACKLIST_VK_V6_FILE}" || true + grep -v ':' "${BLACKLIST_VK_FILE}" | sort -u > "${BLACKLIST_VK_V4_FILE}" || true + rm -f "${tmp_vk_file}" +} diff --git a/blacklists_updater_iptables.sh b/blacklists_updater_iptables.sh index c0cc346..bdcf049 100755 --- a/blacklists_updater_iptables.sh +++ b/blacklists_updater_iptables.sh @@ -1,22 +1,7 @@ #!/bin/sh SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -# Input files (generated by blacklists_updater_txt.sh) -blacklist_file="${SCRIPT_DIR}/blacklists/blacklist.txt" -blacklist_v4_file="${SCRIPT_DIR}/blacklists/blacklist-v4.txt" -blacklist_v6_file="${SCRIPT_DIR}/blacklists/blacklist-v6.txt" - -# Source files for name-based VK filtering -auto_all_v4_file="${SCRIPT_DIR}/auto/all-ru-ipv4.txt" -auto_all_v6_file="${SCRIPT_DIR}/auto/all-ru-ipv6.txt" -auto_ripe_v4_file="${SCRIPT_DIR}/auto/ripe-ru-ipv4.txt" -vk_name_pattern='vk[[:space:]-]*cloud|vkcompany|vkontakte' - -# Additional VK-only text blacklists -blacklist_vk_file="${SCRIPT_DIR}/blacklists/blacklist-vk.txt" -blacklist_vk_v4_file="${SCRIPT_DIR}/blacklists/blacklist-vk-v4.txt" -blacklist_vk_v6_file="${SCRIPT_DIR}/blacklists/blacklist-vk-v6.txt" +. "${SCRIPT_DIR}/blacklists_updater_common.subr" # Output directory and files iptables_output_dir="${SCRIPT_DIR}/blacklists_iptables" @@ -26,18 +11,8 @@ iptables_vk_v4_output_file="${iptables_output_dir}/blacklist-vk-v4.ipset" iptables_vk_v6_output_file="${iptables_output_dir}/blacklist-vk-v6.ipset" # Create required directories if they don't exist -mkdir -p "${iptables_output_dir}" "${SCRIPT_DIR}/blacklists" - -# Build additional VK-only blacklist from network names in auto/*.txt files -tmp_vk_file="$(mktemp "${SCRIPT_DIR}/blacklists/.blacklist-vk.XXXXXX")" -for source_file in "${auto_all_v4_file}" "${auto_all_v6_file}" "${auto_ripe_v4_file}"; do - [ -f "${source_file}" ] || continue - awk -v pattern="${vk_name_pattern}" 'tolower($0) ~ pattern { print $1 }' "${source_file}" >> "${tmp_vk_file}" -done -sort -u "${tmp_vk_file}" > "${blacklist_vk_file}" -grep ':' "${blacklist_vk_file}" | sort -u > "${blacklist_vk_v6_file}" || true -grep -v ':' "${blacklist_vk_file}" | sort -u > "${blacklist_vk_v4_file}" || true -rm -f "${tmp_vk_file}" +mkdir -p "${iptables_output_dir}" "${BLACKLISTS_DIR}" +build_vk_name_blacklists # Function to generate ipset config from input file generate_ipset_config() { @@ -99,10 +74,10 @@ EOF } # Generate ipset configurations from blacklist files -generate_ipset_config "${blacklist_v4_file}" "${iptables_v4_output_file}" "(IPv4 only)" "blacklist-v4" "inet" -generate_ipset_config "${blacklist_v6_file}" "${iptables_v6_output_file}" "(IPv6 only)" "blacklist-v6" "inet6" -generate_ipset_config "${blacklist_vk_v4_file}" "${iptables_vk_v4_output_file}" "(VK names, IPv4 only)" "blacklist-vk-v4" "inet" -generate_ipset_config "${blacklist_vk_v6_file}" "${iptables_vk_v6_output_file}" "(VK names, IPv6 only)" "blacklist-vk-v6" "inet6" +generate_ipset_config "${BLACKLIST_V4_FILE}" "${iptables_v4_output_file}" "(IPv4 only)" "blacklist-v4" "inet" +generate_ipset_config "${BLACKLIST_V6_FILE}" "${iptables_v6_output_file}" "(IPv6 only)" "blacklist-v6" "inet6" +generate_ipset_config "${BLACKLIST_VK_V4_FILE}" "${iptables_vk_v4_output_file}" "(VK names, IPv4 only)" "blacklist-vk-v4" "inet" +generate_ipset_config "${BLACKLIST_VK_V6_FILE}" "${iptables_vk_v6_output_file}" "(VK names, IPv6 only)" "blacklist-vk-v6" "inet6" echo "" echo "VK outgoing block examples (iptables/ipset):" diff --git a/blacklists_updater_nftables.sh b/blacklists_updater_nftables.sh index 1ca4c3d..3de5b0a 100755 --- a/blacklists_updater_nftables.sh +++ b/blacklists_updater_nftables.sh @@ -4,35 +4,16 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -INPUT_FILE="$SCRIPT_DIR/blacklists/blacklist.txt" +. "${SCRIPT_DIR}/blacklists_updater_common.subr" +INPUT_FILE="${BLACKLIST_FILE}" OUTPUT_DIR="$SCRIPT_DIR/blacklists_nftables" -# Source files for name-based VK filtering -AUTO_ALL_V4_FILE="$SCRIPT_DIR/auto/all-ru-ipv4.txt" -AUTO_ALL_V6_FILE="$SCRIPT_DIR/auto/all-ru-ipv6.txt" -AUTO_RIPE_V4_FILE="$SCRIPT_DIR/auto/ripe-ru-ipv4.txt" -VK_NAME_PATTERN='vk[[:space:]-]*cloud|vkcompany|vkontakte' - -# Additional VK-only text blacklists -VK_INPUT_FILE="$SCRIPT_DIR/blacklists/blacklist-vk.txt" -VK_INPUT_V4_FILE="$SCRIPT_DIR/blacklists/blacklist-vk-v4.txt" -VK_INPUT_V6_FILE="$SCRIPT_DIR/blacklists/blacklist-vk-v6.txt" - # Create required directories if they don't exist -mkdir -p "$OUTPUT_DIR" "$SCRIPT_DIR/blacklists" +mkdir -p "$OUTPUT_DIR" "${BLACKLISTS_DIR}" echo "Generating nftables blacklists..." -# Build additional VK-only blacklist from network names in auto/*.txt files -TMP_VK_FILE="$(mktemp "$SCRIPT_DIR/blacklists/.blacklist-vk.XXXXXX")" -for source_file in "$AUTO_ALL_V4_FILE" "$AUTO_ALL_V6_FILE" "$AUTO_RIPE_V4_FILE"; do - [[ -f "$source_file" ]] || continue - awk -v pattern="$VK_NAME_PATTERN" 'tolower($0) ~ pattern { print $1 }' "$source_file" >> "$TMP_VK_FILE" -done -sort -u "$TMP_VK_FILE" > "$VK_INPUT_FILE" -grep ':' "$VK_INPUT_FILE" | sort -u > "$VK_INPUT_V6_FILE" || true -grep -v ':' "$VK_INPUT_FILE" | sort -u > "$VK_INPUT_V4_FILE" || true -rm -f "$TMP_VK_FILE" +build_vk_name_blacklists # Generate mixed IPv4/IPv6 blacklist (recommended single-file load) python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \ @@ -53,15 +34,15 @@ python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \ "$TMP_V6_FILE" \ "$OUTPUT_DIR/blacklist-v6.nft" -# Generate VK-only blacklists (network names: VK Cloud / VKCOMPANY / VKONTAKTE) +# Generate VK-only blacklists from the narrowed MAX/VK service name filter python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \ - "$VK_INPUT_FILE" \ + "${BLACKLIST_VK_FILE}" \ "$OUTPUT_DIR/blacklist-vk.nft" python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \ - "$VK_INPUT_V4_FILE" \ + "${BLACKLIST_VK_V4_FILE}" \ "$OUTPUT_DIR/blacklist-vk-v4.nft" python3 "$SCRIPT_DIR/generate_nft_blacklist.py" \ - "$VK_INPUT_V6_FILE" \ + "${BLACKLIST_VK_V6_FILE}" \ "$OUTPUT_DIR/blacklist-vk-v6.nft" # Clean up temp files diff --git a/blacklists_updater_nginx.sh b/blacklists_updater_nginx.sh index e0b0e7b..b02c834 100755 --- a/blacklists_updater_nginx.sh +++ b/blacklists_updater_nginx.sh @@ -1,18 +1,16 @@ #!/bin/sh -# Input files (generated by blacklists_updater_txt.sh) -blacklist_file="blacklists/blacklist.txt" -blacklist_v4_file="blacklists/blacklist-v4.txt" -blacklist_v6_file="blacklists/blacklist-v6.txt" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "${SCRIPT_DIR}/blacklists_updater_common.subr" # Output directory and files -nginx_output_dir="blacklists_nginx" +nginx_output_dir="${SCRIPT_DIR}/blacklists_nginx" nginx_output_file="${nginx_output_dir}/blacklist.conf" nginx_v4_output_file="${nginx_output_dir}/blacklist-v4.conf" nginx_v6_output_file="${nginx_output_dir}/blacklist-v6.conf" # Create required directories if they don't exist -mkdir -p "${nginx_output_dir}" "blacklists" +mkdir -p "${nginx_output_dir}" "${BLACKLISTS_DIR}" # Function to generate nginx config from input file generate_nginx_config() { @@ -47,6 +45,6 @@ EOF } # Generate nginx configurations from blacklist files -generate_nginx_config "${blacklist_file}" "${nginx_output_file}" "(mixed IPv4/IPv6)" -generate_nginx_config "${blacklist_v4_file}" "${nginx_v4_output_file}" "(IPv4 only)" -generate_nginx_config "${blacklist_v6_file}" "${nginx_v6_output_file}" "(IPv6 only)" +generate_nginx_config "${BLACKLIST_FILE}" "${nginx_output_file}" "(mixed IPv4/IPv6)" +generate_nginx_config "${BLACKLIST_V4_FILE}" "${nginx_v4_output_file}" "(IPv4 only)" +generate_nginx_config "${BLACKLIST_V6_FILE}" "${nginx_v6_output_file}" "(IPv6 only)" diff --git a/blacklists_updater_routes.sh b/blacklists_updater_routes.sh index b724ed8..629dab9 100755 --- a/blacklists_updater_routes.sh +++ b/blacklists_updater_routes.sh @@ -3,38 +3,18 @@ set -e SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -# Source files for name-based VK filtering -AUTO_ALL_V4_FILE="${SCRIPT_DIR}/auto/all-ru-ipv4.txt" -AUTO_ALL_V6_FILE="${SCRIPT_DIR}/auto/all-ru-ipv6.txt" -AUTO_RIPE_V4_FILE="${SCRIPT_DIR}/auto/ripe-ru-ipv4.txt" -VK_NAME_PATTERN='vk[[:space:]-]*cloud|vkcompany|vkontakte' - -# Additional VK-only text blacklists -VK_INPUT_FILE="${SCRIPT_DIR}/blacklists/blacklist-vk.txt" -VK_INPUT_V4_FILE="${SCRIPT_DIR}/blacklists/blacklist-vk-v4.txt" -VK_INPUT_V6_FILE="${SCRIPT_DIR}/blacklists/blacklist-vk-v6.txt" +. "${SCRIPT_DIR}/blacklists_updater_common.subr" # Output directory and files ROUTES_OUTPUT_DIR="${SCRIPT_DIR}/blacklists_route" ROUTES_V4_FILE="${ROUTES_OUTPUT_DIR}/blacklist-vk-v4.routes" ROUTES_V6_FILE="${ROUTES_OUTPUT_DIR}/blacklist-vk-v6.routes" -mkdir -p "${ROUTES_OUTPUT_DIR}" "${SCRIPT_DIR}/blacklists" +mkdir -p "${ROUTES_OUTPUT_DIR}" "${BLACKLISTS_DIR}" echo "Generating VK route blacklists..." -# Build additional VK-only blacklist from network names in auto/*.txt files -TMP_VK_FILE="$(mktemp "${SCRIPT_DIR}/blacklists/.blacklist-vk.XXXXXX")" -for source_file in "${AUTO_ALL_V4_FILE}" "${AUTO_ALL_V6_FILE}" "${AUTO_RIPE_V4_FILE}"; do - [ -f "${source_file}" ] || continue - awk -v pattern="${VK_NAME_PATTERN}" 'tolower($0) ~ pattern { print $1 }' "${source_file}" >> "${TMP_VK_FILE}" -done - -sort -u "${TMP_VK_FILE}" > "${VK_INPUT_FILE}" -grep ':' "${VK_INPUT_FILE}" | sort -u > "${VK_INPUT_V6_FILE}" || true -grep -v ':' "${VK_INPUT_FILE}" | sort -u > "${VK_INPUT_V4_FILE}" || true -rm -f "${TMP_VK_FILE}" +build_vk_name_blacklists # Generate IPv4 routes file (route VK prefixes to loopback via 127.0.0.1) cat > "${ROUTES_V4_FILE}" << EOF @@ -51,7 +31,7 @@ EOF while IFS= read -r network; do [ -n "${network}" ] || continue printf 'ip route replace %s via 127.0.0.1 dev lo onlink\n' "${network}" >> "${ROUTES_V4_FILE}" -done < "${VK_INPUT_V4_FILE}" +done < "${BLACKLIST_VK_V4_FILE}" # Generate IPv6 routes file (route VK prefixes to loopback via ::1) cat > "${ROUTES_V6_FILE}" << EOF @@ -68,10 +48,10 @@ EOF while IFS= read -r network; do [ -n "${network}" ] || continue printf 'ip -6 route replace %s via ::1 dev lo\n' "${network}" >> "${ROUTES_V6_FILE}" -done < "${VK_INPUT_V6_FILE}" +done < "${BLACKLIST_VK_V6_FILE}" -echo "✓ Generated: ${ROUTES_V4_FILE} (entries: $(wc -l < "${VK_INPUT_V4_FILE}" | tr -d ' '))" -echo "✓ Generated: ${ROUTES_V6_FILE} (entries: $(wc -l < "${VK_INPUT_V6_FILE}" | tr -d ' '))" +echo "✓ Generated: ${ROUTES_V4_FILE} (entries: $(wc -l < "${BLACKLIST_VK_V4_FILE}" | tr -d ' '))" +echo "✓ Generated: ${ROUTES_V6_FILE} (entries: $(wc -l < "${BLACKLIST_VK_V6_FILE}" | tr -d ' '))" echo "" echo "Examples:" echo " sudo sh ${ROUTES_V4_FILE}" diff --git a/blacklists_updater_txt.sh b/blacklists_updater_txt.sh index 764c8c0..5af6d63 100755 --- a/blacklists_updater_txt.sh +++ b/blacklists_updater_txt.sh @@ -1,32 +1,24 @@ #!/bin/sh -outfile_wo_comments="blacklists/blacklist.txt" -outfile_w_comments="blacklists/blacklist_with_comments.txt" -blacklist_v4_file="blacklists/blacklist-v4.txt" -blacklist_v6_file="blacklists/blacklist-v6.txt" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +. "${SCRIPT_DIR}/blacklists_updater_common.subr" -auto_black_ass="auto/black_ass.txt" +ensure_blacklist_base_dirs -black_names="uvd|umvd|fgup|grchc|roskomnad|federalnaya sluzhba|ufsb|zonatelecom|llc vk|vkontakte|ODNOKLASSNIKI|VKCOMPANY|mail.ru|mail-ru|mail_ru|VK-AS|M100" -# M100 - mail.ru -white_names="ruvds" - -mkdir -p blacklists auto - -grep -iE "${black_names}" auto/all-ru-asn.txt | grep -viE "${white_names}" | awk '{ print "# AS-Name: " $0 "\n" $1}' > ${auto_black_ass} -./network_list_from_as.py ${auto_black_ass} > ${outfile_w_comments} -./network_list_from_netname.py lists/ru-gov-netnames.txt >> ${outfile_w_comments} -grep -iE "${black_names}" auto/all-ru-ipv4.txt | grep -viE "${white_names}" | awk '{ print "# NET-Name: " $0 "\n" $1}' >> ${outfile_w_comments} -grep -iE "${black_names}" auto/ripe-ru-ipv4.txt | grep -viE "${white_names}" | awk '{ print "# NET-Name: " $0 "\n" $1}' >> ${outfile_w_comments} +grep -iE "${BLACK_NAMES}" "${AUTO_ALL_ASN_FILE}" | grep -viE "${WHITE_NAMES}" | awk '{ print "# AS-Name: " $0 "\n" $1}' > "${AUTO_BLACK_ASS_FILE}" +"${NETWORK_LIST_FROM_AS}" "${AUTO_BLACK_ASS_FILE}" > "${BLACKLIST_WITH_COMMENTS_FILE}" +"${NETWORK_LIST_FROM_NETNAME}" "${RU_GOV_NETNAMES_FILE}" >> "${BLACKLIST_WITH_COMMENTS_FILE}" +grep -iE "${BLACK_NAMES}" "${AUTO_ALL_V4_FILE}" | grep -viE "${WHITE_NAMES}" | awk '{ print "# NET-Name: " $0 "\n" $1}' >> "${BLACKLIST_WITH_COMMENTS_FILE}" +grep -iE "${BLACK_NAMES}" "${AUTO_RIPE_V4_FILE}" | grep -viE "${WHITE_NAMES}" | awk '{ print "# NET-Name: " $0 "\n" $1}' >> "${BLACKLIST_WITH_COMMENTS_FILE}" # Remove comments by sed to avoid flooding WHOIS servers -grep -v "#" ${outfile_w_comments} | sort | uniq > ${outfile_wo_comments} +grep -v "#" "${BLACKLIST_WITH_COMMENTS_FILE}" | sort | uniq > "${BLACKLIST_FILE}" # Split blacklist into IPv4 and IPv6 -grep ':' "${outfile_wo_comments}" | sort | uniq > "${blacklist_v6_file}" -grep -v ':' "${outfile_wo_comments}" | sort | uniq > "${blacklist_v4_file}" +grep ':' "${BLACKLIST_FILE}" | sort | uniq > "${BLACKLIST_V6_FILE}" +grep -v ':' "${BLACKLIST_FILE}" | sort | uniq > "${BLACKLIST_V4_FILE}" echo "✓ Generated blacklist files" -echo " Mixed (IPv4/IPv6): ${outfile_wo_comments} ($(wc -l < "${outfile_wo_comments}" | tr -d ' ') entries)" -echo " IPv4 only: ${blacklist_v4_file} ($(wc -l < "${blacklist_v4_file}" | tr -d ' ') entries)" -echo " IPv6 only: ${blacklist_v6_file} ($(wc -l < "${blacklist_v6_file}" | tr -d ' ') entries)" +echo " Mixed (IPv4/IPv6): ${BLACKLIST_FILE} ($(wc -l < "${BLACKLIST_FILE}" | tr -d ' ') entries)" +echo " IPv4 only: ${BLACKLIST_V4_FILE} ($(wc -l < "${BLACKLIST_V4_FILE}" | tr -d ' ') entries)" +echo " IPv6 only: ${BLACKLIST_V6_FILE} ($(wc -l < "${BLACKLIST_V6_FILE}" | tr -d ' ') entries)"