From 41738082f0fb2e931df6b2ab1eb8e0e7c3446aba Mon Sep 17 00:00:00 2001 From: symstu Date: Wed, 29 Oct 2025 08:05:27 +0100 Subject: [PATCH] Renamed JA5 hashes to TF hashes --- README.md | 16 +-- app.py | 78 +++++------ blockers/__init__.py | 4 +- blockers/base.py | 2 +- blockers/{ja5h.py => tfh.py} | 22 ++-- blockers/{ja5t.py => tft.py} | 20 +-- config.py | 60 ++++----- detectors/base.py | 6 +- detectors/geoip.py | 11 +- detectors/ip.py | 12 +- detectors/{ja5h.py => tfh.py} | 32 ++--- detectors/{ja5t.py => tft.py} | 32 ++--- example.env | 124 +++++++++--------- tests/base.py | 4 +- ...st_blocker_ja5h.py => test_blocker_tfh.py} | 22 ++-- ...st_blocker_ja5t.py => test_blocker_tft.py} | 22 ++-- tests/test_detector_base.py | 2 - tests/test_detector_ip_based.py | 6 +- ...5h_based.py => test_detector_tfh_based.py} | 42 +++--- ...5t_based.py => test_detector_tft_based.py} | 42 +++--- ...fespan_background_monitor_release_users.py | 10 +- ...lifespan_background_monitor_risky_users.py | 28 ++-- tests/test_lifespan_initialization.py | 6 +- .../test_lifespan_training_historical_mode.py | 12 +- tests/test_lifespan_training_real_mode.py | 12 +- .../{test_ja5_config.py => test_tf_config.py} | 14 +- tests/test_user_agents.py | 2 - utils/datatypes.py | 6 +- utils/{ja5_config.py => tf_config.py} | 48 +++---- 29 files changed, 345 insertions(+), 352 deletions(-) rename blockers/{ja5h.py => tfh.py} (53%) rename blockers/{ja5t.py => tft.py} (82%) rename detectors/{ja5h.py => tfh.py} (77%) rename detectors/{ja5t.py => tft.py} (77%) rename tests/{test_blocker_ja5h.py => test_blocker_tfh.py} (74%) rename tests/{test_blocker_ja5t.py => test_blocker_tft.py} (74%) rename tests/{test_detector_ja5h_based.py => test_detector_tfh_based.py} (87%) rename tests/{test_detector_ja5t_based.py => test_detector_tft_based.py} (87%) rename tests/{test_ja5_config.py => test_tf_config.py} (80%) rename utils/{ja5_config.py => tf_config.py} (65%) diff --git a/README.md b/README.md index 3fe835c..5bee114 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Tempesta WebShield -Block users by JA5T, JA5H, or IP based on Tempesta FW access +Block users by TFT, TFH, or IP based on Tempesta FW access logs stored in the ClickHouse database. [**WIKI**](https://tempesta-tech.com/knowledge-base/Bot-Protection/) @@ -58,19 +58,19 @@ isort . # How to block ### Prepare Tempesta FW config -It's useful to define separate directories for different groups of JA5 hashes +It's useful to define separate directories for different groups of TF hashes in the Tempesta FW configuration file (/etc/tempesta/tempesta_fw.conf). ```nginx -ja5t { - !include /etc/tempesta/ja5t/ +tft { + !include /etc/tempesta/tft/ } -ja5h { - !include /etc/tempesta/ja5h/ +tfh { + !include /etc/tempesta/tfh/ } ``` Then add 2 files -- /etc/tempesta/ja5t/blocked.conf -- /etc/tempesta/ja5h/blocked.conf +- /etc/tempesta/tft/blocked.conf +- /etc/tempesta/tfh/blocked.conf These files should be used by default by the WebShield to add new blocking hashes. diff --git a/app.py b/app.py index 825d1f1..43c9994 100755 --- a/app.py +++ b/app.py @@ -13,18 +13,18 @@ IPErrorRequestDetector, IPRPSDetector, ) -from detectors.ja5t import ( - Ja5tAccumulativeTimeDetector, - Ja5tErrorRequestDetector, - Ja5tRPSDetector, +from detectors.tft import ( + TFtAccumulativeTimeDetector, + TFtErrorRequestDetector, + TFtRPSDetector, ) -from detectors.ja5h import ( - Ja5hAccumulativeTimeDetector, - Ja5hErrorRequestDetector, - Ja5hRPSDetector, +from detectors.tfh import ( + TFhAccumulativeTimeDetector, + TFhErrorRequestDetector, + TFhRPSDetector, ) from utils.access_log import ClickhouseAccessLog -from utils.ja5_config import Ja5Config +from utils.tf_config import TFConfig from utils.logger import logger from utils.user_agents import UserAgentsManager @@ -50,13 +50,13 @@ ) context = AppContext( blockers={ - blockers.Ja5tBlocker.name(): blockers.Ja5tBlocker( - config=Ja5Config(file_path=app_config.path_to_ja5t_config), + blockers.TFtBlocker.name(): blockers.TFtBlocker( + config=TFConfig(file_path=app_config.path_to_tft_config), tempesta_executable_path=app_config.tempesta_executable_path, tempesta_config_path=app_config.tempesta_config_path, ), - blockers.Ja5hBlocker.name(): blockers.Ja5hBlocker( - config=Ja5Config(file_path=app_config.path_to_ja5h_config), + blockers.TFhBlocker.name(): blockers.TFhBlocker( + config=TFConfig(file_path=app_config.path_to_tfh_config), tempesta_executable_path=app_config.tempesta_executable_path, tempesta_config_path=app_config.tempesta_config_path, ), @@ -87,43 +87,43 @@ block_users_per_iteration=app_config.detector_ip_errors_block_users_per_iteration, allowed_statues=app_config.detector_ip_errors_allowed_statuses, ), - Ja5tRPSDetector.name(): Ja5tRPSDetector( + TFtRPSDetector.name(): TFtRPSDetector( access_log=clickhouse_client, - default_threshold=app_config.detector_ja5t_rps_default_threshold, - intersection_percent=app_config.detector_ja5t_rps_intersection_percent, - block_users_per_iteration=app_config.detector_ja5t_rps_block_users_per_iteration, + default_threshold=app_config.detector_tft_rps_default_threshold, + intersection_percent=app_config.detector_tft_rps_intersection_percent, + block_users_per_iteration=app_config.detector_tft_rps_block_users_per_iteration, ), - Ja5tAccumulativeTimeDetector.name(): Ja5tAccumulativeTimeDetector( + TFtAccumulativeTimeDetector.name(): TFtAccumulativeTimeDetector( access_log=clickhouse_client, - default_threshold=app_config.detector_ja5t_time_default_threshold, - intersection_percent=app_config.detector_ja5t_time_intersection_percent, - block_users_per_iteration=app_config.detector_ja5t_time_block_users_per_iteration, + default_threshold=app_config.detector_tft_time_default_threshold, + intersection_percent=app_config.detector_tft_time_intersection_percent, + block_users_per_iteration=app_config.detector_tft_time_block_users_per_iteration, ), - Ja5tErrorRequestDetector.name(): Ja5tErrorRequestDetector( + TFtErrorRequestDetector.name(): TFtErrorRequestDetector( access_log=clickhouse_client, - default_threshold=app_config.detector_ja5t_errors_default_threshold, - intersection_percent=app_config.detector_ja5t_errors_intersection_percent, - block_users_per_iteration=app_config.detector_ja5t_errors_block_users_per_iteration, - allowed_statues=app_config.detector_ja5t_errors_allowed_statuses, + default_threshold=app_config.detector_tft_errors_default_threshold, + intersection_percent=app_config.detector_tft_errors_intersection_percent, + block_users_per_iteration=app_config.detector_tft_errors_block_users_per_iteration, + allowed_statues=app_config.detector_tft_errors_allowed_statuses, ), - Ja5hRPSDetector.name(): Ja5hRPSDetector( + TFhRPSDetector.name(): TFhRPSDetector( access_log=clickhouse_client, - default_threshold=app_config.detector_ja5h_rps_default_threshold, - intersection_percent=app_config.detector_ja5h_rps_intersection_percent, - block_users_per_iteration=app_config.detector_ja5h_rps_block_users_per_iteration, + default_threshold=app_config.detector_tfh_rps_default_threshold, + intersection_percent=app_config.detector_tfh_rps_intersection_percent, + block_users_per_iteration=app_config.detector_tfh_rps_block_users_per_iteration, ), - Ja5hAccumulativeTimeDetector.name(): Ja5hAccumulativeTimeDetector( + TFhAccumulativeTimeDetector.name(): TFhAccumulativeTimeDetector( access_log=clickhouse_client, - default_threshold=app_config.detector_ja5h_time_default_threshold, - intersection_percent=app_config.detector_ja5h_time_intersection_percent, - block_users_per_iteration=app_config.detector_ja5h_time_block_users_per_iteration, + default_threshold=app_config.detector_tfh_time_default_threshold, + intersection_percent=app_config.detector_tfh_time_intersection_percent, + block_users_per_iteration=app_config.detector_tfh_time_block_users_per_iteration, ), - Ja5hErrorRequestDetector.name(): Ja5tErrorRequestDetector( + TFhErrorRequestDetector.name(): TFtErrorRequestDetector( access_log=clickhouse_client, - default_threshold=app_config.detector_ja5h_errors_default_threshold, - intersection_percent=app_config.detector_ja5h_errors_intersection_percent, - block_users_per_iteration=app_config.detector_ja5h_errors_block_users_per_iteration, - allowed_statues=app_config.detector_ja5h_errors_allowed_statuses, + default_threshold=app_config.detector_tfh_errors_default_threshold, + intersection_percent=app_config.detector_tfh_errors_intersection_percent, + block_users_per_iteration=app_config.detector_tfh_errors_block_users_per_iteration, + allowed_statues=app_config.detector_tfh_errors_allowed_statuses, ), GeoIPDetector.name(): GeoIPDetector( access_log=clickhouse_client, diff --git a/blockers/__init__.py b/blockers/__init__.py index d7b5517..a4d1799 100644 --- a/blockers/__init__.py +++ b/blockers/__init__.py @@ -1,6 +1,6 @@ from .ipset import IpSetBlocker -from .ja5h import Ja5hBlocker -from .ja5t import Ja5tBlocker +from .tfh import TFhBlocker +from .tft import TFtBlocker from .nft import NFTBlocker __author__ = "Tempesta Technologies, Inc." diff --git a/blockers/base.py b/blockers/base.py index b108e08..6def89d 100644 --- a/blockers/base.py +++ b/blockers/base.py @@ -53,7 +53,7 @@ def release(self, user: User): def apply(self): """ Apply blocking rules. Some blockers may block immediately - without calling this method, but others — like ja5t — apply + without calling this method, but others — like tft — apply rules only after multiple config changes. """ diff --git a/blockers/ja5h.py b/blockers/tfh.py similarity index 53% rename from blockers/ja5h.py rename to blockers/tfh.py index b6178a7..0fad933 100644 --- a/blockers/ja5h.py +++ b/blockers/tfh.py @@ -1,8 +1,8 @@ import time -from blockers.ja5t import Ja5tBlocker +from blockers.tft import TFtBlocker from utils.datatypes import User -from utils.ja5_config import Ja5Hash +from utils.tf_config import TFHash from utils.logger import logger __author__ = "Tempesta Technologies, Inc." @@ -10,11 +10,11 @@ __license__ = "GPL2" -class Ja5hBlocker(Ja5tBlocker): +class TFhBlocker(TFtBlocker): @staticmethod def name() -> str: - return "ja5h" + return "tfh" def load(self) -> dict[int, User]: self.config.load() @@ -22,23 +22,23 @@ def load(self) -> dict[int, User]: result = dict() for hash_value in self.config.hashes: - user = User(ja5h=[hash_value], blocked_at=current_time) + user = User(tfh=[hash_value], blocked_at=current_time) result[hash(user)] = user return result def block(self, user: User): - if self.config.exists(user.ja5h[0]): + if self.config.exists(user.tfh[0]): return None - self.config.add(Ja5Hash(value=user.ja5h[0], packets=0, connections=0)) - logger.warning(f"Blocked user {user} by ja5h") + self.config.add(TFHash(value=user.tfh[0], packets=0, connections=0)) + logger.warning(f"Blocked user {user} by tfh") def release(self, user: User): - if not self.config.exists(user.ja5h[0]): + if not self.config.exists(user.tfh[0]): return None - self.config.remove(user.ja5h[0]) + self.config.remove(user.tfh[0]) def info(self) -> list[User]: - return [User(ja5h=[ja5_hash.value]) for ja5_hash in self.config.hashes.values()] + return [User(tfh=[tf_hash.value]) for tf_hash in self.config.hashes.values()] diff --git a/blockers/ja5t.py b/blockers/tft.py similarity index 82% rename from blockers/ja5t.py rename to blockers/tft.py index e7f55e0..226efee 100644 --- a/blockers/ja5t.py +++ b/blockers/tft.py @@ -3,7 +3,7 @@ from blockers.base import BaseBlocker, PreparationError from utils.datatypes import User -from utils.ja5_config import Ja5Config, Ja5Hash +from utils.tf_config import TFConfig, TFHash from utils.logger import logger from utils.shell import run_in_shell @@ -12,10 +12,10 @@ __license__ = "GPL2" -class Ja5tBlocker(BaseBlocker): +class TFtBlocker(BaseBlocker): def __init__( self, - config: Ja5Config, + config: TFConfig, tempesta_executable_path: str = None, tempesta_config_path: str = None, ): @@ -25,7 +25,7 @@ def __init__( @staticmethod def name() -> str: - return "ja5t" + return "tft" def __tempesta_app_exists(self) -> bool: if self.tempesta_executable_path and os.path.isfile( @@ -53,21 +53,21 @@ def load(self) -> dict[int, User]: result = dict() for hash_value in self.config.hashes: - user = User(ja5t=[hash_value], blocked_at=current_time) + user = User(tft=[hash_value], blocked_at=current_time) result[hash(user)] = user return result def block(self, user: User): - for hash_value in user.ja5t: + for hash_value in user.tft: if self.config.exists(hash_value): continue - self.config.add(Ja5Hash(value=hash_value, packets=0, connections=0)) - logger.warning(f"Blocked user {user} by ja5t") + self.config.add(TFHash(value=hash_value, packets=0, connections=0)) + logger.warning(f"Blocked user {user} by tft") def release(self, user: User): - for hash_value in user.ja5t: + for hash_value in user.tft: if not self.config.exists(hash_value): continue @@ -93,4 +93,4 @@ def apply(self): ) def info(self) -> list[User]: - return [User(ja5t=[ja5_hash.value]) for ja5_hash in self.config.hashes.values()] + return [User(tft=[tf_hash.value]) for tf_hash in self.config.hashes.values()] diff --git a/config.py b/config.py index 77b23a8..5f3407a 100644 --- a/config.py +++ b/config.py @@ -9,8 +9,8 @@ class AppConfig(BaseSettings): - path_to_ja5t_config: str = "/etc/tempesta/ja5t/blocked.conf" - path_to_ja5h_config: str = "/etc/tempesta/ja5h/blocked.conf" + path_to_tft_config: str = "/etc/tempesta/tft/blocked.conf" + path_to_tfh_config: str = "/etc/tempesta/tfh/blocked.conf" clickhouse_host: str = "192.168.0.104" clickhouse_port: int = 8123 @@ -28,17 +28,17 @@ class AppConfig(BaseSettings): "ip_rps", "ip_time", "ip_errors", - "ja5t_rps", - "ja5t_time", - "ja5t_errors", - "ja5h_rps", - "ja5h_time", - "ja5h_errors", + "tft_rps", + "tft_time", + "tft_errors", + "tfh_rps", + "tfh_time", + "tfh_errors", "geoip", ] - ] = {"ja5t_rps", "ja5t_time", "ja5t_errors"} + ] = {"tft_rps", "tft_time", "tft_errors"} - blocking_types: set[Literal["ja5t", "ja5h", "ipset", "nftables"]] = {"ja5t"} + blocking_types: set[Literal["tft", "tfh", "ipset", "nftables"]] = {"tft"} blocking_window_duration_sec: int = 10 blocking_ipset_name: str = "tempesta_blocked_ips" blocking_time_min: int = 60 @@ -77,18 +77,18 @@ class AppConfig(BaseSettings): 403, ] - detector_ja5t_rps_default_threshold: Decimal = Decimal(10) - detector_ja5t_rps_intersection_percent: Decimal = Decimal(10) - detector_ja5t_rps_block_users_per_iteration: Decimal = Decimal(10) + detector_tft_rps_default_threshold: Decimal = Decimal(10) + detector_tft_rps_intersection_percent: Decimal = Decimal(10) + detector_tft_rps_block_users_per_iteration: Decimal = Decimal(10) - detector_ja5t_time_default_threshold: Decimal = Decimal(10) - detector_ja5t_time_intersection_percent: Decimal = Decimal(10) - detector_ja5t_time_block_users_per_iteration: Decimal = Decimal(10) + detector_tft_time_default_threshold: Decimal = Decimal(10) + detector_tft_time_intersection_percent: Decimal = Decimal(10) + detector_tft_time_block_users_per_iteration: Decimal = Decimal(10) - detector_ja5t_errors_default_threshold: Decimal = Decimal(10) - detector_ja5t_errors_intersection_percent: Decimal = Decimal(10) - detector_ja5t_errors_block_users_per_iteration: Decimal = Decimal(10) - detector_ja5t_errors_allowed_statuses: list[int] = [ + detector_tft_errors_default_threshold: Decimal = Decimal(10) + detector_tft_errors_intersection_percent: Decimal = Decimal(10) + detector_tft_errors_block_users_per_iteration: Decimal = Decimal(10) + detector_tft_errors_allowed_statuses: list[int] = [ 100, 101, 200, @@ -107,18 +107,18 @@ class AppConfig(BaseSettings): 403, ] - detector_ja5h_rps_default_threshold: Decimal = Decimal(10) - detector_ja5h_rps_intersection_percent: Decimal = Decimal(10) - detector_ja5h_rps_block_users_per_iteration: Decimal = Decimal(10) + detector_tfh_rps_default_threshold: Decimal = Decimal(10) + detector_tfh_rps_intersection_percent: Decimal = Decimal(10) + detector_tfh_rps_block_users_per_iteration: Decimal = Decimal(10) - detector_ja5h_time_default_threshold: Decimal = Decimal(10) - detector_ja5h_time_intersection_percent: Decimal = Decimal(10) - detector_ja5h_time_block_users_per_iteration: Decimal = Decimal(10) + detector_tfh_time_default_threshold: Decimal = Decimal(10) + detector_tfh_time_intersection_percent: Decimal = Decimal(10) + detector_tfh_time_block_users_per_iteration: Decimal = Decimal(10) - detector_ja5h_errors_default_threshold: Decimal = Decimal(10) - detector_ja5h_errors_intersection_percent: Decimal = Decimal(10) - detector_ja5h_errors_block_users_per_iteration: Decimal = Decimal(10) - detector_ja5h_errors_allowed_statuses: list[int] = [ + detector_tfh_errors_default_threshold: Decimal = Decimal(10) + detector_tfh_errors_intersection_percent: Decimal = Decimal(10) + detector_tfh_errors_block_users_per_iteration: Decimal = Decimal(10) + detector_tfh_errors_allowed_statuses: list[int] = [ 100, 101, 200, diff --git a/detectors/base.py b/detectors/base.py index 48ebf5e..64b7b33 100644 --- a/detectors/base.py +++ b/detectors/base.py @@ -91,7 +91,7 @@ async def find_users( ) @property - def validation_key(self) -> typing.Literal['ip', 'ja5t', 'ja5h']: + def validation_key(self) -> typing.Literal['ip', 'tft', 'tfh']: """ The user model validation field """ @@ -210,8 +210,8 @@ async def fetch_for_period(self, start_at: int, finish_at: int) -> list[User]: return [ User( - ja5t=[str(hex(ja5t))[2:] for ja5t in user[0]], - ja5h=[str(hex(ja5h))[2:] for ja5h in user[1]], + tft=[str(hex(tft))[2:] for tft in user[0]], + tfh=[str(hex(tfh))[2:] for tfh in user[1]], ip=user[2], value=user[3], # type=user[4] diff --git a/detectors/geoip.py b/detectors/geoip.py index ff4d1ee..18d0f7d 100644 --- a/detectors/geoip.py +++ b/detectors/geoip.py @@ -1,13 +1,10 @@ import os -import time from dataclasses import dataclass, field from decimal import Decimal from geoip2.database import City, Reader -from config import AppConfig from detectors.base import BaseDetector -from utils.access_log import ClickhouseAccessLog from utils.datatypes import User from utils.logger import logger @@ -81,8 +78,8 @@ async def fetch_for_period(self, start_at: int, finish_at: int) -> list[User]: and timestamp < {finish_at} ) SELECT - groupUniqArray(ja5t) ja5t, - groupUniqArray(ja5h) ja5h, + groupUniqArray(tft) tft, + groupUniqArray(tfh) tfh, address address, count(1) value FROM prepared_users @@ -92,8 +89,8 @@ async def fetch_for_period(self, start_at: int, finish_at: int) -> list[User]: return [ User( - ja5t=[str(hex(ja5t))[2:] for ja5t in user[0]], - ja5h=[str(hex(ja5h))[2:] for ja5h in user[1]], + tft=[str(hex(tft))[2:] for tft in user[0]], + tfh=[str(hex(tfh))[2:] for tfh in user[1]], ip=[user[2]], value=user[3], ) diff --git a/detectors/ip.py b/detectors/ip.py index e76452f..d6459de 100644 --- a/detectors/ip.py +++ b/detectors/ip.py @@ -31,8 +31,8 @@ def get_request(self, start_at, finish_at): return self.shared_filter( f""" SELECT - groupUniqArray(ja5t) ja5t, - groupUniqArray(ja5h) ja5h, + groupUniqArray(tft) tft, + groupUniqArray(tfh) tfh, array(address) address, count(1) value FROM prepared_users @@ -61,8 +61,8 @@ def get_request(self, start_at, finish_at): return self.shared_filter( f""" SELECT - groupUniqArray(ja5t) ja5t, - groupUniqArray(ja5h) ja5h, + groupUniqArray(tft) tft, + groupUniqArray(tfh) tfh, array(address) address, countIf(status not in ({statuses})) value FROM prepared_users @@ -86,8 +86,8 @@ def get_request(self, start_at, finish_at): return self.shared_filter( f""" SELECT - groupUniqArray(ja5t) ja5t, - groupUniqArray(ja5h) ja5h, + groupUniqArray(tft) tft, + groupUniqArray(tfh) tfh, array(address) address, sum(response_time) value FROM prepared_users diff --git a/detectors/ja5h.py b/detectors/tfh.py similarity index 77% rename from detectors/ja5h.py rename to detectors/tfh.py index 46d461a..6300faa 100644 --- a/detectors/ja5h.py +++ b/detectors/tfh.py @@ -5,25 +5,25 @@ __license__ = "GPL2" -class Ja5hRPSDetector(IPRPSDetector): +class TFhRPSDetector(IPRPSDetector): @staticmethod def name() -> str: - return "ja5h_rps" + return "tfh_rps" @property def validation_key(self) -> str: - return 'ja5h' + return 'tfh' def get_request(self, start_at, finish_at): return self.shared_filter( f""" SELECT - groupUniqArray(ja5t) ja5t, - array(ja5h) ja5h, + groupUniqArray(tft) tft, + array(tfh) tfh, groupUniqArray(address) address, count(1) value FROM prepared_users - GROUP by ja5h + GROUP by tfh HAVING value >= {self.threshold} LIMIT {self.block_limit_per_check} @@ -33,26 +33,26 @@ def get_request(self, start_at, finish_at): ) -class Ja5hErrorRequestDetector(Ja5hRPSDetector): +class TFhErrorRequestDetector(TFhRPSDetector): def __init__(self, *args, allowed_statues: list[int] = (), **kwargs): super().__init__(*args, **kwargs) self.allowed_statues = allowed_statues @staticmethod def name() -> str: - return "ja5h_errors" + return "tfh_errors" def get_request(self, start_at, finish_at): statuses = ", ".join(list(map(str, self.allowed_statues))) return self.shared_filter( f""" SELECT - groupUniqArray(ja5t) ja5t, - array(ja5h) ja5h, + groupUniqArray(tft) tft, + array(tfh) tfh, groupUniqArray(address) address, countIf(status not in ({statuses})) value FROM prepared_users - GROUP by ja5h + GROUP by tfh HAVING value >= {self.threshold} LIMIT {self.block_limit_per_check} @@ -62,22 +62,22 @@ def get_request(self, start_at, finish_at): ) -class Ja5hAccumulativeTimeDetector(Ja5hRPSDetector): +class TFhAccumulativeTimeDetector(TFhRPSDetector): @staticmethod def name() -> str: - return "ja5h_time" + return "tfh_time" def get_request(self, start_at, finish_at): return self.shared_filter( f""" SELECT - groupUniqArray(ja5t) ja5t, - array(ja5h) ja5h, + groupUniqArray(tft) tft, + array(tfh) tfh, groupUniqArray(address) address, sum(response_time) value FROM prepared_users - GROUP by ja5h + GROUP by tfh HAVING value >= {self.threshold} LIMIT {self.block_limit_per_check} diff --git a/detectors/ja5t.py b/detectors/tft.py similarity index 77% rename from detectors/ja5t.py rename to detectors/tft.py index c99b66e..67ec1b6 100644 --- a/detectors/ja5t.py +++ b/detectors/tft.py @@ -5,25 +5,25 @@ __license__ = "GPL2" -class Ja5tRPSDetector(IPRPSDetector): +class TFtRPSDetector(IPRPSDetector): @staticmethod def name() -> str: - return "ja5t_rps" + return "tft_rps" @property def validation_key(self) -> str: - return 'ja5t' + return 'tft' def get_request(self, start_at, finish_at): return self.shared_filter( f""" SELECT - array(ja5t) ja5t, - groupUniqArray(ja5h) ja5h, + array(tft) tft, + groupUniqArray(tfh) tfh, groupUniqArray(address) address, count(1) value FROM prepared_users - GROUP by ja5t + GROUP by tft HAVING value >= {self.threshold} LIMIT {self.block_limit_per_check} @@ -33,26 +33,26 @@ def get_request(self, start_at, finish_at): ) -class Ja5tErrorRequestDetector(Ja5tRPSDetector): +class TFtErrorRequestDetector(TFtRPSDetector): def __init__(self, *args, allowed_statues: list[int] = (), **kwargs): super().__init__(*args, **kwargs) self.allowed_statues = allowed_statues @staticmethod def name() -> str: - return "ja5t_errors" + return "tft_errors" def get_request(self, start_at, finish_at): statuses = ", ".join(list(map(str, self.allowed_statues))) return self.shared_filter( f""" SELECT - array(ja5t) ja5t, - groupUniqArray(ja5h) ja5h, + array(tft) tft, + groupUniqArray(tfh) tfh, groupUniqArray(address) address, countIf(status not in ({statuses})) value FROM prepared_users - GROUP by ja5t + GROUP by tft HAVING value >= {self.threshold} LIMIT {self.block_limit_per_check} @@ -62,22 +62,22 @@ def get_request(self, start_at, finish_at): ) -class Ja5tAccumulativeTimeDetector(Ja5tRPSDetector): +class TFtAccumulativeTimeDetector(TFtRPSDetector): @staticmethod def name() -> str: - return "ja5t_time" + return "tft_time" def get_request(self, start_at, finish_at): return self.shared_filter( f""" SELECT - array(ja5t) ja5t, - groupUniqArray(ja5h) ja5h, + array(tft) tft, + groupUniqArray(tfh) tfh, groupUniqArray(address) address, sum(response_time) value FROM prepared_users - GROUP by ja5t + GROUP by tft HAVING value >= {self.threshold} LIMIT {self.block_limit_per_check} diff --git a/example.env b/example.env index 2b6df5e..30cd83d 100644 --- a/example.env +++ b/example.env @@ -1,10 +1,10 @@ -# Path to the Tempesta config file with predefined JA5T-blocked hashes. +# Path to the Tempesta config file with predefined TFT-blocked hashes. # This file is also used to add new blocking hashes. -PATH_TO_JA5T_CONFIG="/etc/tempesta/ja5t/blocked.conf" +PATH_TO_TFT_CONFIG="/etc/tempesta/tft/blocked.conf" -# Path to the Tempesta config file with predefined JA5H-blocked hashes. +# Path to the Tempesta config file with predefined TFH-blocked hashes. # This file is also used to add new blocking hashes. -PATH_TO_JA5H_CONFIG="/etc/tempesta/ja5h/blocked.conf" +PATH_TO_TFH_CONFIG="/etc/tempesta/tfh/blocked.conf" # ClickHouse server host where Tempesta FW stores access logs. CLICKHOUSE_HOST="127.0.0.1" @@ -44,23 +44,23 @@ PERSISTENT_USERS_WINDOW_DURATION_MIN=60 # ip_rps - Fetch users by IP who generate the highest RPS (requests per second) traffic. # ip_time - Fetch users by IP who generate traffic with the highest accumulated response time. # ip_errors - Fetch users by IP who generate traffic with the highest number of response errors. -# ja5t_rps - Fetch users by JA5T-hashes who generate the highest RPS (requests per second) traffic. -# ja5t_time - Fetch users by JA5T-hashes who generate traffic with the highest accumulated response time. -# ja5t_errors - Fetch users by JA5T-hashes who generate traffic with the highest number of response errors. -# ja5h_rps - Fetch users by JA5H-hashes who generate the highest RPS (requests per second) traffic. -# ja5h_time - Fetch users by JA5H-hashes who generate traffic with the highest accumulated response time. -# ja5h_errors - Fetch users by JA5H-hashes who generate traffic with the highest number of response errors. +# tft_rps - Fetch users by TFT-hashes who generate the highest RPS (requests per second) traffic. +# tft_time - Fetch users by TFT-hashes who generate traffic with the highest accumulated response time. +# tft_errors - Fetch users by TFT-hashes who generate traffic with the highest number of response errors. +# tfh_rps - Fetch users by TFH-hashes who generate the highest RPS (requests per second) traffic. +# tfh_time - Fetch users by TFH-hashes who generate traffic with the highest accumulated response time. +# tfh_errors - Fetch users by TFH-hashes who generate traffic with the highest number of response errors. # geoip - Fetch users by cities who generate traffic with the highest RPS and block all users from those cities. -DETECTORS=["ja5t_rps","ja5t_time","ja5t_errors"] +DETECTORS=["tft_rps","tft_time","tft_errors"] # Defines how the WebShield should block users. Multiple values can be set. -# Possible values: ja5t, ja5h, ipset, nftables +# Possible values: tft, tfh, ipset, nftables # -# ja5t – Blocks users by their JA5T fingerprint based on TLS connection data. -# ja5h – Blocks users by their JA5H fingerprint based on HTTP request data. +# tft – Blocks users by their TFT fingerprint based on TLS connection data. +# tfh – Blocks users by their TFH fingerprint based on HTTP request data. # ipset – Blocks users by IP using IPSet and iptables. # nftables – Blocks users by IP using nftables. -BLOCKING_TYPES=["ja5t"] +BLOCKING_TYPES=["tft"] # The duration of the time interval during which users are fetched # for traffic analysis. Used in combination with the `start_at` position. @@ -137,93 +137,93 @@ DETECTOR_IP_ERRORS_BLOCK_USERS_PER_ITERATION=100 # status codes that should be ignored by the filters. DETECTOR_IP_ERRORS_ALLOWED_STATUSES=[100,101,200,201,204,300,301,302,303,304,305,307,308,400,401,403] -# The configuration parameter of the JA5T_RPS detector. Installs the default RPS threshold. -DETECTOR_JA5T_RPS_DEFAULT_THRESHOLD=10 +# The configuration parameter of the TFT_RPS detector. Installs the default RPS threshold. +DETECTOR_TFT_RPS_DEFAULT_THRESHOLD=10 -# The configuration parameter of the JA5T_RPS detector. Defines the difference between two user groups. -# If the last users group generates DETECTOR_JA5T_RPS_INTERSECTION_PERCENT times more traffic than the previous one, +# The configuration parameter of the TFT_RPS detector. Defines the difference between two user groups. +# If the last users group generates DETECTOR_TFT_RPS_INTERSECTION_PERCENT times more traffic than the previous one, # some of those users should be blocked. -DETECTOR_JA5T_RPS_INTERSECTION_PERCENT=10 +DETECTOR_TFT_RPS_INTERSECTION_PERCENT=10 -# The configuration parameter of the JA5T_RPS detector. Defines the number of +# The configuration parameter of the TFT_RPS detector. Defines the number of # users that can be blocked per check. -DETECTOR_JA5T_RPS_BLOCK_USERS_PER_ITERATION=100 +DETECTOR_TFT_RPS_BLOCK_USERS_PER_ITERATION=100 -# The configuration parameter of the JA5T_TIME detector. Installs the default +# The configuration parameter of the TFT_TIME detector. Installs the default # accumulative time threshold. -DETECTOR_JA5T_TIME_DEFAULT_THRESHOLD=10 +DETECTOR_TFT_TIME_DEFAULT_THRESHOLD=10 -# The configuration parameter of the JA5T_TIME detector. Defines the difference between two user groups. -# If the last users group generates DETECTOR_JA5T_TIME_INTERSECTION_PERCENT times more traffic than the previous one, +# The configuration parameter of the TFT_TIME detector. Defines the difference between two user groups. +# If the last users group generates DETECTOR_TFT_TIME_INTERSECTION_PERCENT times more traffic than the previous one, # some of those users should be blocked. -DETECTOR_JA5T_TIME_INTERSECTION_PERCENT=10 +DETECTOR_TFT_TIME_INTERSECTION_PERCENT=10 -# The configuration parameter of the JA5T_TIME detector. Defines the number of +# The configuration parameter of the TFT_TIME detector. Defines the number of # users that can be blocked per check. -DETECTOR_JA5T_TIME_BLOCK_USERS_PER_ITERATION=100 +DETECTOR_TFT_TIME_BLOCK_USERS_PER_ITERATION=100 -# The configuration parameter of the JA5T_ERRORS detector. Installs the default error responses threshold. -DETECTOR_JA5T_ERRORS_DEFAULT_THRESHOLD=10 +# The configuration parameter of the TFT_ERRORS detector. Installs the default error responses threshold. +DETECTOR_TFT_ERRORS_DEFAULT_THRESHOLD=10 -# The configuration parameter of the JA5T_ERRORS detector. Defines the difference between two user groups. -# If the last users group generates DETECTOR_JA5T_ERRORS_INTERSECTION_PERCENT times more traffic than the previous one, +# The configuration parameter of the TFT_ERRORS detector. Defines the difference between two user groups. +# If the last users group generates DETECTOR_TFT_ERRORS_INTERSECTION_PERCENT times more traffic than the previous one, # some of those users should be blocked. -DETECTOR_JA5T_ERRORS_INTERSECTION_PERCENT=10 +DETECTOR_TFT_ERRORS_INTERSECTION_PERCENT=10 -# The configuration parameter of the JA5T_ERRORS detector. Defines the number of +# The configuration parameter of the TFT_ERRORS detector. Defines the number of # users that can be blocked per check. -DETECTOR_JA5T_ERRORS_BLOCK_USERS_PER_ITERATION=100 +DETECTOR_TFT_ERRORS_BLOCK_USERS_PER_ITERATION=100 -# The configuration parameter of the JA5T_ERRORS detector. Defines the list of whitelisted response +# The configuration parameter of the TFT_ERRORS detector. Defines the list of whitelisted response # status codes that should be ignored by the filters. -DETECTOR_JA5T_ERRORS_ALLOWED_STATUSES=[100,101,200,201,204,300,301,302,303,304,305,307,308,400,401,403] +DETECTOR_TFT_ERRORS_ALLOWED_STATUSES=[100,101,200,201,204,300,301,302,303,304,305,307,308,400,401,403] -# The configuration parameter of the JA5H_RPS detector. Installs the default RPS threshold. -DETECTOR_JA5H_RPS_DEFAULT_THRESHOLD=10 +# The configuration parameter of the TFH_RPS detector. Installs the default RPS threshold. +DETECTOR_TFH_RPS_DEFAULT_THRESHOLD=10 -# The configuration parameter of the JA5H_RPS detector. Defines the difference between two user groups. -# If the last users group generates DETECTOR_JA5H_RPS_INTERSECTION_PERCENT times more traffic than the previous one, +# The configuration parameter of the TFH_RPS detector. Defines the difference between two user groups. +# If the last users group generates DETECTOR_TFH_RPS_INTERSECTION_PERCENT times more traffic than the previous one, # some of those users should be blocked. -DETECTOR_JA5H_RPS_INTERSECTION_PERCENT=10 +DETECTOR_TFH_RPS_INTERSECTION_PERCENT=10 -# The configuration parameter of the JA5H_RPS detector. Defines the number of +# The configuration parameter of the TFH_RPS detector. Defines the number of # users that can be blocked per check. -DETECTOR_JA5H_RPS_BLOCK_USERS_PER_ITERATION=100 +DETECTOR_TFH_RPS_BLOCK_USERS_PER_ITERATION=100 -# The configuration parameter of the JA5H_TIME detector. Installs the default +# The configuration parameter of the TFH_TIME detector. Installs the default # accumulative time threshold. -DETECTOR_JA5H_TIME_DEFAULT_THRESHOLD=10 +DETECTOR_TFH_TIME_DEFAULT_THRESHOLD=10 -# The configuration parameter of the JA5H_TIME detector. Defines the difference between two user groups. -# If the last users group generates DETECTOR_JA5H_TIME_INTERSECTION_PERCENT times more traffic than the previous one, +# The configuration parameter of the TFH_TIME detector. Defines the difference between two user groups. +# If the last users group generates DETECTOR_TFH_TIME_INTERSECTION_PERCENT times more traffic than the previous one, # some of those users should be blocked. -DETECTOR_JA5H_TIME_INTERSECTION_PERCENT=10 +DETECTOR_TFH_TIME_INTERSECTION_PERCENT=10 -# The configuration parameter of the JA5H_TIME detector. Defines the number of +# The configuration parameter of the TFH_TIME detector. Defines the number of # users that can be blocked per check. -DETECTOR_JA5H_TIME_BLOCK_USERS_PER_ITERATION=100 +DETECTOR_TFH_TIME_BLOCK_USERS_PER_ITERATION=100 -# The configuration parameter of the JA5H_ERRORS detector. Installs the default error responses threshold. -DETECTOR_JA5H_ERRORS_DEFAULT_THRESHOLD=10 +# The configuration parameter of the TFH_ERRORS detector. Installs the default error responses threshold. +DETECTOR_TFH_ERRORS_DEFAULT_THRESHOLD=10 -# The configuration parameter of the JA5H_ERRORS detector. Defines the difference between two user groups. -# If the last users group generates DETECTOR_JA5H_ERRORS_INTERSECTION_PERCENT times more traffic than the previous one, +# The configuration parameter of the TFH_ERRORS detector. Defines the difference between two user groups. +# If the last users group generates DETECTOR_TFH_ERRORS_INTERSECTION_PERCENT times more traffic than the previous one, # some of those users should be blocked. -DETECTOR_JA5H_ERRORS_INTERSECTION_PERCENT=10 +DETECTOR_TFH_ERRORS_INTERSECTION_PERCENT=10 -# The configuration parameter of the JA5H_ERRORS detector. Defines the number of +# The configuration parameter of the TFH_ERRORS detector. Defines the number of # users that can be blocked per check. -DETECTOR_JA5H_ERRORS_BLOCK_USERS_PER_ITERATION=100 +DETECTOR_TFH_ERRORS_BLOCK_USERS_PER_ITERATION=100 -# The configuration parameter of the JA5H_ERRORS detector. Defines the list of whitelisted response +# The configuration parameter of the TFH_ERRORS detector. Defines the list of whitelisted response # status codes that should be ignored by the filters. -DETECTOR_JA5H_ERRORS_ALLOWED_STATUSES=[100,101,200,201,204,300,301,302,303,304,305,307,308,400,401,403] +DETECTOR_TFH_ERRORS_ALLOWED_STATUSES=[100,101,200,201,204,300,301,302,303,304,305,307,308,400,401,403] # The configuration parameter of the GEOIP detector. Installs the default RPS threshold. DETECTOR_GEOIP_RPS_DEFAULT_THRESHOLD=10 # The configuration parameter of the GEOIP detector. Defines the difference between two user groups. -# If the last users group generates DETECTOR_JA5_RPS_INTERSECTION_PERCENT times more traffic than the previous one, +# If the last users group generates DETECTOR_TF_RPS_INTERSECTION_PERCENT times more traffic than the previous one, # some of those users should be blocked. DETECTOR_GEOIP_INTERSECTION_PERCENT=10 diff --git a/tests/base.py b/tests/base.py index b6b4e29..4a6da9e 100644 --- a/tests/base.py +++ b/tests/base.py @@ -43,8 +43,8 @@ async def asyncSetUp(self): uri String, referer String, user_agent String, - ja5t UInt64, - ja5h UInt64, + tft UInt64, + tfh UInt64, dropped_events UInt64, PRIMARY KEY(timestamp) ); diff --git a/tests/test_blocker_ja5h.py b/tests/test_blocker_tfh.py similarity index 74% rename from tests/test_blocker_ja5h.py rename to tests/test_blocker_tfh.py index eb7eb1d..88843c8 100644 --- a/tests/test_blocker_ja5h.py +++ b/tests/test_blocker_tfh.py @@ -3,9 +3,9 @@ import pytest from blockers.base import PreparationError -from blockers.ja5h import Ja5hBlocker +from blockers.tfh import TFhBlocker from utils.datatypes import User -from utils.ja5_config import Ja5Config, Ja5Hash +from utils.tf_config import TFConfig, TFHash __author__ = "Tempesta Technologies, Inc." __copyright__ = "Copyright (C) 2023-2025 Tempesta Technologies, Inc." @@ -14,12 +14,12 @@ @pytest.fixture def config_path() -> str: - return "/tmp/test_ja5h_config" + return "/tmp/test_tfh_config" @pytest.fixture def blocker(config_path): - blocker = Ja5hBlocker(Ja5Config(config_path)) + blocker = TFhBlocker(TFConfig(config_path)) open(config_path, "w").close() yield blocker @@ -33,25 +33,25 @@ def test_load(blocker, config_path): users = blocker.load() assert len(users) == 2 - assert [item.ja5h for item in users.values()] == [["1111"], ["2222"]] + assert [item.tfh for item in users.values()] == [["1111"], ["2222"]] def test_block(blocker): - user = User(ja5h=["11111"]) + user = User(tfh=["11111"]) blocker.block(user) assert len(blocker.config.hashes) == 1 assert blocker.config.hashes["11111"].value == "11111" def test_release(blocker): - user = User(ja5h=["3333"]) - blocker.config.hashes["3333"] = Ja5Hash(value="3333", connections=0, packets=0) + user = User(tfh=["3333"]) + blocker.config.hashes["3333"] = TFHash(value="3333", connections=0, packets=0) blocker.release(user) assert len(blocker.config.hashes) == 0 def test_apply(blocker, config_path): - blocker.block(User(ja5h=["11111"])) + blocker.block(User(tfh=["11111"])) blocker.apply() with open(config_path, "r") as f: @@ -61,10 +61,10 @@ def test_apply(blocker, config_path): def test_info(blocker): - blocker.block(User(ja5h=["11111"])) + blocker.block(User(tfh=["11111"])) users = blocker.info() assert len(users) == 1 - assert users[0].ja5h == ["11111"] + assert users[0].tfh == ["11111"] def test_prepare_no_tempesta_service(blocker): diff --git a/tests/test_blocker_ja5t.py b/tests/test_blocker_tft.py similarity index 74% rename from tests/test_blocker_ja5t.py rename to tests/test_blocker_tft.py index 7977591..cfc71c3 100644 --- a/tests/test_blocker_ja5t.py +++ b/tests/test_blocker_tft.py @@ -3,9 +3,9 @@ import pytest from blockers.base import PreparationError -from blockers.ja5t import Ja5tBlocker +from blockers.tft import TFtBlocker from utils.datatypes import User -from utils.ja5_config import Ja5Config, Ja5Hash +from utils.tf_config import TFConfig, TFHash __author__ = "Tempesta Technologies, Inc." __copyright__ = "Copyright (C) 2023-2025 Tempesta Technologies, Inc." @@ -14,12 +14,12 @@ @pytest.fixture def config_path() -> str: - return "/tmp/test_ja5h_config" + return "/tmp/test_tfh_config" @pytest.fixture def blocker(config_path): - blocker = Ja5tBlocker(Ja5Config(config_path)) + blocker = TFtBlocker(TFConfig(config_path)) open(config_path, "w").close() yield blocker @@ -33,25 +33,25 @@ def test_load(blocker, config_path): users = blocker.load() assert len(users) == 2 - assert [item.ja5t for item in users.values()] == [["1111"], ["2222"]] + assert [item.tft for item in users.values()] == [["1111"], ["2222"]] def test_block(blocker): - user = User(ja5t=["11111"]) + user = User(tft=["11111"]) blocker.block(user) assert len(blocker.config.hashes) == 1 assert blocker.config.hashes["11111"].value == "11111" def test_release(blocker): - user = User(ja5t=["3333"]) - blocker.config.hashes["3333"] = Ja5Hash(value="3333", connections=0, packets=0) + user = User(tft=["3333"]) + blocker.config.hashes["3333"] = TFHash(value="3333", connections=0, packets=0) blocker.release(user) assert len(blocker.config.hashes) == 0 def test_apply(blocker, config_path): - blocker.block(User(ja5t=["11111"])) + blocker.block(User(tft=["11111"])) blocker.apply() with open(config_path, "r") as f: @@ -61,10 +61,10 @@ def test_apply(blocker, config_path): def test_info(blocker): - blocker.block(User(ja5t=["11111"])) + blocker.block(User(tft=["11111"])) users = blocker.info() assert len(users) == 1 - assert users[0].ja5t == ["11111"] + assert users[0].tft == ["11111"] def test_prepare_no_tempesta_service(blocker): diff --git a/tests/test_detector_base.py b/tests/test_detector_base.py index b66bcba..10d7575 100644 --- a/tests/test_detector_base.py +++ b/tests/test_detector_base.py @@ -1,11 +1,9 @@ -import unittest from decimal import Decimal from ipaddress import IPv4Address import pytest from detectors.base import BaseDetector -from utils.access_log import ClickhouseAccessLog from utils.datatypes import User __author__ = "Tempesta Technologies, Inc." diff --git a/tests/test_detector_ip_based.py b/tests/test_detector_ip_based.py index e0014e2..8b872d2 100644 --- a/tests/test_detector_ip_based.py +++ b/tests/test_detector_ip_based.py @@ -39,7 +39,7 @@ async def test_rps(access_log): current_time=1751535010, interval=5 ) assert users_before == [] - assert users_after == [User(ja5t=['d'], ja5h=['17'], ip=[IPv4Address("127.0.0.3")])] + assert users_after == [User(tft=['d'], tfh=['17'], ip=[IPv4Address("127.0.0.3")])] async def test_rps_with_user_agents(access_log): @@ -85,7 +85,7 @@ async def test_errors(access_log): current_time=1751535010, interval=5 ) assert users_before == [] - assert users_after == [User(ja5t=['d'], ja5h=['17'], ip=[IPv4Address("127.0.0.3")])] + assert users_after == [User(tft=['d'], tfh=['17'], ip=[IPv4Address("127.0.0.3")])] async def test_errors_with_user_agents(access_log): @@ -146,7 +146,7 @@ async def test_time(access_log): current_time=1751535010, interval=5 ) assert users_before == [] - assert users_after == [User(ja5t=['d'], ja5h=['17'], ip=[IPv4Address("127.0.0.3")])] + assert users_after == [User(tft=['d'], tfh=['17'], ip=[IPv4Address("127.0.0.3")])] async def test_time_with_user_agents(access_log): diff --git a/tests/test_detector_ja5h_based.py b/tests/test_detector_tfh_based.py similarity index 87% rename from tests/test_detector_ja5h_based.py rename to tests/test_detector_tfh_based.py index be31f83..f4107a6 100644 --- a/tests/test_detector_ja5h_based.py +++ b/tests/test_detector_tfh_based.py @@ -3,10 +3,10 @@ import pytest -from detectors.ja5h import ( - Ja5hAccumulativeTimeDetector, - Ja5hErrorRequestDetector, - Ja5hRPSDetector, +from detectors.tfh import ( + TFhAccumulativeTimeDetector, + TFhErrorRequestDetector, + TFhRPSDetector, ) from utils.datatypes import User @@ -30,7 +30,7 @@ async def data(access_log): async def test_rps(access_log): - detector = Ja5hRPSDetector( + detector = TFhRPSDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -40,7 +40,7 @@ async def test_rps(access_log): ) assert users_before == [] assert len(users_after) == 1 - assert users_after[0].ja5t == ['d'] + assert users_after[0].tft == ['d'] assert set(users_after[0].ip) == { IPv4Address("127.0.0.3"), IPv4Address("127.0.0.4"), @@ -49,7 +49,7 @@ async def test_rps(access_log): async def test_rps_with_user_agents(access_log): await access_log.user_agents_table_insert([["UserAgent"], ["UserAgent4"]]) - detector = Ja5hRPSDetector( + detector = TFhRPSDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -58,7 +58,7 @@ async def test_rps_with_user_agents(access_log): current_time=1751535010, interval=5 ) assert users_before == [] - assert users_after == [User(ja5t=['d'], ja5h=['17'], ip=[IPv4Address("127.0.0.3")])] + assert users_after == [User(tft=['d'], tfh=['17'], ip=[IPv4Address("127.0.0.3")])] async def test_rps_with_persistent_users(access_log): @@ -67,7 +67,7 @@ async def test_rps_with_persistent_users(access_log): ["127.0.0.3"], ] ) - detector = Ja5hRPSDetector( + detector = TFhRPSDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -80,7 +80,7 @@ async def test_rps_with_persistent_users(access_log): async def test_errors(access_log): - detector = Ja5hErrorRequestDetector( + detector = TFhErrorRequestDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -92,8 +92,8 @@ async def test_errors(access_log): assert users_before == [] assert users_after == [ User( - ja5t=['d'], - ja5h=['17'], + tft=['d'], + tfh=['17'], ip=[IPv4Address("127.0.0.4"), IPv4Address("127.0.0.3")], ) ] @@ -101,7 +101,7 @@ async def test_errors(access_log): async def test_errors_with_user_agents(access_log): await access_log.user_agents_table_insert([["UserAgent"], ["UserAgent3"]]) - detector = Ja5hErrorRequestDetector( + detector = TFhErrorRequestDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -112,7 +112,7 @@ async def test_errors_with_user_agents(access_log): ) assert users_before == [] assert len(users_after) == 1 - assert users_after[0].ja5t == ['d'] + assert users_after[0].tft == ['d'] assert set(users_after[0].ip) == { IPv4Address("127.0.0.3"), IPv4Address("127.0.0.4"), @@ -125,7 +125,7 @@ async def test_errors_with_persistent_users(access_log): ["127.0.0.3"], ] ) - detector = Ja5hErrorRequestDetector( + detector = TFhErrorRequestDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -139,7 +139,7 @@ async def test_errors_with_persistent_users(access_log): async def test_errors_forbidden_statuses(access_log): - detector = Ja5hErrorRequestDetector( + detector = TFhErrorRequestDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -153,7 +153,7 @@ async def test_errors_forbidden_statuses(access_log): async def test_time(access_log): - detector = Ja5hAccumulativeTimeDetector( + detector = TFhAccumulativeTimeDetector( access_log=access_log, default_threshold=Decimal("15"), intersection_percent=Decimal("10"), @@ -163,7 +163,7 @@ async def test_time(access_log): ) assert users_before == [] assert len(users_after) == 1 - assert users_after[0].ja5t == ['d'] + assert users_after[0].tft == ['d'] assert set(users_after[0].ip) == { IPv4Address("127.0.0.3"), IPv4Address("127.0.0.4"), @@ -172,7 +172,7 @@ async def test_time(access_log): async def test_time_with_user_agents(access_log): await access_log.user_agents_table_insert([["UserAgent"], ["UserAgent3"]]) - detector = Ja5hAccumulativeTimeDetector( + detector = TFhAccumulativeTimeDetector( access_log=access_log, default_threshold=Decimal("15"), intersection_percent=Decimal("10"), @@ -182,7 +182,7 @@ async def test_time_with_user_agents(access_log): ) assert users_before == [] assert len(users_after) == 1 - assert users_after[0].ja5t == ['d'] + assert users_after[0].tft == ['d'] assert set(users_after[0].ip) == { IPv4Address("127.0.0.3"), IPv4Address("127.0.0.4"), @@ -195,7 +195,7 @@ async def test_time_with_persistent_users(access_log): ["127.0.0.3"], ] ) - detector = Ja5hAccumulativeTimeDetector( + detector = TFhAccumulativeTimeDetector( access_log=access_log, default_threshold=Decimal("15"), intersection_percent=Decimal("10"), diff --git a/tests/test_detector_ja5t_based.py b/tests/test_detector_tft_based.py similarity index 87% rename from tests/test_detector_ja5t_based.py rename to tests/test_detector_tft_based.py index e4ae838..87cd3bb 100644 --- a/tests/test_detector_ja5t_based.py +++ b/tests/test_detector_tft_based.py @@ -3,10 +3,10 @@ import pytest -from detectors.ja5t import ( - Ja5tAccumulativeTimeDetector, - Ja5tErrorRequestDetector, - Ja5tRPSDetector, +from detectors.tft import ( + TFtAccumulativeTimeDetector, + TFtErrorRequestDetector, + TFtRPSDetector, ) from utils.datatypes import User @@ -30,7 +30,7 @@ async def data(access_log): async def test_rps(access_log): - detector = Ja5tRPSDetector( + detector = TFtRPSDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -40,7 +40,7 @@ async def test_rps(access_log): ) assert users_before == [] assert len(users_after) == 1 - assert users_after[0].ja5t == ['d'] + assert users_after[0].tft == ['d'] assert set(users_after[0].ip) == { IPv4Address("127.0.0.3"), IPv4Address("127.0.0.4"), @@ -49,7 +49,7 @@ async def test_rps(access_log): async def test_rps_with_user_agents(access_log): await access_log.user_agents_table_insert([["UserAgent"], ["UserAgent4"]]) - detector = Ja5tRPSDetector( + detector = TFtRPSDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -58,7 +58,7 @@ async def test_rps_with_user_agents(access_log): current_time=1751535010, interval=5 ) assert users_before == [] - assert users_after == [User(ja5t=['d'], ja5h=['17'], ip=[IPv4Address("127.0.0.3")])] + assert users_after == [User(tft=['d'], tfh=['17'], ip=[IPv4Address("127.0.0.3")])] async def test_rps_with_persistent_users(access_log): @@ -67,7 +67,7 @@ async def test_rps_with_persistent_users(access_log): ["127.0.0.3"], ] ) - detector = Ja5tRPSDetector( + detector = TFtRPSDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -80,7 +80,7 @@ async def test_rps_with_persistent_users(access_log): async def test_errors(access_log): - detector = Ja5tErrorRequestDetector( + detector = TFtErrorRequestDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -92,8 +92,8 @@ async def test_errors(access_log): assert users_before == [] assert users_after == [ User( - ja5t=['d'], - ja5h=['17'], + tft=['d'], + tfh=['17'], ip=[IPv4Address("127.0.0.4"), IPv4Address("127.0.0.3")], ) ] @@ -101,7 +101,7 @@ async def test_errors(access_log): async def test_errors_with_user_agents(access_log): await access_log.user_agents_table_insert([["UserAgent"], ["UserAgent3"]]) - detector = Ja5tErrorRequestDetector( + detector = TFtErrorRequestDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -112,7 +112,7 @@ async def test_errors_with_user_agents(access_log): ) assert users_before == [] assert len(users_after) == 1 - assert users_after[0].ja5t == ['d'] + assert users_after[0].tft == ['d'] assert set(users_after[0].ip) == { IPv4Address("127.0.0.3"), IPv4Address("127.0.0.4"), @@ -125,7 +125,7 @@ async def test_errors_with_persistent_users(access_log): ["127.0.0.3"], ] ) - detector = Ja5tErrorRequestDetector( + detector = TFtErrorRequestDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -139,7 +139,7 @@ async def test_errors_with_persistent_users(access_log): async def test_errors_forbidden_statuses(access_log): - detector = Ja5tErrorRequestDetector( + detector = TFtErrorRequestDetector( access_log=access_log, default_threshold=Decimal("2"), intersection_percent=Decimal("10"), @@ -153,7 +153,7 @@ async def test_errors_forbidden_statuses(access_log): async def test_time(access_log): - detector = Ja5tAccumulativeTimeDetector( + detector = TFtAccumulativeTimeDetector( access_log=access_log, default_threshold=Decimal("15"), intersection_percent=Decimal("10"), @@ -163,7 +163,7 @@ async def test_time(access_log): ) assert users_before == [] assert len(users_after) == 1 - assert users_after[0].ja5t == ['d'] + assert users_after[0].tft == ['d'] assert set(users_after[0].ip) == { IPv4Address("127.0.0.3"), IPv4Address("127.0.0.4"), @@ -172,7 +172,7 @@ async def test_time(access_log): async def test_time_with_user_agents(access_log): await access_log.user_agents_table_insert([["UserAgent"], ["UserAgent3"]]) - detector = Ja5tAccumulativeTimeDetector( + detector = TFtAccumulativeTimeDetector( access_log=access_log, default_threshold=Decimal("15"), intersection_percent=Decimal("10"), @@ -182,7 +182,7 @@ async def test_time_with_user_agents(access_log): ) assert users_before == [] assert len(users_after) == 1 - assert users_after[0].ja5t == ['d'] + assert users_after[0].tft == ['d'] assert set(users_after[0].ip) == { IPv4Address("127.0.0.3"), IPv4Address("127.0.0.4"), @@ -195,7 +195,7 @@ async def test_time_with_persistent_users(access_log): ["127.0.0.3"], ] ) - detector = Ja5tAccumulativeTimeDetector( + detector = TFtAccumulativeTimeDetector( access_log=access_log, default_threshold=Decimal("15"), intersection_percent=Decimal("10"), diff --git a/tests/test_lifespan_background_monitor_release_users.py b/tests/test_lifespan_background_monitor_release_users.py index 4313074..cb32189 100644 --- a/tests/test_lifespan_background_monitor_release_users.py +++ b/tests/test_lifespan_background_monitor_release_users.py @@ -47,10 +47,10 @@ def release(self, user: User): self.release_called += 1 def info(self) -> dict[int, User]: - return {2: User(ja5t=["4444"])} + return {2: User(tft=["4444"])} def load(self) -> dict[int, User]: - return {1: User(ja5t=["3333"])} + return {1: User(tft=["3333"])} class FrozenTimeAppContext(AppContext): def __init__(self, *args, **kwargs): @@ -71,13 +71,13 @@ def utc_now(self) -> int: @pytest.fixture def lifespan(app_context): - user1 = User(ja5t=["4441"], blocked_at=1751535000) + user1 = User(tft=["4441"], blocked_at=1751535000) app_context.blocked[hash(user1)] = user1 - user2 = User(ja5t=["4442"], blocked_at=1751535005) + user2 = User(tft=["4442"], blocked_at=1751535005) app_context.blocked[hash(user2)] = user2 - user3 = User(ja5t=["4443"], blocked_at=1751535009) + user3 = User(tft=["4443"], blocked_at=1751535009) app_context.blocked[hash(user3)] = user3 yield BackgroundReleaseUsersMonitoring(context=app_context) diff --git a/tests/test_lifespan_background_monitor_risky_users.py b/tests/test_lifespan_background_monitor_risky_users.py index 00a2aae..2612c5c 100644 --- a/tests/test_lifespan_background_monitor_risky_users.py +++ b/tests/test_lifespan_background_monitor_risky_users.py @@ -36,10 +36,10 @@ def release(self, user: User): return def info(self) -> dict[int, User]: - return {2: User(ja5t=["4444"])} + return {2: User(tft=["4444"])} def load(self) -> dict[int, User]: - return {1: User(ja5t=["3333"])} + return {1: User(tft=["3333"])} class FrozenTimeAppContext(AppContext): def __init__(self, *args, **kwargs): @@ -53,13 +53,13 @@ def utc_now(self) -> int: class FakeDetector(BaseDetector): groups = [ [ - User(ja5t=["111"], value=Decimal(1), ip=[IPv4Address("127.0.0.1")]), - User(ja5t=["112"], value=Decimal(2), ip=[IPv6Address("ff00::0")]), - User(ja5t=["113"], value=Decimal(3), ip=[IPv4Address("127.0.0.3")]), - User(ja5t=["115"], value=Decimal(3), ip=[IPv6Address("ff00::1")]), + User(tft=["111"], value=Decimal(1), ip=[IPv4Address("127.0.0.1")]), + User(tft=["112"], value=Decimal(2), ip=[IPv6Address("ff00::0")]), + User(tft=["113"], value=Decimal(3), ip=[IPv4Address("127.0.0.3")]), + User(tft=["115"], value=Decimal(3), ip=[IPv6Address("ff00::1")]), ], [ - User(ja5t=["112"], value=Decimal(2), ip=[IPv6Address("ff00::0")]), + User(tft=["112"], value=Decimal(2), ip=[IPv6Address("ff00::0")]), ], ] @@ -79,13 +79,13 @@ async def fetch_for_period(self, start_at: int, finish_at: int) -> list[User]: class FakeDetector2(FakeDetector): groups = [ [ - User(ja5t=["211"], value=Decimal(1), ip=[IPv4Address("127.0.0.1")]), - User(ja5t=["212"], value=Decimal(2), ip=[IPv6Address("ff00::0")]), - User(ja5t=["213"], value=Decimal(3), ip=[IPv4Address("127.0.0.3")]), - User(ja5t=["215"], value=Decimal(3), ip=[IPv6Address("ff00::1")]), + User(tft=["211"], value=Decimal(1), ip=[IPv4Address("127.0.0.1")]), + User(tft=["212"], value=Decimal(2), ip=[IPv6Address("ff00::0")]), + User(tft=["213"], value=Decimal(3), ip=[IPv4Address("127.0.0.3")]), + User(tft=["215"], value=Decimal(3), ip=[IPv6Address("ff00::1")]), ], [ - User(ja5t=["213"], value=Decimal(30), ip=[IPv4Address("127.0.0.3")]), + User(tft=["213"], value=Decimal(30), ip=[IPv4Address("127.0.0.3")]), ], ] @@ -142,6 +142,6 @@ async def test_block_users(app_context, lifespan): assert app_context.blockers["ipset"].block_called == 2 assert set(app_context.blocked.values()) == { - User(ja5t=["112"], value=Decimal(2), ip=[IPv6Address("ff00::0")]), - User(ja5t=["213"], value=Decimal(30), ip=[IPv4Address("127.0.0.3")]), + User(tft=["112"], value=Decimal(2), ip=[IPv6Address("ff00::0")]), + User(tft=["213"], value=Decimal(30), ip=[IPv4Address("127.0.0.3")]), } diff --git a/tests/test_lifespan_initialization.py b/tests/test_lifespan_initialization.py index ef9ef06..ae32bc4 100644 --- a/tests/test_lifespan_initialization.py +++ b/tests/test_lifespan_initialization.py @@ -62,15 +62,15 @@ def release(self, user: User): return def info(self) -> dict[int, User]: - return {2: User(ja5t=["4444"])} + return {2: User(tft=["4444"])} def load(self) -> dict[int, User]: - return {1: User(ja5t=["3333"])} + return {1: User(tft=["3333"])} class FakeBlocker2(FakeBlocker): @staticmethod def name() -> str: - return "ja5t" + return "tft" context = AppContext( blockers={ diff --git a/tests/test_lifespan_training_historical_mode.py b/tests/test_lifespan_training_historical_mode.py index f615f57..a352347 100644 --- a/tests/test_lifespan_training_historical_mode.py +++ b/tests/test_lifespan_training_historical_mode.py @@ -55,9 +55,9 @@ async def fetch_for_period(self, start_at: int, finish_at: int) -> list[User]: self.start_at = start_at self.finish_at = finish_at return [ - User(ja5t=["111"], value=Decimal(1)), - User(ja5t=["112"], value=Decimal(2)), - User(ja5t=["113"], value=Decimal(3)), + User(tft=["111"], value=Decimal(1)), + User(tft=["112"], value=Decimal(2)), + User(tft=["113"], value=Decimal(3)), ] class FakeDetector2(FakeDetector): @@ -69,9 +69,9 @@ async def fetch_for_period(self, start_at: int, finish_at: int) -> list[User]: self.start_at = start_at self.finish_at = finish_at return [ - User(ja5t=["111"], value=Decimal(100)), - User(ja5t=["112"], value=Decimal(200)), - User(ja5t=["113"], value=Decimal(300)), + User(tft=["111"], value=Decimal(100)), + User(tft=["112"], value=Decimal(200)), + User(tft=["113"], value=Decimal(300)), ] context = FrozenTimeAppContext( diff --git a/tests/test_lifespan_training_real_mode.py b/tests/test_lifespan_training_real_mode.py index 0b200a3..1bf50ce 100644 --- a/tests/test_lifespan_training_real_mode.py +++ b/tests/test_lifespan_training_real_mode.py @@ -46,9 +46,9 @@ async def fetch_for_period(self, start_at: int, finish_at: int) -> list[User]: self.start_at = start_at self.finish_at = finish_at return [ - User(ja5t=["111"], value=Decimal(1)), - User(ja5t=["112"], value=Decimal(2)), - User(ja5t=["113"], value=Decimal(3)), + User(tft=["111"], value=Decimal(1)), + User(tft=["112"], value=Decimal(2)), + User(tft=["113"], value=Decimal(3)), ] class FakeDetector2(FakeDetector): @@ -60,9 +60,9 @@ async def fetch_for_period(self, start_at: int, finish_at: int) -> list[User]: self.start_at = start_at self.finish_at = finish_at return [ - User(ja5t=["111"], value=Decimal(100)), - User(ja5t=["112"], value=Decimal(200)), - User(ja5t=["113"], value=Decimal(300)), + User(tft=["111"], value=Decimal(100)), + User(tft=["112"], value=Decimal(200)), + User(tft=["113"], value=Decimal(300)), ] class FrozenTimeAppContext(AppContext): diff --git a/tests/test_ja5_config.py b/tests/test_tf_config.py similarity index 80% rename from tests/test_ja5_config.py rename to tests/test_tf_config.py index af365cf..3966f2e 100644 --- a/tests/test_ja5_config.py +++ b/tests/test_tf_config.py @@ -1,7 +1,7 @@ import os import pytest -from utils.ja5_config import Ja5Config, Ja5Hash +from utils.tf_config import TFConfig, TFHash __author__ = "Tempesta Technologies, Inc." __copyright__ = "Copyright (C) 2023-2025 Tempesta Technologies, Inc." @@ -26,22 +26,22 @@ def config_path(): def test_config_does_not_exists(): with pytest.raises(FileNotFoundError): - config = Ja5Config("/tmp/non-existing.conf") + config = TFConfig("/tmp/non-existing.conf") config.verify_file() def test_load_hashes_from_file(config_path): - config = Ja5Config(config_path) + config = TFConfig(config_path) config.load() assert len(config.hashes) == 1 def test_dump_file(config_path): - config = Ja5Config(config_path) + config = TFConfig(config_path) config.load() - config.hashes = {"test": Ja5Hash(value="0", connections=1, packets=1)} + config.hashes = {"test": TFHash(value="0", connections=1, packets=1)} config.dump() with open(config_path) as f: @@ -51,11 +51,11 @@ def test_dump_file(config_path): def test_modification(config_path): - config = Ja5Config(config_path) + config = TFConfig(config_path) config.load() assert config.need_dump is False - config.add(Ja5Hash(value="100", connections=1, packets=2)) + config.add(TFHash(value="100", connections=1, packets=2)) assert config.need_dump == True config.dump() diff --git a/tests/test_user_agents.py b/tests/test_user_agents.py index 2869603..93999c8 100644 --- a/tests/test_user_agents.py +++ b/tests/test_user_agents.py @@ -1,9 +1,7 @@ import os -import unittest import pytest -from utils.access_log import ClickhouseAccessLog from utils.user_agents import UserAgentsManager __author__ = "Tempesta Technologies, Inc." diff --git a/utils/datatypes.py b/utils/datatypes.py index ec12bea..be9684d 100644 --- a/utils/datatypes.py +++ b/utils/datatypes.py @@ -18,15 +18,15 @@ class AverageStats: @dataclass class User: - ja5t: list[str] = None - ja5h: list[str] = None + tft: list[str] = None + tfh: list[str] = None ip: list[typing.Union[IPv6Address, IPv4Address]] = () value: Optional[Decimal] = None type: Optional[int] = None blocked_at: Optional[int] = None def __hash__(self): - return hash(f"ja5t={self.ja5t}/ja5h={self.ja5h}/ip={self.ip}") + return hash(f"tft={self.tft}/tfh={self.tfh}/ip={self.ip}") def __eq__(self, other): return hash(self) == hash(other) diff --git a/utils/ja5_config.py b/utils/tf_config.py similarity index 65% rename from utils/ja5_config.py rename to utils/tf_config.py index c2f86d9..709fd8a 100644 --- a/utils/ja5_config.py +++ b/utils/tf_config.py @@ -11,15 +11,15 @@ @dataclass -class Ja5Hash: +class TFHash: value: str connections: int packets: int -class Ja5Config: +class TFConfig: """ - Tempesta JA5 Config Manager + Tempesta TF Config Manager """ hash_pattern = re.compile( @@ -31,18 +31,18 @@ class Ja5Config: def __init__(self, file_path: str): self.file_path = file_path - self.hashes: Dict[str, Ja5Hash] = {} + self.hashes: Dict[str, TFHash] = {} self.need_dump: bool = False @staticmethod - def format_line(ja5_hash: Ja5Hash) -> str: + def format_line(tf_hash: TFHash) -> str: """ - Create a string representation of a JA5 hash. + Create a string representation of a TF hash. - :param ja5_hash: JA5 hash value. - :return: Formatted string representation of the JA5 hash. + :param tf_hash: TF hash value. + :return: Formatted string representation of the TF hash. """ - return f"hash {ja5_hash.value} {ja5_hash.connections} {ja5_hash.packets};\n" + return f"hash {tf_hash.value} {tf_hash.connections} {tf_hash.packets};\n" def verify_file(self): """ @@ -60,7 +60,7 @@ def verify_file(self): def load(self): """ - Parse the JA5 configuration file and store the loaded hashes. + Parse the TF configuration file and store the loaded hashes. """ with open(self.file_path, "r") as f: for line in f.readlines(): @@ -71,7 +71,7 @@ def load(self): continue hash_value = result.group("hash") - self.hashes[hash_value] = Ja5Hash( + self.hashes[hash_value] = TFHash( value=result.group("hash"), connections=result.group("connections"), packets=result.group("packets"), @@ -79,7 +79,7 @@ def load(self): def dump(self): """ - Dump the local storage of JA5 hashes into the configuration file. + Dump the local storage of TF hashes into the configuration file. """ with open(self.file_path, "w") as f: for value in self.hashes.values(): @@ -87,29 +87,29 @@ def dump(self): self.need_dump = False - def exists(self, ja5_hash: str) -> bool: + def exists(self, tf_hash: str) -> bool: """ - Check if a JA5 hash exists in local storage. + Check if a TF hash exists in local storage. - :param ja5_hash: JA5 hash value. + :param tf_hash: TF hash value. :return: True if the hash exists, False otherwise. """ - return ja5_hash in self.hashes + return tf_hash in self.hashes - def add(self, ja5_hash: Ja5Hash): + def add(self, tf_hash: TFHash): """ - Add a new JA5 hash to local storage. + Add a new TF hash to local storage. - :param ja5_hash: JA5 hash value. + :param tf_hash: TF hash value. """ - self.hashes[ja5_hash.value] = ja5_hash + self.hashes[tf_hash.value] = tf_hash self.need_dump = True - def remove(self, ja5_hash: str): + def remove(self, tf_hash: str): """ - Remove a JA5 hash from local storage. + Remove a TF hash from local storage. - :param ja5_hash: JA5 hash value. + :param tf_hash: TF hash value. """ - self.hashes.pop(ja5_hash) + self.hashes.pop(tf_hash) self.need_dump = True