Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Tempesta WebShield

Block users by JA5T, JA5H, or IP based on Tempesta FW access
Block users by TFT, TFH, or IP based on Tempesta FW access
logs stored in the ClickHouse database.

[**WIKI**](https://tempesta-tech.com/knowledge-base/Bot-Protection/)
Expand Down Expand Up @@ -58,19 +58,19 @@ isort .
# How to block

### Prepare Tempesta FW config
It's useful to define separate directories for different groups of JA5 hashes
It's useful to define separate directories for different groups of TF hashes
in the Tempesta FW configuration file (/etc/tempesta/tempesta_fw.conf).
```nginx
ja5t {
!include /etc/tempesta/ja5t/
tft {
!include /etc/tempesta/tft/
}
ja5h {
!include /etc/tempesta/ja5h/
tfh {
!include /etc/tempesta/tfh/
}
```
Then add 2 files
- /etc/tempesta/ja5t/blocked.conf
- /etc/tempesta/ja5h/blocked.conf
- /etc/tempesta/tft/blocked.conf
- /etc/tempesta/tfh/blocked.conf

These files should be used by default by the WebShield
to add new blocking hashes.
78 changes: 39 additions & 39 deletions app.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,18 @@
IPErrorRequestDetector,
IPRPSDetector,
)
from detectors.ja5t import (
Ja5tAccumulativeTimeDetector,
Ja5tErrorRequestDetector,
Ja5tRPSDetector,
from detectors.tft import (
TFtAccumulativeTimeDetector,
TFtErrorRequestDetector,
TFtRPSDetector,
)
from detectors.ja5h import (
Ja5hAccumulativeTimeDetector,
Ja5hErrorRequestDetector,
Ja5hRPSDetector,
from detectors.tfh import (
TFhAccumulativeTimeDetector,
TFhErrorRequestDetector,
TFhRPSDetector,
)
from utils.access_log import ClickhouseAccessLog
from utils.ja5_config import Ja5Config
from utils.tf_config import TFConfig
from utils.logger import logger
from utils.user_agents import UserAgentsManager

Expand All @@ -50,13 +50,13 @@
)
context = AppContext(
blockers={
blockers.Ja5tBlocker.name(): blockers.Ja5tBlocker(
config=Ja5Config(file_path=app_config.path_to_ja5t_config),
blockers.TFtBlocker.name(): blockers.TFtBlocker(
config=TFConfig(file_path=app_config.path_to_tft_config),
tempesta_executable_path=app_config.tempesta_executable_path,
tempesta_config_path=app_config.tempesta_config_path,
),
blockers.Ja5hBlocker.name(): blockers.Ja5hBlocker(
config=Ja5Config(file_path=app_config.path_to_ja5h_config),
blockers.TFhBlocker.name(): blockers.TFhBlocker(
config=TFConfig(file_path=app_config.path_to_tfh_config),
tempesta_executable_path=app_config.tempesta_executable_path,
tempesta_config_path=app_config.tempesta_config_path,
),
Expand Down Expand Up @@ -87,43 +87,43 @@
block_users_per_iteration=app_config.detector_ip_errors_block_users_per_iteration,
allowed_statues=app_config.detector_ip_errors_allowed_statuses,
),
Ja5tRPSDetector.name(): Ja5tRPSDetector(
TFtRPSDetector.name(): TFtRPSDetector(
access_log=clickhouse_client,
default_threshold=app_config.detector_ja5t_rps_default_threshold,
intersection_percent=app_config.detector_ja5t_rps_intersection_percent,
block_users_per_iteration=app_config.detector_ja5t_rps_block_users_per_iteration,
default_threshold=app_config.detector_tft_rps_default_threshold,
intersection_percent=app_config.detector_tft_rps_intersection_percent,
block_users_per_iteration=app_config.detector_tft_rps_block_users_per_iteration,
),
Ja5tAccumulativeTimeDetector.name(): Ja5tAccumulativeTimeDetector(
TFtAccumulativeTimeDetector.name(): TFtAccumulativeTimeDetector(
access_log=clickhouse_client,
default_threshold=app_config.detector_ja5t_time_default_threshold,
intersection_percent=app_config.detector_ja5t_time_intersection_percent,
block_users_per_iteration=app_config.detector_ja5t_time_block_users_per_iteration,
default_threshold=app_config.detector_tft_time_default_threshold,
intersection_percent=app_config.detector_tft_time_intersection_percent,
block_users_per_iteration=app_config.detector_tft_time_block_users_per_iteration,
),
Ja5tErrorRequestDetector.name(): Ja5tErrorRequestDetector(
TFtErrorRequestDetector.name(): TFtErrorRequestDetector(
access_log=clickhouse_client,
default_threshold=app_config.detector_ja5t_errors_default_threshold,
intersection_percent=app_config.detector_ja5t_errors_intersection_percent,
block_users_per_iteration=app_config.detector_ja5t_errors_block_users_per_iteration,
allowed_statues=app_config.detector_ja5t_errors_allowed_statuses,
default_threshold=app_config.detector_tft_errors_default_threshold,
intersection_percent=app_config.detector_tft_errors_intersection_percent,
block_users_per_iteration=app_config.detector_tft_errors_block_users_per_iteration,
allowed_statues=app_config.detector_tft_errors_allowed_statuses,
),
Ja5hRPSDetector.name(): Ja5hRPSDetector(
TFhRPSDetector.name(): TFhRPSDetector(
access_log=clickhouse_client,
default_threshold=app_config.detector_ja5h_rps_default_threshold,
intersection_percent=app_config.detector_ja5h_rps_intersection_percent,
block_users_per_iteration=app_config.detector_ja5h_rps_block_users_per_iteration,
default_threshold=app_config.detector_tfh_rps_default_threshold,
intersection_percent=app_config.detector_tfh_rps_intersection_percent,
block_users_per_iteration=app_config.detector_tfh_rps_block_users_per_iteration,
),
Ja5hAccumulativeTimeDetector.name(): Ja5hAccumulativeTimeDetector(
TFhAccumulativeTimeDetector.name(): TFhAccumulativeTimeDetector(
access_log=clickhouse_client,
default_threshold=app_config.detector_ja5h_time_default_threshold,
intersection_percent=app_config.detector_ja5h_time_intersection_percent,
block_users_per_iteration=app_config.detector_ja5h_time_block_users_per_iteration,
default_threshold=app_config.detector_tfh_time_default_threshold,
intersection_percent=app_config.detector_tfh_time_intersection_percent,
block_users_per_iteration=app_config.detector_tfh_time_block_users_per_iteration,
),
Ja5hErrorRequestDetector.name(): Ja5tErrorRequestDetector(
TFhErrorRequestDetector.name(): TFtErrorRequestDetector(
access_log=clickhouse_client,
default_threshold=app_config.detector_ja5h_errors_default_threshold,
intersection_percent=app_config.detector_ja5h_errors_intersection_percent,
block_users_per_iteration=app_config.detector_ja5h_errors_block_users_per_iteration,
allowed_statues=app_config.detector_ja5h_errors_allowed_statuses,
default_threshold=app_config.detector_tfh_errors_default_threshold,
intersection_percent=app_config.detector_tfh_errors_intersection_percent,
block_users_per_iteration=app_config.detector_tfh_errors_block_users_per_iteration,
allowed_statues=app_config.detector_tfh_errors_allowed_statuses,
),
GeoIPDetector.name(): GeoIPDetector(
access_log=clickhouse_client,
Expand Down
4 changes: 2 additions & 2 deletions blockers/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from .ipset import IpSetBlocker
from .ja5h import Ja5hBlocker
from .ja5t import Ja5tBlocker
from .tfh import TFhBlocker
from .tft import TFtBlocker
from .nft import NFTBlocker

__author__ = "Tempesta Technologies, Inc."
Expand Down
2 changes: 1 addition & 1 deletion blockers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def release(self, user: User):
def apply(self):
"""
Apply blocking rules. Some blockers may block immediately
without calling this method, but others — like ja5t — apply
without calling this method, but others — like tft — apply
rules only after multiple config changes.
"""

Expand Down
22 changes: 11 additions & 11 deletions blockers/ja5h.py → blockers/tfh.py
Original file line number Diff line number Diff line change
@@ -1,44 +1,44 @@
import time

from blockers.ja5t import Ja5tBlocker
from blockers.tft import TFtBlocker
from utils.datatypes import User
from utils.ja5_config import Ja5Hash
from utils.tf_config import TFHash
from utils.logger import logger

__author__ = "Tempesta Technologies, Inc."
__copyright__ = "Copyright (C) 2023-2025 Tempesta Technologies, Inc."
__license__ = "GPL2"


class Ja5hBlocker(Ja5tBlocker):
class TFhBlocker(TFtBlocker):

@staticmethod
def name() -> str:
return "ja5h"
return "tfh"

def load(self) -> dict[int, User]:
self.config.load()
current_time = int(time.time())
result = dict()

for hash_value in self.config.hashes:
user = User(ja5h=[hash_value], blocked_at=current_time)
user = User(tfh=[hash_value], blocked_at=current_time)
result[hash(user)] = user

return result

def block(self, user: User):
if self.config.exists(user.ja5h[0]):
if self.config.exists(user.tfh[0]):
return None

self.config.add(Ja5Hash(value=user.ja5h[0], packets=0, connections=0))
logger.warning(f"Blocked user {user} by ja5h")
self.config.add(TFHash(value=user.tfh[0], packets=0, connections=0))
logger.warning(f"Blocked user {user} by tfh")

def release(self, user: User):
if not self.config.exists(user.ja5h[0]):
if not self.config.exists(user.tfh[0]):
return None

self.config.remove(user.ja5h[0])
self.config.remove(user.tfh[0])

def info(self) -> list[User]:
return [User(ja5h=[ja5_hash.value]) for ja5_hash in self.config.hashes.values()]
return [User(tfh=[tf_hash.value]) for tf_hash in self.config.hashes.values()]
20 changes: 10 additions & 10 deletions blockers/ja5t.py → blockers/tft.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from blockers.base import BaseBlocker, PreparationError
from utils.datatypes import User
from utils.ja5_config import Ja5Config, Ja5Hash
from utils.tf_config import TFConfig, TFHash
from utils.logger import logger
from utils.shell import run_in_shell

Expand All @@ -12,10 +12,10 @@
__license__ = "GPL2"


class Ja5tBlocker(BaseBlocker):
class TFtBlocker(BaseBlocker):
def __init__(
self,
config: Ja5Config,
config: TFConfig,
tempesta_executable_path: str = None,
tempesta_config_path: str = None,
):
Expand All @@ -25,7 +25,7 @@ def __init__(

@staticmethod
def name() -> str:
return "ja5t"
return "tft"

def __tempesta_app_exists(self) -> bool:
if self.tempesta_executable_path and os.path.isfile(
Expand Down Expand Up @@ -53,21 +53,21 @@ def load(self) -> dict[int, User]:
result = dict()

for hash_value in self.config.hashes:
user = User(ja5t=[hash_value], blocked_at=current_time)
user = User(tft=[hash_value], blocked_at=current_time)
result[hash(user)] = user

return result

def block(self, user: User):
for hash_value in user.ja5t:
for hash_value in user.tft:
if self.config.exists(hash_value):
continue

self.config.add(Ja5Hash(value=hash_value, packets=0, connections=0))
logger.warning(f"Blocked user {user} by ja5t")
self.config.add(TFHash(value=hash_value, packets=0, connections=0))
logger.warning(f"Blocked user {user} by tft")

def release(self, user: User):
for hash_value in user.ja5t:
for hash_value in user.tft:
if not self.config.exists(hash_value):
continue

Expand All @@ -93,4 +93,4 @@ def apply(self):
)

def info(self) -> list[User]:
return [User(ja5t=[ja5_hash.value]) for ja5_hash in self.config.hashes.values()]
return [User(tft=[tf_hash.value]) for tf_hash in self.config.hashes.values()]
60 changes: 30 additions & 30 deletions config.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@


class AppConfig(BaseSettings):
path_to_ja5t_config: str = "/etc/tempesta/ja5t/blocked.conf"
path_to_ja5h_config: str = "/etc/tempesta/ja5h/blocked.conf"
path_to_tft_config: str = "/etc/tempesta/tft/blocked.conf"
path_to_tfh_config: str = "/etc/tempesta/tfh/blocked.conf"

clickhouse_host: str = "192.168.0.104"
clickhouse_port: int = 8123
Expand All @@ -28,17 +28,17 @@ class AppConfig(BaseSettings):
"ip_rps",
"ip_time",
"ip_errors",
"ja5t_rps",
"ja5t_time",
"ja5t_errors",
"ja5h_rps",
"ja5h_time",
"ja5h_errors",
"tft_rps",
"tft_time",
"tft_errors",
"tfh_rps",
"tfh_time",
"tfh_errors",
"geoip",
]
] = {"ja5t_rps", "ja5t_time", "ja5t_errors"}
] = {"tft_rps", "tft_time", "tft_errors"}

blocking_types: set[Literal["ja5t", "ja5h", "ipset", "nftables"]] = {"ja5t"}
blocking_types: set[Literal["tft", "tfh", "ipset", "nftables"]] = {"tft"}
blocking_window_duration_sec: int = 10
blocking_ipset_name: str = "tempesta_blocked_ips"
blocking_time_min: int = 60
Expand Down Expand Up @@ -77,18 +77,18 @@ class AppConfig(BaseSettings):
403,
]

detector_ja5t_rps_default_threshold: Decimal = Decimal(10)
detector_ja5t_rps_intersection_percent: Decimal = Decimal(10)
detector_ja5t_rps_block_users_per_iteration: Decimal = Decimal(10)
detector_tft_rps_default_threshold: Decimal = Decimal(10)
detector_tft_rps_intersection_percent: Decimal = Decimal(10)
detector_tft_rps_block_users_per_iteration: Decimal = Decimal(10)

detector_ja5t_time_default_threshold: Decimal = Decimal(10)
detector_ja5t_time_intersection_percent: Decimal = Decimal(10)
detector_ja5t_time_block_users_per_iteration: Decimal = Decimal(10)
detector_tft_time_default_threshold: Decimal = Decimal(10)
detector_tft_time_intersection_percent: Decimal = Decimal(10)
detector_tft_time_block_users_per_iteration: Decimal = Decimal(10)

detector_ja5t_errors_default_threshold: Decimal = Decimal(10)
detector_ja5t_errors_intersection_percent: Decimal = Decimal(10)
detector_ja5t_errors_block_users_per_iteration: Decimal = Decimal(10)
detector_ja5t_errors_allowed_statuses: list[int] = [
detector_tft_errors_default_threshold: Decimal = Decimal(10)
detector_tft_errors_intersection_percent: Decimal = Decimal(10)
detector_tft_errors_block_users_per_iteration: Decimal = Decimal(10)
detector_tft_errors_allowed_statuses: list[int] = [
100,
101,
200,
Expand All @@ -107,18 +107,18 @@ class AppConfig(BaseSettings):
403,
]

detector_ja5h_rps_default_threshold: Decimal = Decimal(10)
detector_ja5h_rps_intersection_percent: Decimal = Decimal(10)
detector_ja5h_rps_block_users_per_iteration: Decimal = Decimal(10)
detector_tfh_rps_default_threshold: Decimal = Decimal(10)
detector_tfh_rps_intersection_percent: Decimal = Decimal(10)
detector_tfh_rps_block_users_per_iteration: Decimal = Decimal(10)

detector_ja5h_time_default_threshold: Decimal = Decimal(10)
detector_ja5h_time_intersection_percent: Decimal = Decimal(10)
detector_ja5h_time_block_users_per_iteration: Decimal = Decimal(10)
detector_tfh_time_default_threshold: Decimal = Decimal(10)
detector_tfh_time_intersection_percent: Decimal = Decimal(10)
detector_tfh_time_block_users_per_iteration: Decimal = Decimal(10)

detector_ja5h_errors_default_threshold: Decimal = Decimal(10)
detector_ja5h_errors_intersection_percent: Decimal = Decimal(10)
detector_ja5h_errors_block_users_per_iteration: Decimal = Decimal(10)
detector_ja5h_errors_allowed_statuses: list[int] = [
detector_tfh_errors_default_threshold: Decimal = Decimal(10)
detector_tfh_errors_intersection_percent: Decimal = Decimal(10)
detector_tfh_errors_block_users_per_iteration: Decimal = Decimal(10)
detector_tfh_errors_allowed_statuses: list[int] = [
100,
101,
200,
Expand Down
Loading