diff --git a/Scripts/deduplicate.py b/Scripts/deduplicate.py index 4f1a5a1..3dddf34 100755 --- a/Scripts/deduplicate.py +++ b/Scripts/deduplicate.py @@ -19,6 +19,8 @@ from common import is_valid_domain, write_lines +HEADER_PREFIXES = ("! ", "#", "[", ";") + @dataclass(slots=True) class Stats: @@ -34,7 +36,7 @@ def compression_ratio(self) -> float: def is_header(line: str) -> bool: """Check if line is a header/metadata line""" - return line.startswith(("! ", "#", "[", ";")) or not line + return line.startswith(HEADER_PREFIXES) or not line def is_valid_rule(line: str) -> bool: diff --git a/Scripts/update-lists.py b/Scripts/update-lists.py index 6bc4d15..29660fe 100644 --- a/Scripts/update-lists.py +++ b/Scripts/update-lists.py @@ -32,6 +32,7 @@ SOURCES_CONFIG: Final[str] = "lists/sources-urls.json" DEFAULT_OUTPUT: Final[str] = "lists/sources" METADATA_FILE: Final[str] = "lists/sources-metadata.json" +HEADER_PREFIXES: Final[tuple[str, ...]] = ("! ", "#", "[") TIMEOUT: Final[int] = 60 CHUNK_SIZE: Final[int] = 65536 MAX_CONCURRENT: Final[int] = 10 @@ -90,7 +91,7 @@ def count_rules(content: str) -> int: return sum( 1 for line in io.StringIO(content) - if (stripped := line.strip()) and not stripped.startswith(("! ", "#", "[")) + if (stripped := line.strip()) and not stripped.startswith(HEADER_PREFIXES) )