1.4.1

Author: PadishahIII

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "secretscraper"
-version = "1.4"
+version = "1.4.1"
 description = "SecretScraper is a web scraper tool that can scrape the content through target websites and extract secret information via regular expression."
 readme = "README.md"
 authors = ["Padishah <[email protected]>"]

pyproject.toml

@@ -81,9 +81,9 @@ aiohttp==3.9.5 ; python_version >= "3.9" and python_version < "4.0" \
 aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "4.0" \
     --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \
     --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17
-anyio==4.3.0 ; python_version >= "3.9" and python_version < "4.0" \
-    --hash=sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8 \
-    --hash=sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6
+anyio==4.4.0 ; python_version >= "3.9" and python_version < "4.0" \
+    --hash=sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94 \
+    --hash=sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7
 async-timeout==4.0.3 ; python_version >= "3.9" and python_version < "3.11" \
     --hash=sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f \
     --hash=sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028
@@ -96,9 +96,9 @@ beautifulsoup4==4.12.3 ; python_version >= "3.9" and python_version < "4.0" \
 bs4==0.0.2 ; python_version >= "3.9" and python_version < "4.0" \
     --hash=sha256:a48685c58f50fe127722417bae83fe6badf500d54b55f7e39ffe43b798653925 \
     --hash=sha256:abf8742c0805ef7f662dce4b51cca104cffe52b835238afc169142ab9b3fbccc
-certifi==2024.2.2 ; python_version >= "3.9" and python_version < "4.0" \
-    --hash=sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f \
-    --hash=sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1
+certifi==2024.6.2 ; python_version >= "3.9" and python_version < "4.0" \
+    --hash=sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516 \
+    --hash=sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56
 charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "4.0" \
     --hash=sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027 \
     --hash=sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087 \
@@ -417,9 +417,9 @@ multidict==6.0.5 ; python_version >= "3.9" and python_version < "4.0" \
 requests-file==2.1.0 ; python_version >= "3.9" and python_version < "4.0" \
     --hash=sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658 \
     --hash=sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c
-requests==2.32.2 ; python_version >= "3.9" and python_version < "4.0" \
-    --hash=sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289 \
-    --hash=sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c
+requests==2.32.3 ; python_version >= "3.9" and python_version < "4.0" \
+    --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \
+    --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
 sniffio==1.3.1 ; python_version >= "3.9" and python_version < "4.0" \
     --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \
     --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc
@@ -432,9 +432,9 @@ soupsieve==2.5 ; python_version >= "3.9" and python_version < "4.0" \
 tldextract==5.1.2 ; python_version >= "3.9" and python_version < "4.0" \
     --hash=sha256:4dfc4c277b6b97fa053899fcdb892d2dc27295851ab5fac4e07797b6a21b2e46 \
     --hash=sha256:c9e17f756f05afb5abac04fe8f766e7e70f9fe387adb1859f0f52408ee060200
-typing-extensions==4.12.0 ; python_version >= "3.9" and python_version < "3.11" \
-    --hash=sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8 \
-    --hash=sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594
+typing-extensions==4.12.1 ; python_version >= "3.9" and python_version < "3.11" \
+    --hash=sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a \
+    --hash=sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1
 urllib3==2.2.1 ; python_version >= "3.9" and python_version < "4.0" \
     --hash=sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d \
     --hash=sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19

requirements.txt

@@ -1,3 +1,3 @@
 """SecretScraper"""
 
-__version__ = "1.4"
+__version__ = "1.4.1"

src/secretscraper/__init__.py

@@ -364,7 +364,13 @@ def init(self) -> FileScanner:
 
         # Read rules from config file
         rules: typing.Dict[str, str] = read_rules_from_setting(self.settings)
-        handler = get_regex_handler(rules)
+        handler_type = self.settings.get("handler_type", "re")
+        if handler_type == "hyperscan":
+            handler = get_regex_handler(rules)
+            print_config(f"Using regex handler: Hyperscan")
+        else:
+            handler = get_regex_handler(rules, type_="regex", use_groups=True)
+            print_config(f"Using regex handler: Re")
 
         # Get all files from directory
         base: typing.Optional[pathlib.Path] = self.custom_settings.get('local', None)

src/secretscraper/facade.py

@@ -1,6 +1,7 @@
 """Common utility functions."""
-
+import os
 import re
+import sys
 import typing
 from collections import namedtuple
 from pathlib import Path
@@ -120,7 +121,11 @@ def get_response_title(response: requests.Response) -> str:
 
 def start_local_test_http_server(host: str, port: int, server_dir: Path = None) -> tuple[
     Thread, http.server.HTTPServer]:
-    """Start local test server"""
+    """Start local test server
+
+    """
+    # if sys.platform.startswith("win"):
+    #     return None, None
 
     if server_dir is None:
         DIR = str(
@@ -133,7 +138,10 @@ class Handler(http.server.SimpleHTTPRequestHandler):
         def __init__(self, *args, **kwargs):
             super().__init__(*args, directory=DIR, **kwargs)
 
-    httpd = http.server.HTTPServer((host, port), Handler)
-    thread = Thread(target=httpd.serve_forever)
-    thread.start()
-    return thread, httpd
+    try:
+        httpd = http.server.HTTPServer((host, port), Handler)
+        thread = Thread(target=httpd.serve_forever)
+        thread.start()
+        return thread, httpd
+    except OSError:
+        return None, None

src/secretscraper/util.py

@@ -242,7 +242,8 @@ def test_normal_run(clicker: CliRunner, invoke_args: typing.List[str]):
             click.echo(result.output, file=f)
         print(result)
     finally:
-        httpd.shutdown()
+        if httpd is not None:
+            httpd.shutdown()
 
 
 # @pytest.mark.parametrize( # TODO: cannot copy file in github actions

tests/test_facade.py

@@ -1,6 +1,5 @@
 from secretscraper.util import read_rules_from_setting, start_local_test_http_server
 import requests
-from signal import pthread_kill, SIGKILL
 
 from . import settings
 
@@ -18,5 +17,6 @@ def test_start_local_test_http_server():
     except AssertionError as e:
         raise e
     finally:
-        httpd.shutdown()
-        print(1)
+        if httpd is not None:
+            httpd.shutdown()
+        # print(1)