diff --git a/ddtrace/appsec/_iast/_ast/iastpatch.c b/ddtrace/appsec/_iast/_ast/iastpatch.c index 3ab9921e973..af0d5d07b3b 100644 --- a/ddtrace/appsec/_iast/_ast/iastpatch.c +++ b/ddtrace/appsec/_iast/_ast/iastpatch.c @@ -17,373 +17,216 @@ static char** cached_packages = NULL; static size_t cached_packages_count = 0; /* Static Lists */ -static const char* static_allowlist[] = { "jinja2.", "pygments.", "multipart.", "sqlalchemy.", "python_multipart.", - "attrs.", "jsonschema.", "s3fs.", "mysql.", "pymysql.", - "markupsafe" }; +static const char* static_allowlist[] = { + "jinja2.", "pygments.", "multipart.", "sqlalchemy.", "python_multipart.", "attrs.", + "jsonschema.", "s3fs.", "mysql.", "pymysql.", "markupsafe.", "werkzeug.utils." +}; static const size_t static_allowlist_count = sizeof(static_allowlist) / sizeof(static_allowlist[0]); -static const char* static_denylist[] = { "django.apps.config.", - "django.apps.registry.", - "django.conf.", - "django.contrib.admin.actions.", - "django.contrib.admin.admin.", - "django.contrib.admin.apps.", - "django.contrib.admin.checks.", - "django.contrib.admin.decorators.", - "django.contrib.admin.exceptions.", - "django.contrib.admin.helpers.", - "django.contrib.admin.image_formats.", - "django.contrib.admin.options.", - "django.contrib.admin.sites.", - "django.contrib.admin.templatetags.", - "django.contrib.admin.views.autocomplete.", - "django.contrib.admin.views.decorators.", - "django.contrib.admin.views.main.", - "django.contrib.admin.wagtail_hooks.", - "django.contrib.admin.widgets.", - "django.contrib.admindocs.utils.", - "django.contrib.admindocs.views.", - "django.contrib.auth.admin.", - "django.contrib.auth.apps.", - "django.contrib.auth.backends.", - "django.contrib.auth.base_user.", - "django.contrib.auth.checks.", - "django.contrib.auth.context_processors.", - "django.contrib.auth.decorators.", - "django.contrib.auth.hashers.", - "django.contrib.auth.image_formats.", - "django.contrib.auth.management.", - "django.contrib.auth.middleware.", - "django.contrib.auth.password_validation.", - "django.contrib.auth.signals.", - "django.contrib.auth.templatetags.", - "django.contrib.auth.validators.", - "django.contrib.auth.wagtail_hooks.", - "django.contrib.contenttypes.admin.", - "django.contrib.contenttypes.apps.", - "django.contrib.contenttypes.checks.", - "django.contrib.contenttypes.fields.", - "django.contrib.contenttypes.forms.", - "django.contrib.contenttypes.image_formats.", - "django.contrib.contenttypes.management.", - "django.contrib.contenttypes.models.", - "django.contrib.contenttypes.templatetags.", - "django.contrib.contenttypes.views.", - "django.contrib.contenttypes.wagtail_hooks.", - "django.contrib.humanize.templatetags.", - "django.contrib.messages.admin.", - "django.contrib.messages.api.", - "django.contrib.messages.apps.", - "django.contrib.messages.constants.", - "django.contrib.messages.context_processors.", - "django.contrib.messages.image_formats.", - "django.contrib.messages.middleware.", - "django.contrib.messages.storage.", - "django.contrib.messages.templatetags.", - "django.contrib.messages.utils.", - "django.contrib.messages.wagtail_hooks.", - "django.contrib.sessions.admin.", - "django.contrib.sessions.apps.", - "django.contrib.sessions.backends.", - "django.contrib.sessions.base_session.", - "django.contrib.sessions.exceptions.", - "django.contrib.sessions.image_formats.", - "django.contrib.sessions.middleware.", - "django.contrib.sessions.templatetags.", - "django.contrib.sessions.wagtail_hooks.", - "django.contrib.sites.", - "django.contrib.staticfiles.admin.", - "django.contrib.staticfiles.apps.", - "django.contrib.staticfiles.checks.", - "django.contrib.staticfiles.finders.", - "django.contrib.staticfiles.image_formats.", - "django.contrib.staticfiles.models.", - "django.contrib.staticfiles.storage.", - "django.contrib.staticfiles.templatetags.", - "django.contrib.staticfiles.utils.", - "django.contrib.staticfiles.wagtail_hooks.", - "django.core.cache.backends.", - "django.core.cache.utils.", - "django.core.checks.async_checks.", - "django.core.checks.caches.", - "django.core.checks.compatibility.", - "django.core.checks.compatibility.django_4_0.", - "django.core.checks.database.", - "django.core.checks.files.", - "django.core.checks.messages.", - "django.core.checks.model_checks.", - "django.core.checks.registry.", - "django.core.checks.security.", - "django.core.checks.security.base.", - "django.core.checks.security.csrf.", - "django.core.checks.security.sessions.", - "django.core.checks.templates.", - "django.core.checks.translation.", - "django.core.checks.urls", - "django.core.exceptions.", - "django.core.mail.", - "django.core.management.base.", - "django.core.management.color.", - "django.core.management.sql.", - "django.core.paginator.", - "django.core.signing.", - "django.core.validators.", - "django.dispatch.dispatcher.", - "django.template.autoreload.", - "django.template.backends.", - "django.template.base.", - "django.template.context.", - "django.template.context_processors.", - "django.template.defaultfilters.", - "django.template.defaulttags.", - "django.template.engine.", - "django.template.exceptions.", - "django.template.library.", - "django.template.loader.", - "django.template.loader_tags.", - "django.template.loaders.", - "django.template.response.", - "django.template.smartif.", - "django.template.utils.", - "django.templatetags.", - "django.test.", - "django.urls.base.", - "django.urls.conf.", - "django.urls.converters.", - "django.urls.exceptions.", - "django.urls.resolvers.", - "django.urls.utils.", - "django.utils.", - "django_filters.compat.", - "django_filters.conf.", - "django_filters.constants.", - "django_filters.exceptions.", - "django_filters.fields.", - "django_filters.filters.", - "django_filters.filterset.", - "django_filters.rest_framework.", - "django_filters.rest_framework.backends.", - "django_filters.rest_framework.filters.", - "django_filters.rest_framework.filterset.", - "django_filters.utils.", - "django_filters.widgets." }; +static const char* static_denylist[] = { + "django.apps.config.", + "django.apps.registry.", + "django.conf.", + "django.contrib.admin.actions.", + "django.contrib.admin.admin.", + "django.contrib.admin.apps.", + "django.contrib.admin.checks.", + "django.contrib.admin.decorators.", + "django.contrib.admin.exceptions.", + "django.contrib.admin.helpers.", + "django.contrib.admin.image_formats.", + "django.contrib.admin.options.", + "django.contrib.admin.sites.", + "django.contrib.admin.templatetags.", + "django.contrib.admin.views.autocomplete.", + "django.contrib.admin.views.decorators.", + "django.contrib.admin.views.main.", + "django.contrib.admin.wagtail_hooks.", + "django.contrib.admin.widgets.", + "django.contrib.admindocs.utils.", + "django.contrib.admindocs.views.", + "django.contrib.auth.admin.", + "django.contrib.auth.apps.", + "django.contrib.auth.backends.", + "django.contrib.auth.base_user.", + "django.contrib.auth.checks.", + "django.contrib.auth.context_processors.", + "django.contrib.auth.decorators.", + "django.contrib.auth.hashers.", + "django.contrib.auth.image_formats.", + "django.contrib.auth.management.", + "django.contrib.auth.middleware.", + "django.contrib.auth.password_validation.", + "django.contrib.auth.signals.", + "django.contrib.auth.templatetags.", + "django.contrib.auth.validators.", + "django.contrib.auth.wagtail_hooks.", + "django.contrib.contenttypes.admin.", + "django.contrib.contenttypes.apps.", + "django.contrib.contenttypes.checks.", + "django.contrib.contenttypes.fields.", + "django.contrib.contenttypes.forms.", + "django.contrib.contenttypes.image_formats.", + "django.contrib.contenttypes.management.", + "django.contrib.contenttypes.models.", + "django.contrib.contenttypes.templatetags.", + "django.contrib.contenttypes.views.", + "django.contrib.contenttypes.wagtail_hooks.", + "django.contrib.humanize.templatetags.", + "django.contrib.messages.admin.", + "django.contrib.messages.api.", + "django.contrib.messages.apps.", + "django.contrib.messages.constants.", + "django.contrib.messages.context_processors.", + "django.contrib.messages.image_formats.", + "django.contrib.messages.middleware.", + "django.contrib.messages.storage.", + "django.contrib.messages.templatetags.", + "django.contrib.messages.utils.", + "django.contrib.messages.wagtail_hooks.", + "django.contrib.sessions.admin.", + "django.contrib.sessions.apps.", + "django.contrib.sessions.backends.", + "django.contrib.sessions.base_session.", + "django.contrib.sessions.exceptions.", + "django.contrib.sessions.image_formats.", + "django.contrib.sessions.middleware.", + "django.contrib.sessions.templatetags.", + "django.contrib.sessions.wagtail_hooks.", + "django.contrib.sites.", + "django.contrib.staticfiles.admin.", + "django.contrib.staticfiles.apps.", + "django.contrib.staticfiles.checks.", + "django.contrib.staticfiles.finders.", + "django.contrib.staticfiles.image_formats.", + "django.contrib.staticfiles.models.", + "django.contrib.staticfiles.storage.", + "django.contrib.staticfiles.templatetags.", + "django.contrib.staticfiles.utils.", + "django.contrib.staticfiles.wagtail_hooks.", + "django.core.cache.backends.", + "django.core.cache.utils.", + "django.core.checks.async_checks.", + "django.core.checks.caches.", + "django.core.checks.compatibility.", + "django.core.checks.compatibility.django_4_0.", + "django.core.checks.database.", + "django.core.checks.files.", + "django.core.checks.messages.", + "django.core.checks.model_checks.", + "django.core.checks.registry.", + "django.core.checks.security.", + "django.core.checks.security.base.", + "django.core.checks.security.csrf.", + "django.core.checks.security.sessions.", + "django.core.checks.templates.", + "django.core.checks.translation.", + "django.core.checks.urls", + "django.core.exceptions.", + "django.core.mail.", + "django.core.management.base.", + "django.core.management.color.", + "django.core.management.sql.", + "django.core.paginator.", + "django.core.signing.", + "django.core.validators.", + "django.dispatch.dispatcher.", + "django.template.autoreload.", + "django.template.backends.", + "django.template.base.", + "django.template.context.", + "django.template.context_processors.", + "django.template.defaultfilters.", + "django.template.defaulttags.", + "django.template.engine.", + "django.template.exceptions.", + "django.template.library.", + "django.template.loader.", + "django.template.loader_tags.", + "django.template.loaders.", + "django.template.response.", + "django.template.smartif.", + "django.template.utils.", + "django.templatetags.", + "django.test.", + "django.urls.base.", + "django.urls.conf.", + "django.urls.converters.", + "django.urls.exceptions.", + "django.urls.resolvers.", + "django.urls.utils.", + "django.utils.", + "django_filters.compat.", + "django_filters.conf.", + "django_filters.constants.", + "django_filters.exceptions.", + "django_filters.fields.", + "django_filters.filters.", + "django_filters.filterset.", + "django_filters.rest_framework.", + "django_filters.rest_framework.backends.", + "django_filters.rest_framework.filters.", + "django_filters.rest_framework.filterset.", + "django_filters.utils.", + "django_filters.widgets.", +}; static const size_t static_denylist_count = sizeof(static_denylist) / sizeof(static_denylist[0]); static const char* static_stdlib_denylist[] = { - "__future__", - "_ast", - "_compression", - "_thread", - "abc", - "aifc", - "argparse", - "array", - "ast", - "asynchat", - "asyncio", - "asyncore", - "atexit", - "audioop", - "base64", - "bdb", - "binascii", - "bisect", - "builtins", - "bz2", - "cProfile", - "calendar", - "cgi", - "cgitb", - "chunk", - "cmath", - "cmd", - "code", - "codecs", - "codeop", - "collections", - "colorsys", - "compileall", - "concurrent", - "configparser", - "contextlib", - "contextvars", - "copy", - "copyreg", - "crypt", - "csv", - "ctypes", - "curses", - "dataclasses", - "datetime", - "dbm", - "decimal", - "difflib", - "dis", - "distutils", - "doctest", - "email", - "encodings", - "ensurepip", - "enum", - "errno", - "faulthandler", - "fcntl", - "filecmp", - "fileinput", - "fnmatch", - "fractions", - "ftplib", - "functools", - "gc", - "getopt", - "getpass", - "gettext", - "glob", - "graphlib", - "grp", - "gzip", - "hashlib", - "heapq", - "hmac", - "http", - "idlelib", - "imaplib", - "imghdr", - "imp", - "importlib", - "inspect", - "io", - "_io", - "ipaddress", - "itertools", - "json", - "keyword", - "lib2to3", - "linecache", - "locale", - "logging", - "lzma", - "mailbox", - "mailcap", - "marshal", - "math", - "mimetypes", - "mmap", - "modulefinder", - "msilib", - "msvcrt", - "multiprocessing", - "netrc", - "nis", - "nntplib", - "ntpath", - "numbers", - "opcode", - "operator", - "optparse", - "os", - "ossaudiodev", - "pathlib", - "pdb", - "pickle", - "pickletools", - "pipes", - "pkgutil", - "platform", - "plistlib", - "poplib", - "posix", - "posixpath", - "pprint", - "profile", - "pstats", - "pty", - "pwd", - "py_compile", - "pyclbr", - "pydoc", - "queue", - "quopri", - "random", - "re", - "readline", - "reprlib", - "resource", - "rlcompleter", - "runpy", - "sched", - "secrets", - "select", - "selectors", - "shelve", - "shutil", - "signal", - "site", - "smtpd", - "smtplib", - "sndhdr", - "socket", - "socketserver", - "spwd", - "sqlite3", - "sre", - "sre_compile", - "sre_constants", - "sre_parse", - "ssl", - "stat", - "statistics", - "string", - "stringprep", - "struct", - "subprocess", - "sunau", - "symtable", - "sys", - "sysconfig", - "syslog", - "tabnanny", - "tarfile", - "telnetlib", - "tempfile", - "termios", - "test", - "textwrap", - "threading", - "time", - "timeit", - "tkinter", - "token", - "tokenize", - "tomllib", - "trace", - "traceback", - "tracemalloc", - "tty", - "turtle", - "turtledemo", - "types", - "typing", - "unicodedata", - "unittest", - "uu", - "uuid", - "venv", - "warnings", - "wave", - "weakref", - "webbrowser", - "winreg", - "winsound", - "wsgiref", - "xdrlib", - "xml", - "xmlrpc", - "zipapp", - "zipfile", - "zipimport", - "zlib", - "zoneinfo", + "__future__", "_ast", "_compression", "_csv", + "_thread", "abc", "aifc", "argparse", + "array", "ast", "asynchat", "asyncio", + "asyncore", "atexit", "audioop", "base64", + "bdb", "binascii", "bisect", "builtins", + "bz2", "cProfile", "calendar", "cgi", + "cgitb", "chunk", "cmath", "cmd", + "code", "codecs", "codeop", "collections", + "colorsys", "compileall", "concurrent", "configparser", + "contextlib", "contextvars", "copy", "copyreg", + "crypt", "csv", "ctypes", "curses", + "dataclasses", "datetime", "dbm", "decimal", + "difflib", "dis", "distutils", "doctest", + "email", "encodings", "ensurepip", "enum", + "errno", "faulthandler", "fcntl", "filecmp", + "fileinput", "fnmatch", "fractions", "ftplib", + "functools", "gc", "getopt", "getpass", + "gettext", "glob", "graphlib", "grp", + "gzip", "hashlib", "heapq", "hmac", + "http", "idlelib", "imaplib", "imghdr", + "imp", "importlib", "inspect", "io", + "_io", "ipaddress", "itertools", "json", + "keyword", "lib2to3", "linecache", "locale", + "logging", "lzma", "mailbox", "mailcap", + "marshal", "math", "mimetypes", "mmap", + "modulefinder", "msilib", "msvcrt", "multiprocessing", + "netrc", "nis", "nntplib", "ntpath", + "numbers", "opcode", "operator", "optparse", + "os", "ossaudiodev", "pathlib", "pdb", + "pickle", "pickletools", "pipes", "pkgutil", + "platform", "plistlib", "poplib", "posix", + "posixpath", "pprint", "profile", "pstats", + "pty", "pwd", "py_compile", "pyclbr", + "pydoc", "queue", "quopri", "random", + "re", "readline", "reprlib", "resource", + "rlcompleter", "runpy", "sched", "secrets", + "select", "selectors", "shelve", "shutil", + "signal", "site", "smtpd", "smtplib", + "sndhdr", "socket", "socketserver", "spwd", + "sqlite3", "sre", "sre_compile", "sre_constants", + "sre_parse", "ssl", "stat", "statistics", + "string", "stringprep", "struct", "subprocess", + "sunau", "symtable", "sys", "sysconfig", + "syslog", "tabnanny", "tarfile", "telnetlib", + "tempfile", "termios", "test", "textwrap", + "threading", "time", "timeit", "tkinter", + "token", "tokenize", "tomllib", "trace", + "traceback", "tracemalloc", "tty", "turtle", + "turtledemo", "types", "typing", "unicodedata", + "unittest", "uu", "uuid", "venv", + "warnings", "wave", "weakref", "webbrowser", + "winreg", "winsound", "wsgiref", "xdrlib", + "xml", "xmlrpc", "zipapp", "zipfile", + "zipimport", "zlib", "zoneinfo", }; static const size_t static_stdlib_denylist_count = sizeof(static_stdlib_denylist) / sizeof(static_stdlib_denylist[0]); @@ -498,6 +341,13 @@ is_first_party(const char* module_name) } } } + // Print all cached_packages for debugging + // printf("cached_packages (count: %zu):\n", cached_packages_count); + // for (size_t i = 0; i < cached_packages_count; i++) { + // if (cached_packages[i]) { + // printf(" [%zu]: %s\n", i, cached_packages[i]); + // } + // } Py_DECREF(fast); } @@ -709,11 +559,6 @@ py_should_iast_patch(PyObject* self, PyObject* args) return PyLong_FromLong(DENIED_BUILTINS_DENYLIST); } - /* Allow if it's a first-party module */ - if (is_first_party(module_name)) { - return PyLong_FromLong(ALLOWED_FIRST_PARTY_ALLOWLIST); - } - /* Check in the static allow/deny lists */ if (str_in_list(lower_module, static_allowlist, static_allowlist_count)) { if (str_in_list(lower_module, static_denylist, static_denylist_count)) { @@ -721,6 +566,12 @@ py_should_iast_patch(PyObject* self, PyObject* args) } return PyLong_FromLong(ALLOWED_STATIC_ALLOWLIST); } + + /* Allow if it's a first-party module */ + if (is_first_party(module_name)) { + return PyLong_FromLong(ALLOWED_FIRST_PARTY_ALLOWLIST); + } + return PyLong_FromLong(DENIED_NOT_FOUND); } diff --git a/ddtrace/appsec/_iast/_logs.py b/ddtrace/appsec/_iast/_logs.py index c82617daab9..6ce2986e370 100644 --- a/ddtrace/appsec/_iast/_logs.py +++ b/ddtrace/appsec/_iast/_logs.py @@ -32,6 +32,10 @@ def iast_propagation_debug_log(msg, *args, **kwargs): log.debug("iast::propagation::error::%s", msg, *args, **kwargs) +def iast_propagation_sink_point_debug_log(msg, *args, **kwargs): + log.debug("iast::propagation::sink_point::%s", msg, *args, **kwargs) + + def iast_instrumentation_ast_patching_errorr_log(msg): iast_error(msg, default_prefix="iast::instrumentation::ast_patching::") diff --git a/ddtrace/appsec/_iast/taint_sinks/command_injection.py b/ddtrace/appsec/_iast/taint_sinks/command_injection.py index f34a5a54b6f..266cf7ea0c5 100644 --- a/ddtrace/appsec/_iast/taint_sinks/command_injection.py +++ b/ddtrace/appsec/_iast/taint_sinks/command_injection.py @@ -12,6 +12,7 @@ from ddtrace.settings.asm import config as asm_config from .._logs import iast_error +from .._logs import iast_propagation_sink_point_debug_log from .._overhead_control_engine import oce from ._base import VulnerabilityBase @@ -51,6 +52,7 @@ def _iast_report_cmdi(shell_args: Union[str, List[str]]) -> None: try: if asm_config.is_iast_request_enabled: if CommandInjection.has_quota(): + iast_propagation_sink_point_debug_log("Check command injection sink point") from .._taint_tracking.aspects import join_aspect if isinstance(shell_args, (list, tuple)): @@ -62,6 +64,7 @@ def _iast_report_cmdi(shell_args: Union[str, List[str]]) -> None: report_cmdi = shell_args if report_cmdi: + iast_propagation_sink_point_debug_log("Reporting command injection") CommandInjection.report(evidence_value=report_cmdi) # Reports Span Metrics @@ -69,4 +72,4 @@ def _iast_report_cmdi(shell_args: Union[str, List[str]]) -> None: # Report Telemetry Metrics _set_metric_iast_executed_sink(CommandInjection.vulnerability_type) except Exception as e: - iast_error(f"propagation::sink_point::Error in _iast_report_ssrf. {e}") + iast_error(f"propagation::sink_point::Error in _iast_report_cmdi. {e}") diff --git a/tests/appsec/app.py b/tests/appsec/app.py index 163e4755c1d..772f46ccafe 100644 --- a/tests/appsec/app.py +++ b/tests/appsec/app.py @@ -3,6 +3,7 @@ import copy import os import re +import shlex import subprocess # nosec from flask import Flask @@ -200,6 +201,14 @@ def iast_cmdi_vulnerability(): return resp +@app.route("/iast-cmdi-vulnerability-secure", methods=["GET"]) +def view_cmdi_secure(): + filename = request.args.get("filename") + subp = subprocess.Popen(args=["ls", "-la", shlex.quote(filename)]) + subp.wait() + return Response("OK") + + @app.route("/shutdown", methods=["GET"]) def shutdown_view(): tracer._span_aggregator.writer.flush_queue() diff --git a/tests/appsec/iast/_ast/test_ast_patching.py b/tests/appsec/iast/_ast/test_ast_patching.py index 61854b4f1a6..4fd59940a9c 100644 --- a/tests/appsec/iast/_ast/test_ast_patching.py +++ b/tests/appsec/iast/_ast/test_ast_patching.py @@ -151,6 +151,7 @@ def test_astpatch_source_unchanged(module_name): def test_should_iast_patch_allow_first_party(): assert iastpatch.should_iast_patch("file_in_my_project.main") == iastpatch.ALLOWED_FIRST_PARTY_ALLOWLIST assert iastpatch.should_iast_patch("file_in_my_project.print_str") == iastpatch.ALLOWED_FIRST_PARTY_ALLOWLIST + assert iastpatch.should_iast_patch("html") == iastpatch.ALLOWED_FIRST_PARTY_ALLOWLIST def test_should_iast_patch_allow_user_allowlist(): @@ -186,15 +187,228 @@ def test_should_not_iast_patch_if_not_in_static_allowlist(): assert iastpatch.should_iast_patch("pip.foo.bar") == iastpatch.DENIED_NOT_FOUND -def test_should_not_iast_patch_if_stdlib(): - assert iastpatch.should_iast_patch("base64") == iastpatch.DENIED_BUILTINS_DENYLIST - assert iastpatch.should_iast_patch("itertools") == iastpatch.DENIED_BUILTINS_DENYLIST - assert iastpatch.should_iast_patch("http") == iastpatch.DENIED_BUILTINS_DENYLIST - assert iastpatch.should_iast_patch("os.path") == iastpatch.DENIED_BUILTINS_DENYLIST - assert iastpatch.should_iast_patch("os") == iastpatch.DENIED_BUILTINS_DENYLIST - assert iastpatch.should_iast_patch("sys.platform") == iastpatch.DENIED_BUILTINS_DENYLIST - assert iastpatch.should_iast_patch("sys") == iastpatch.DENIED_BUILTINS_DENYLIST - assert iastpatch.should_iast_patch("sys.my.sub.module") == iastpatch.DENIED_BUILTINS_DENYLIST +@pytest.mark.parametrize( + "module_name", + { + "__future__", + "_ast", + "_compression", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "http", + "idlelib", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "_io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "opcode", + "operator", + "optparse", + "os", + "os.path", + "ossaudiodev", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "tomllib", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", + }, +) +def test_should_not_iast_patch_if_stdlib(module_name): + assert iastpatch.should_iast_patch(module_name) == iastpatch.DENIED_BUILTINS_DENYLIST def test_module_path_none(caplog): diff --git a/tests/appsec/iast/aspects/test_split_aspect.py b/tests/appsec/iast/aspects/test_split_aspect.py index 891209258d9..3371b05e783 100644 --- a/tests/appsec/iast/aspects/test_split_aspect.py +++ b/tests/appsec/iast/aspects/test_split_aspect.py @@ -1,6 +1,8 @@ import logging import sys +from hypothesis import given +from hypothesis.strategies import one_of import pytest from ddtrace.appsec._iast._taint_tracking import OriginType @@ -15,6 +17,7 @@ from ddtrace.appsec._iast._taint_tracking._context import reset_context from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from tests.appsec.iast.aspects.test_aspect_helpers import _build_sample_range +from tests.appsec.iast.iast_utils import non_empty_text from tests.utils import override_global_config @@ -23,6 +26,15 @@ def wrap_somesplit(func, *args, **kwargs): return func(None, 0, *args, **kwargs) +@given(one_of(non_empty_text)) +def test_aspect_split(text): + text_1 = text + text_2 = text * 3 + s = text_1 + " " + text_2 + res = wrap_somesplit(_aspect_split, s) + assert res == s.split() + + # These tests are simple ones testing the calls and replacements since most of the # actual testing is in test_aspect_helpers' test for set_ranges_on_splitted which these # functions call internally. diff --git a/tests/appsec/iast/aspects/test_str_aspect.py b/tests/appsec/iast/aspects/test_str_aspect.py index 21483e99fcc..c803c420334 100644 --- a/tests/appsec/iast/aspects/test_str_aspect.py +++ b/tests/appsec/iast/aspects/test_str_aspect.py @@ -1,6 +1,10 @@ -# -*- coding: utf-8 -*- +from pathlib import Path +from pathlib import PosixPath from unittest import mock +from hypothesis import given +from hypothesis.strategies import from_type +from hypothesis.strategies import one_of import pytest from ddtrace.appsec._iast._taint_tracking import OriginType @@ -14,11 +18,49 @@ from tests.appsec.iast.aspects.aspect_utils import BaseReplacement from tests.appsec.iast.aspects.aspect_utils import create_taint_range_with_format from tests.appsec.iast.iast_utils import _iast_patched_module +from tests.appsec.iast.iast_utils import string_strategies mod = _iast_patched_module("benchmarks.bm.iast_fixtures.str_methods") +@given( + one_of(string_strategies), +) +def test_str_aspect(text): + kwargs = {} + assert ddtrace_aspects.str_aspect(str, 0, text, **kwargs) == str(text, **kwargs) + + +@given(from_type(Path)) +def test_str_aspect_path(path): + assert ddtrace_aspects.str_aspect(str, 0, path) == str(path) + + +@given(from_type(PosixPath)) +def test_str_aspect_posixpath(path): + assert ddtrace_aspects.str_aspect(str, 0, path) == str(path) + + +@given(from_type(PosixPath)) +def test_str_aspect_posixpath_error(posixpath): + with pytest.raises(TypeError) as exc_info: + ddtrace_aspects.add_aspect(posixpath, posixpath) + assert str(exc_info.value) == "unsupported operand type(s) for +: 'PosixPath' and 'PosixPath'" + + +def test_str_aspect_error(): + text = "abc" + kwargs = {"encoding": "utf-8", "errors": "strict"} + with pytest.raises(TypeError) as exc_info: + str(text, **kwargs) + assert str(exc_info.value) == "decoding str is not supported" + + with pytest.raises(TypeError) as exc_info: + ddtrace_aspects.str_aspect(str, 0, "text", **kwargs) + assert str(exc_info.value) == "decoding str is not supported" + + @pytest.mark.parametrize( "obj, args, kwargs", [ @@ -34,7 +76,7 @@ (["a", "b", "c", "d"], (), {}), ], ) -def test_str_aspect(obj, args, kwargs): +def test_str_aspect_encodings(obj, args, kwargs): import ddtrace.appsec._iast._taint_tracking.aspects as ddtrace_aspects obj = taint_pyobject( diff --git a/tests/appsec/iast/iast_utils.py b/tests/appsec/iast/iast_utils.py index aac450d3e29..796db228e15 100644 --- a/tests/appsec/iast/iast_utils.py +++ b/tests/appsec/iast/iast_utils.py @@ -20,8 +20,8 @@ # Check if the log contains "iast::" to raise an error if that’s the case BUT, if the logs contains # "iast::instrumentation::" or "iast::instrumentation::" -# are valid logs -IAST_VALID_LOG = re.compile(r"^iast::(?!instrumentation::|propagation::context::).*$") +# are valid +IAST_VALID_LOG = re.compile(r"^iast::(?!instrumentation::|propagation::context::|propagation::sink_point).*$") class IastTestException(Exception): diff --git a/tests/appsec/integrations/django_tests/django_app/urls.py b/tests/appsec/integrations/django_tests/django_app/urls.py index a0591bfc038..c5e7518200b 100644 --- a/tests/appsec/integrations/django_tests/django_app/urls.py +++ b/tests/appsec/integrations/django_tests/django_app/urls.py @@ -28,6 +28,9 @@ def shutdown(request): handler("appsec/weak-hash/$", views.weak_hash_view, name="weak_hash"), handler("appsec/block/$", views.block_callable_view, name="block"), handler("appsec/command-injection/$", views.command_injection, name="command_injection"), + handler( + "appsec/command-injection-subprocess/$", views.command_injection_subprocess, name="command_injection_subprocess" + ), handler( "appsec/command-injection/secure-mark/$", views.command_injection_secure_mark, diff --git a/tests/appsec/integrations/django_tests/django_app/views.py b/tests/appsec/integrations/django_tests/django_app/views.py index 4c04599145d..9cecace598b 100644 --- a/tests/appsec/integrations/django_tests/django_app/views.py +++ b/tests/appsec/integrations/django_tests/django_app/views.py @@ -8,6 +8,7 @@ from pathlib import Path from pathlib import PosixPath import shlex +import subprocess from typing import Any from django.db import connection @@ -315,6 +316,16 @@ def command_injection(request): return HttpResponse("OK", status=200) +def command_injection_subprocess(request): + cmd = request.POST.get("cmd", "") + filename = "/" + # label iast_command_injection_subprocess + subp = subprocess.Popen(args=[cmd, "-la", filename], shell=True) + subp.communicate() + subp.wait() + return HttpResponse("OK", status=200) + + def command_injection_secure_mark(request): value = request.body.decode() # label iast_command_injection diff --git a/tests/appsec/integrations/django_tests/test_django_appsec_iast.py b/tests/appsec/integrations/django_tests/test_django_appsec_iast.py index 630ad0cec7c..d9e94f42a07 100644 --- a/tests/appsec/integrations/django_tests/test_django_appsec_iast.py +++ b/tests/appsec/integrations/django_tests/test_django_appsec_iast.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- import json from urllib.parse import urlencode import pytest -from ddtrace.appsec._common_module_patches import patch_common_modules from ddtrace.appsec._constants import IAST from ddtrace.appsec._constants import IAST_SPAN_TAGS from ddtrace.appsec._iast.constants import VULN_CMDI @@ -20,14 +18,6 @@ TEST_FILE = "tests/appsec/integrations/django_tests/django_app/views.py" -@pytest.fixture(autouse=True) -def iast_context(): - with override_global_config( - dict(_iast_enabled=True, _iast_deduplication_enabled=False, _iast_request_sampling=100.0) - ): - yield - - def _aux_appsec_get_root_span( client, test_spans, @@ -822,7 +812,6 @@ def test_django_querydict(client, test_spans, tracer): @pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") def test_django_command_injection(client, test_spans, tracer): - patch_common_modules() root_span, _ = _aux_appsec_get_root_span( client, test_spans, @@ -848,9 +837,35 @@ def test_django_command_injection(client, test_spans, tracer): assert loaded["vulnerabilities"][0]["location"]["path"] == TEST_FILE +@pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") +def test_django_command_injection_subprocess(client, test_spans, tracer): + root_span, _ = _aux_appsec_get_root_span( + client, + test_spans, + tracer, + url="/appsec/command-injection-subprocess/", + payload=urlencode({"cmd": "ls"}), + content_type="application/x-www-form-urlencoded", + ) + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + + line, hash_value = get_line_and_hash("iast_command_injection_subprocess", VULN_CMDI, filename=TEST_FILE) + + assert loaded["sources"] == [ + {"name": "cmd", "origin": "http.request.body", "value": "ls"} + ], f'Assertion error: {loaded["sources"]}' + assert loaded["vulnerabilities"][0]["type"] == VULN_CMDI + assert loaded["vulnerabilities"][0]["hash"] == hash_value + assert loaded["vulnerabilities"][0]["evidence"] == { + "valueParts": [{"value": "ls", "source": 0}, {"value": " "}, {"redacted": True}] + }, f'Assertion error: {loaded["vulnerabilities"][0]["evidence"]}' + assert loaded["vulnerabilities"][0]["location"]["line"] == line + assert loaded["vulnerabilities"][0]["location"]["path"] == TEST_FILE + + @pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") def test_django_command_injection_span_metrics(client, test_spans, tracer): - patch_common_modules() root_span, _ = _aux_appsec_get_root_span( client, test_spans, @@ -870,7 +885,6 @@ def test_django_command_injection_span_metrics(client, test_spans, tracer): @pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") def test_django_command_injection_span_metrics_disabled(client, iast_spans_with_zero_sampling, tracer): - patch_common_modules() root_span, _ = _aux_appsec_get_root_span( client, iast_spans_with_zero_sampling, @@ -890,7 +904,6 @@ def test_django_command_injection_span_metrics_disabled(client, iast_spans_with_ @pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") def test_django_command_injection_secure_mark(client, test_spans, tracer): - patch_common_modules() root_span, _ = _aux_appsec_get_root_span( client, test_spans, @@ -906,7 +919,6 @@ def test_django_command_injection_secure_mark(client, test_spans, tracer): @pytest.mark.skipif(not asm_config._iast_supported, reason="Python version not supported by IAST") def test_django_xss_secure_mark(client, test_spans, tracer): - patch_common_modules() root_span, _ = _aux_appsec_get_root_span( client, test_spans, diff --git a/tests/appsec/integrations/flask_tests/test_iast_flask_testagent.py b/tests/appsec/integrations/flask_tests/test_iast_flask_testagent.py index 7aabf2370c7..1b7f304d9ae 100644 --- a/tests/appsec/integrations/flask_tests/test_iast_flask_testagent.py +++ b/tests/appsec/integrations/flask_tests/test_iast_flask_testagent.py @@ -2,6 +2,7 @@ import pytest +from ddtrace.appsec._iast.constants import VULN_CMDI from ddtrace.appsec._iast.constants import VULN_STACKTRACE_LEAK from tests.appsec.appsec_utils import flask_server from tests.appsec.integrations.flask_tests.test_flask_remoteconfig import _get_agent_client @@ -48,8 +49,6 @@ def test_iast_stacktrace_error(): b"ValueError: Check my stacktrace!" in response.content ), f"Exception doesn't found in CONTENT: {response.content}" - # flask_client.get("/", headers={"X-Datadog-Test-Session-Token": token}) - response_tracer = _get_span(token) spans_with_iast = [] vulnerabilities = [] @@ -68,3 +67,56 @@ def test_iast_stacktrace_error(): assert "path" not in vulnerability["location"] assert "line" not in vulnerability["location"] assert vulnerability["hash"] + + +def test_iast_cmdi(): + token = "test_iast_cmdi" + _ = start_trace(token) + with flask_server( + iast_enabled="true", token=token, port=8050, env={"FLASK_DEBUG": "true", "DD_TRACE_DEBUG": "true"} + ) as context: + _, flask_client, pid = context + + response = flask_client.get("/iast-cmdi-vulnerability?filename=path_traversal_test_file.txt") + + assert response.status_code == 200 + + response_tracer = _get_span(token) + spans_with_iast = [] + vulnerabilities = [] + for trace in response_tracer: + for span in trace: + if span.get("metrics", {}).get("_dd.iast.enabled"): + spans_with_iast.append(span) + iast_data = span["meta"].get("_dd.iast.json") + if iast_data: + vulnerabilities.append(json.loads(iast_data).get("vulnerabilities")) + clear_session(token) + assert len(vulnerabilities) == 1 + vulnerability = vulnerabilities[0][0] + assert vulnerability["type"] == VULN_CMDI + assert vulnerability["evidence"]["valueParts"] == [ + {"value": "ls "}, + {"redacted": True}, + {"pattern": "abcdefghijklmnopqrstuvwxyzAB", "redacted": True, "source": 0}, + ] + assert vulnerability["hash"] + + +def test_iast_cmdi_secure(): + token = "test_iast_cmdi_secure" + _ = start_trace(token) + with flask_server(iast_enabled="true", token=token, port=8050, env={"FLASK_DEBUG": "true"}) as context: + _, flask_client, pid = context + + response = flask_client.get("/iast-cmdi-vulnerability-secure?filename=path_traversal_test_file.txt") + + assert response.status_code == 200 + + response_tracer = _get_span(token) + for trace in response_tracer: + for span in trace: + iast_data = span["meta"].get("_dd.iast.json") + if iast_data: + pytest.fail(f"There is iast vulnerabilities: {iast_data}") + clear_session(token) diff --git a/tests/appsec/integrations/packages_tests/test_ast_patching.py b/tests/appsec/integrations/packages_tests/test_ast_patching.py new file mode 100644 index 00000000000..1793c16a103 --- /dev/null +++ b/tests/appsec/integrations/packages_tests/test_ast_patching.py @@ -0,0 +1,47 @@ +""" +Some related tests are located in integrations/packages_tests because they require +external packages (e.g., werkzeug) to be installed in the environment. +This is necessary for detection by should_iast_patch, +which checks for packages in site-packages. +""" +import sys + +import pytest + +from ddtrace.appsec._iast._ast import iastpatch + + +def test_should_iast_patch_packages_allow(): + assert iastpatch.should_iast_patch("markupsafe") == iastpatch.ALLOWED_STATIC_ALLOWLIST + assert iastpatch.should_iast_patch("pymysql") == iastpatch.ALLOWED_STATIC_ALLOWLIST + assert iastpatch.should_iast_patch("pymysql.charset") == iastpatch.ALLOWED_STATIC_ALLOWLIST + assert iastpatch.should_iast_patch("pymysql.connections") == iastpatch.ALLOWED_STATIC_ALLOWLIST + assert iastpatch.should_iast_patch("pymysql.cursors") == iastpatch.ALLOWED_STATIC_ALLOWLIST + assert iastpatch.should_iast_patch("sqlalchemy") == iastpatch.ALLOWED_STATIC_ALLOWLIST + assert iastpatch.should_iast_patch("sqlalchemy.connectors") == iastpatch.ALLOWED_STATIC_ALLOWLIST + assert iastpatch.should_iast_patch("sqlalchemy.engine") == iastpatch.ALLOWED_STATIC_ALLOWLIST + + +def test_should_iast_patch_packages_deny_not_found(): + assert iastpatch.should_iast_patch("pytest") == iastpatch.DENIED_NOT_FOUND + assert iastpatch.should_iast_patch("hypothesis") == iastpatch.DENIED_NOT_FOUND + assert iastpatch.should_iast_patch("psycopg2") == iastpatch.DENIED_NOT_FOUND + + +def test_should_iast_patch_packages_deny_builtins(): + assert iastpatch.should_iast_patch("pytest") == iastpatch.DENIED_NOT_FOUND + assert iastpatch.should_iast_patch("requests") == iastpatch.DENIED_BUILTINS_DENYLIST + + +@pytest.mark.skipif(sys.version_info[:2] == (3, 10), reason="Python 3.10 doesn't recognize werkzeug as first_party") +def test_should_iast_patch_packages_allow_werkzeug(): + assert iastpatch.should_iast_patch("werkzeug.utils") == iastpatch.ALLOWED_STATIC_ALLOWLIST + + +@pytest.mark.skipif(sys.version_info[:2] == (3, 10), reason="Profiling unsupported with 3.13") +def test_should_iast_patch_packages_werkzeug_not_found(): + assert iastpatch.should_iast_patch("werkzeug") == iastpatch.DENIED_NOT_FOUND + assert iastpatch.should_iast_patch("werkzeug.datastructures") == iastpatch.DENIED_NOT_FOUND + assert iastpatch.should_iast_patch("werkzeug.datastructures.structures") == iastpatch.DENIED_NOT_FOUND + assert iastpatch.should_iast_patch("werkzeug.datastructures.ranges") == iastpatch.DENIED_NOT_FOUND + assert iastpatch.should_iast_patch("werkzeug.utils") != iastpatch.DENIED_NOT_FOUND