diff --git a/.gitignore b/.gitignore index baf47da..9916cd8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +.env .venv __pycache__ .coverage +*/*.egg-info +uv.lock +.vscode diff --git a/README.md b/README.md index c4cea62..e38d8e1 100644 --- a/README.md +++ b/README.md @@ -1,78 +1,62 @@ -# Python project template +# API Tracer -This is a template repository for any Python project that comes with the following dev tools: +This library adds basic telemetry to Python projects that traces the usage and run time of Python functions within a given scope. -* `ruff`: identifies many errors and style issues (`flake8`, `isort`, `pyupgrade`) -* `black`: auto-formats code +## Installation -Those checks are run as pre-commit hooks using the `pre-commit` library. +Prerequisites: -It includes `pytest` for testing plus the `pytest-cov` plugin to measure coverage. - -The checks and tests are all run using Github actions on every pull request and merge to main. - -This repository is setup for Python 3.11. To change the version: -1. Change the `image` argument in `.devcontainer/devcontainer.json` (see [https://github.com/devcontainers/images/tree/main/src/python](https://github.com/devcontainers/images/tree/main/src/python#configuration) for a list of pre-built Docker images) -1. Change the config options in `.precommit-config.yaml` -1. Change the version number in `.github/workflows/python.yaml` - -## Development instructions - -## With devcontainer - -This repository comes with a devcontainer (a Dockerized Python environment). If you open it in Codespaces, it should automatically initialize the devcontainer. +``` +pip install opentelemetry-distro +pip install opentelemetry-exporter-otlp +opentelemetry-bootstrap --action=install +``` -Locally, you can open it in VS Code with the Dev Containers extension installed. +## Usage -## Without devcontainer +To track usage of one or more existing Python projects, run: -If you can't or don't want to use the devcontainer, then you should first create a virtual environment: +```python +from api_tracer import install, start_span_processor -``` -python3 -m venv .venv -source .venv/bin/activate +install( + [ + my_project.my_module + ] +) +start_span_processor('my-project-service') ``` -Then install the dev tools and pre-commit hooks: +To explicitly add instrumentation to functions you want to trace, use the `span` decorator: -``` -python3 -m pip install --user -r requirements-dev.txt -pre-commit install -``` +```python +from api_tracer import span, start_span_processor -## Adding code and tests -This repository starts with a very simple `main.py` and a test for it at `tests/main_test.py`. -You'll want to replace that with your own code, and you'll probably want to add additional files -as your code grows in complexity. +@span +def foo(bar): + print(bar) -When you're ready to run tests, run: - -``` -python3 -m pytest +if __name__ == "__main__": + start_span_processor("test-service") + foo(bar="baz") ``` -# File breakdown +## Start collector -Here's a short explanation of each file/folder in this template: +To start a collector that prints each log message to stdout, run `cd tests/collector` and run -* `.devcontainer`: Folder containing files used for setting up a devcontainer - * `devcontainer.json`: File configuring the devcontainer, includes VS Code settings -* `.github`: Folder for Github-specific files and folders - * `workflows`: Folder containing Github actions config files - * `python.yaml`: File configuring Github action that runs tools and tests -* `tests`: Folder containing Python tests - * `main_test.py`: File with pytest-style tests of main.py -* `.gitignore`: File describing what file patterns Git should never track -* `.pre-commit-config.yaml`: File listing all the pre-commit hooks and args -* `main.py`: The main (and currently only) Python file for the program -* `pyproject.toml`: File configuring most of the Python dev tools -* `README.md`: You're reading it! -* `requirements-dev.txt`: File listing all PyPi packages required for development -* `requirements.txt`: File listing all PyPi packages required for production - -For a longer explanation, read [this blog post](http://blog.pamelafox.org/2022/09/how-i-setup-python-project.html). +```bash +docker run -p 4317:4317 -p 4318:4318 --rm -v $(pwd)/collector-config.yaml:/etc/otelcol/config.yaml otel/opentelemetry-collector +``` -# 🔎 Found an issue or have an idea for improvement? +To start a Jaeger collector that starts a basic dashboard, run: -Help me make this template repository better by letting us know and opening an issue! +```bash +docker run --name jaeger \ + -e COLLECTOR_OTLP_ENABLED=true \ + -p 16686:16686 \ + -p 4317:4317 \ + -p 4318:4318 \ + jaegertracing/all-in-one:1.35 +``` diff --git a/collector/collector-config.yaml b/collector/collector-config.yaml new file mode 100644 index 0000000..b24459c --- /dev/null +++ b/collector/collector-config.yaml @@ -0,0 +1,22 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +exporters: + debug: + verbosity: detailed +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + metrics: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] + diff --git a/pyproject.toml b/pyproject.toml index fadfc3b..e128d11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,31 @@ +[project] +name = "api_tracer" +authors = [ + { name = "guenp", email = "guenp@hey.com" }, +] +description = "A great package." +readme = "README.md" +license = "MIT" +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 1 - Planning", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering", + "Typing :: Typed", +] +dynamic = ["version"] +dependencies = [] + [tool.ruff] line-length = 120 target-version = "py311" diff --git a/requirements.txt b/requirements.txt index e69de29..01c429b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,3 @@ +opentelemetry-api==1.25.0 +opentelemetry-sdk==1.33.0 +opentelemetry-exporter-otlp-proto-grpc==1.25.0 diff --git a/src/api_tracer/__init__.py b/src/api_tracer/__init__.py new file mode 100644 index 0000000..d1c5156 --- /dev/null +++ b/src/api_tracer/__init__.py @@ -0,0 +1,2 @@ +from api_tracer.path_finder import install +from api_tracer.span import span diff --git a/src/api_tracer/console.py b/src/api_tracer/console.py new file mode 100644 index 0000000..6ec0f93 --- /dev/null +++ b/src/api_tracer/console.py @@ -0,0 +1,24 @@ +import os + +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter + +__all__ = ["setup_console"] + + +def setup_console(service_name: str | None = None): + if service_name is None: + attributes = os.environ.get("OTEL_RESOURCE_ATTRIBUTES") + attributes = dict(k.split("=") for k in attributes.split(",")) + else: + attributes = {"service.name": service_name} + + resource = Resource(attributes=attributes) + trace.set_tracer_provider(TracerProvider(resource=resource)) + console_exporter = ConsoleSpanExporter() + + trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(console_exporter) + ) diff --git a/src/api_tracer/path_finder.py b/src/api_tracer/path_finder.py new file mode 100644 index 0000000..932fcb8 --- /dev/null +++ b/src/api_tracer/path_finder.py @@ -0,0 +1,69 @@ +import inspect +import sys +from importlib.abc import MetaPathFinder +from importlib.machinery import SourceFileLoader +from importlib.util import spec_from_loader + +from api_tracer.span import span + +__all__ = ["install"] + + +class TelemetryMetaFinder(MetaPathFinder): + def __init__(self, module_names, *args, **kwargs): + """MetaPathFinder implementation that overrides a spec loader + of type SourceFileLoader with a TelemetrySpanLoader. + + Args: + module_names (List[str]): Module names to include. + """ + self._module_names = module_names + super().__init__(*args, **kwargs) + + def find_spec(self, fullname, path, target=None): + if any([name in fullname for name in self._module_names]): + for finder in sys.meta_path: + if finder != self: + spec = finder.find_spec(fullname, path, target) + if spec is not None: + if isinstance(spec.loader, SourceFileLoader): + return spec_from_loader( + name=spec.name, + loader=TelemetrySpanSourceFileLoader( + spec.name, + spec.origin + ), + origin=spec.origin + ) + else: + return spec + + return None + + +class TelemetrySpanSourceFileLoader(SourceFileLoader): + def exec_module(self, module): + super().exec_module(module) + functions = inspect.getmembers(module, predicate=inspect.isfunction) + classes = inspect.getmembers(module, predicate=inspect.isclass) + + # Add telemetry to functions + for name, _function in functions: + _module = inspect.getmodule(_function) + if module == _module: + setattr(_module, name, span(_function)) + + # Add telemetry to methods + for _, _class in classes: + for name, method in inspect.getmembers( + _class, + predicate=inspect.isfunction + ): + if inspect.getmodule(_class) == module: + if not name.startswith("_"): + setattr(_class, name, span(method)) + + +def install(module_names: list[str]): + """Inserts the finder into the import machinery""" + sys.meta_path.insert(0, TelemetryMetaFinder(module_names)) diff --git a/src/api_tracer/span.py b/src/api_tracer/span.py new file mode 100644 index 0000000..1931365 --- /dev/null +++ b/src/api_tracer/span.py @@ -0,0 +1,42 @@ +from collections.abc import Sequence +from contextlib import wraps + +from opentelemetry import trace + +ALLOWED_TYPES = [bool, str, bytes, int, float] + +__all__ = ["span"] + + +def _get_func_name(func): + return f"{func.__module__}.{func.__qualname__}" + + +def _serialize(arg): + for _type in ALLOWED_TYPES: + if isinstance(arg, _type): + return arg + if isinstance(arg, Sequence) and len(arg) > 0: + if isinstance(arg[0], _type): + return arg + return str(arg) + + +def span(func): + # Creates a tracer from the global tracer provider + tracer = trace.get_tracer(__name__) + func_name = _get_func_name(func) + + @wraps(func) + def span_wrapper(*args, **kwargs): + with tracer.start_as_current_span(func_name) as span: + span.set_attribute("num_args", len(args)) + span.set_attribute("num_kwargs", len(kwargs)) + for n, arg in enumerate(args): + span.set_attribute(f"args.{n}", _serialize(arg)) + for k, v in kwargs.items(): + span.set_attribute(f"kwargs.{k}", v) + span.set_status(trace.StatusCode.OK) + return func(*args, **kwargs) + + return span_wrapper diff --git a/tests/collector/collector-config.yaml b/tests/collector/collector-config.yaml new file mode 100644 index 0000000..29b4a93 --- /dev/null +++ b/tests/collector/collector-config.yaml @@ -0,0 +1,21 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +exporters: + debug: + verbosity: detailed +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + metrics: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] diff --git a/tests/test_scipy.py b/tests/test_scipy.py new file mode 100644 index 0000000..9f97849 --- /dev/null +++ b/tests/test_scipy.py @@ -0,0 +1,22 @@ +from opentelemetry.instrumentation.auto_instrumentation import initialize + +from api_tracer import install +from api_tracer.console import setup_console + +install([ + "scipy.stats._correlation", + "scipy.stats._distn_infrastructure" +]) +initialize() +setup_console() + +from scipy import stats + +stats.norm.pdf(x=1, loc=1, scale=0.01) +stats.norm(loc=1, scale=0.02).pdf(1) +stats.chatterjeexi([1, 2, 3, 4],[1.1, 2.2, 3.3, 4.4]) + +# X = stats.Normal() +# Y = stats.exp((X + 1)*0.01) +# from scipy import test +# test() diff --git a/tests/test_span.py b/tests/test_span.py new file mode 100644 index 0000000..7094604 --- /dev/null +++ b/tests/test_span.py @@ -0,0 +1,27 @@ +from time import sleep + +from api_tracer.console import setup_console +from api_tracer.span import span + + +@span +def foo(hello="world", delay=1): + print(hello) + sleep(delay) + +@span +def bar(spam="eggs", delay=1): + print(spam) + sleep(delay) + +@span +def baz(apple="orange", delay=1): + print(apple) + sleep(delay) + + +if __name__ == "__main__": + setup_console("test") + foo(hello="foo", delay=1) + bar(spam="bar", delay=2) + baz(apple="baz", delay=3)