Skip to content

Commit 23b92ce

Browse files
committed
feat: first pass at the package
1 parent 5844ad0 commit 23b92ce

File tree

10 files changed

+209
-43
lines changed

10 files changed

+209
-43
lines changed

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ dmypy.json
144144
.vscode/*
145145
*.csv
146146
*.doc*
147-
*.jsonl
147+
# *.jsonl
148148
*.lnk
149149
*.log
150150
*.pdf
@@ -203,3 +203,5 @@ coverage.json
203203

204204
# ----------------------------------------------------------------------------------------------------------------------
205205
# Custom Rules
206+
207+
.logs/*.jsonl

docs/README.md

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,26 @@
22

33
Tail JSONL Logs
44

5-
## Installation
5+
## Background
66

7-
1. `poetry add `
7+
I wanted to find a tool that could:
88

9-
1. ...
9+
1. Convert a stream of JSONL logs into a readable `logfmt`-like output with minimal configuration
10+
1. Show exceptions on their own line
1011

11-
```sh
12-
import
12+
I investigated a lot of alternatives such as: [humanlog](https://github.com/humanlogio/humanlog), [lnav](https://docs.lnav.org/en/latest/formats.html#), [goaccess](https://goaccess.io/get-started), [angle-grinder](https://github.com/rcoh/angle-grinder#rendering), [jq](https://github.com/stedolan/jq), [textualog](https://github.com/rhuygen/textualog), etc. but None had the exception formatting I wanted.
1313

14-
# < TODO: Add example code here >
15-
```
14+
## Installation
1615

17-
1. ...
16+
```sh
17+
pipx install tail-jsonl
18+
```
1819

1920
## Usage
2021

21-
<!-- < TODO: Show an example (screenshots, terminal recording, etc.) > -->
22-
23-
For more example code, see the [scripts] directory or the [tests].
22+
```sh
23+
echo '{"message": "message", "timestamp": "2023-01-01T01:01:01.0123456Z", "level": "debug", "data": true, "more-data": [null, true, -123.123]}' | tail-jsonl
24+
```
2425

2526
## Project Status
2627

@@ -54,6 +55,4 @@ If you have any security issue to report, please contact the project maintainers
5455
[contributor-covenant]: https://www.contributor-covenant.org
5556
[developer_guide]: ./docs/DEVELOPER_GUIDE.md
5657
[license]: https://github.com/kyleking/tail-jsonl/LICENSE
57-
[scripts]: https://github.com/kyleking/tail-jsonl/scripts
5858
[style_guide]: ./docs/STYLE_GUIDE.md
59-
[tests]: https://github.com/kyleking/tail-jsonl/tests

poetry.lock

Lines changed: 5 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pyproject.toml

Lines changed: 26 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,34 +23,39 @@ line_length = 120
2323
multi_line_output = 5
2424

2525
[tool.poetry]
26-
name = "tail_jsonl"
27-
version = "0.0.1"
28-
description = "Tail JSONL Logs"
29-
license = "MIT"
3026
authors = ["Kyle King <[email protected]>"]
31-
maintainers = []
32-
repository = "https://github.com/kyleking/tail-jsonl"
27+
classifiers = [
28+
"Development Status :: 1 - Planning",
29+
"License :: OSI Approved :: MIT License",
30+
"Operating System :: OS Independent",
31+
"Programming Language :: Python :: 3.10",
32+
"Programming Language :: Python :: 3.11",
33+
"Programming Language :: Python :: 3.8",
34+
"Programming Language :: Python :: 3.9"
35+
] # https://pypi.org/classifiers/
36+
description = "Tail JSONL Logs"
3337
documentation = "https://tail-jsonl.kyleking.me"
34-
readme = "docs/README.md"
3538
include = ["LICENSE.md"]
3639
keywords = []
37-
classifiers = [
38-
"Development Status :: 1 - Planning",
39-
"License :: OSI Approved :: MIT License",
40-
"Operating System :: OS Independent",
41-
"Programming Language :: Python :: 3.8",
42-
"Programming Language :: Python :: 3.9",
43-
"Programming Language :: Python :: 3.10",
44-
"Programming Language :: Python :: 3.11",
45-
] # https://pypi.org/classifiers/
40+
license = "MIT"
41+
maintainers = []
42+
name = "tail_jsonl"
43+
readme = "docs/README.md"
44+
repository = "https://github.com/kyleking/tail-jsonl"
45+
version = "0.0.1"
4646

47-
[tool.poetry.urls]
48-
"Bug Tracker" = "https://github.com/kyleking/tail-jsonl/issues"
49-
"Changelog" = "https://github.com/kyleking/tail-jsonl/blob/main/docs/docs/CHANGELOG.md"
47+
48+
[tool.poetry.scripts]
49+
tail-jsonl = "tail_jsonl:main"
5050

5151
[tool.poetry.dependencies]
52-
python = "^3.8.12"
5352
calcipy = ">=0.21.3"
53+
python = "^3.8.12"
54+
rich = ">=13.3.1"
5455

5556
[tool.poetry.dev-dependencies]
56-
calcipy = { version = "*", extras = ["dev", "lint", "test"] }
57+
calcipy = {extras = ["dev", "lint", "test"], version = "*"}
58+
59+
[tool.poetry.urls]
60+
"Bug Tracker" = "https://github.com/kyleking/tail-jsonl/issues"
61+
"Changelog" = "https://github.com/kyleking/tail-jsonl/blob/main/docs/docs/CHANGELOG.md"

tail_jsonl/__init__.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
"""tail_jsonl."""
22

3-
from loguru import logger
3+
from loguru import logger # noqa: F401
44

55
__version__ = '0.0.1'
66
__pkg_name__ = 'tail_jsonl'
77

8-
logger.disable(__pkg_name__)
9-
108
# ====== Above is the recommended code from calcipy_template and may be updated on new releases ======
9+
10+
from .main import main # noqa: F401

tail_jsonl/_private/__init__.py

Whitespace-only changes.

tail_jsonl/_private/config.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
"""Configuration."""
2+
3+
from functools import cached_property
4+
5+
from beartype import beartype
6+
from pydantic import BaseModel, Field
7+
8+
9+
class Styles(BaseModel):
10+
"""Styles configuration.
11+
12+
Refer to https://rich.readthedocs.io/en/latest/style.html for available style
13+
14+
Inspired by: https://github.com/Delgan/loguru/blob/07f94f3c8373733119f85aa8b9ca05ace3325a4b/loguru/_defaults.py#L31-L73
15+
16+
And: https://github.com/hynek/structlog/blob/bcfc7f9e60640c150bffbdaeed6328e582f93d1e/src/structlog/dev.py#L126-L141
17+
18+
"""
19+
20+
timestamp: str = 'dim grey'
21+
message: str = ''
22+
23+
level_error: str = 'bold red'
24+
level_warn: str = 'bold yellow'
25+
level_info: str = 'bold green'
26+
level_debug: str = 'bold blue'
27+
level_fallback: str = ''
28+
29+
key: str = 'green'
30+
value: str = ''
31+
32+
@cached_property
33+
def _level_lookup(self) -> dict[str, str]:
34+
return {
35+
'ERROR': self.level_error,
36+
'WARNING': self.level_warn,
37+
'WARN': self.level_warn,
38+
'INFO': self.level_info,
39+
'DEBUG': self.level_debug,
40+
}
41+
42+
@beartype
43+
def get_level_style(self, level: str) -> str:
44+
"""Return the right style for the specified level."""
45+
return self._level_lookup.get(level.upper(), self.level_fallback)
46+
47+
48+
class Keys(BaseModel):
49+
"""Special Keys."""
50+
51+
timestamp: list[str] = Field(default_factory=lambda: ['timestamp'])
52+
level: list[str] = Field(default_factory=lambda: ['level'])
53+
message: list[str] = Field(default_factory=lambda: ['event', 'message'])
54+
55+
on_own_line: list[str] = Field(default_factory=lambda: ['exception'])
56+
57+
58+
class Config(BaseModel):
59+
"""`tail-jsonl` config."""
60+
61+
styles: Styles = Field(default_factory=Styles)
62+
keys: Keys = Field(default_factory=Keys)

tail_jsonl/_private/core.py

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
"""Core print logic."""
2+
3+
import json
4+
from copy import copy
5+
from typing import Any
6+
7+
from beartype import beartype
8+
from loguru import logger
9+
from pydantic import BaseModel
10+
from rich.console import Console
11+
from rich.text import Text
12+
13+
from .config import Config
14+
15+
16+
@beartype
17+
def pop_key(data: dict, keys: list[str], fallback: str) -> Any:
18+
"""Recursively pop whichever key matches first or default to the fallback."""
19+
try:
20+
key = keys.pop(0)
21+
return data.pop(key, None) or pop_key(data, keys, fallback)
22+
except IndexError:
23+
return fallback
24+
25+
26+
class Record(BaseModel):
27+
"""Record Model."""
28+
29+
timestamp: str
30+
level: str
31+
message: str
32+
data: dict
33+
34+
@classmethod
35+
def from_line(cls, data: dict, config: Config) -> 'Record':
36+
"""Extract Record from jsonl."""
37+
return cls(
38+
timestamp=pop_key(data, copy(config.keys.timestamp), '<no timestamp>'),
39+
level=pop_key(data, copy(config.keys.level), '<no level>'),
40+
message=pop_key(data, copy(config.keys.message), '<no message>'),
41+
data=data,
42+
)
43+
44+
45+
@beartype
46+
def print_record(line: str, console: Console, config: Config) -> None:
47+
"""Format and print the record."""
48+
try:
49+
record = Record.from_line(json.loads(line), config=config)
50+
except Exception:
51+
logger.exception('Error in tail-json to parse line', line=line)
52+
console.print('') # Line break
53+
return
54+
55+
text = Text(tab_size=4) # FIXME: Why isn't this indenting what is wrapped?
56+
text.append(f'{record.timestamp: <28}', style=config.styles.timestamp)
57+
text.append(f' {record.level: <7}', style=config.styles.get_level_style(record.level))
58+
text.append(f' {record.message: <20}', style=config.styles.message)
59+
60+
full_lines = []
61+
for key in config.keys.on_own_line:
62+
line = record.data.pop(key, None)
63+
if line:
64+
full_lines.append((key, line))
65+
66+
for key, value in record.data.items():
67+
text.append(f' {key}:', style=config.styles.key)
68+
text.append(f' {str(value): <10}', style=config.styles.value)
69+
70+
console.print(text)
71+
for key, line in full_lines:
72+
new_text = Text()
73+
new_text.append(f' ∟ {key}', style='bold green')
74+
new_text.append(f': {line}')
75+
console.print(new_text)

tail_jsonl/main.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
"""Public CLI interface."""
2+
3+
import fileinput
4+
5+
from beartype import beartype
6+
from rich.console import Console
7+
8+
from ._private.config import Config
9+
from ._private.core import print_record
10+
11+
12+
@beartype
13+
def main() -> None:
14+
"""CLI Entrypoint."""
15+
console = Console()
16+
# PLANNED: Support configuration with argparse or environment variable
17+
config = Config()
18+
for line in fileinput.input():
19+
print_record(line, console, config)

tail_jsonl/tmp.jsonl

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{"jsonrpc": "2.0", "method": "sum", "params": [ null, 1, 2, 4, false, true], "id": "1"}
2+
{"jsonrpc": "2.0", "method": "notify_hello", "params": [7]}
3+
{"jsonrpc": "2.0", "method": "subtract", "params": [42, 23], "id": "2"}
4+
{"event": "Testing", "user_id": "UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012')", "something": 123, "level": "debug", "timestamp": "2022-12-22T16:52:28.380253Z", "func_name": "<module>", "filename": "test.py", "lineno": 15, "exception": "line 1\n\t\tline 2 .... \n\t...?"}

0 commit comments

Comments
 (0)