From a5372cf079542bcb482282059a7e1ffe59f277d8 Mon Sep 17 00:00:00 2001 From: Kristian Krsnik Date: Sat, 9 Mar 2024 00:20:52 +0100 Subject: [PATCH] intitial commit --- .envrc | 1 + .gitignore | 7 +++ README.md | 7 +++ flake.lock | 141 ++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 49 +++++++++++++++++ poetry.lock | 7 +++ pyproject.toml | 17 ++++++ src/__init__.py | 0 src/filter.py | 0 src/helper.py | 77 ++++++++++++++++++++++++++ src/logger.py | 89 ++++++++++++++++++++++++++++++ src/main.py | 16 ++++++ 12 files changed, 411 insertions(+) create mode 100644 .envrc create mode 100644 .gitignore create mode 100644 README.md create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 src/__init__.py create mode 100644 src/filter.py create mode 100644 src/helper.py create mode 100644 src/logger.py create mode 100644 src/main.py diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..488901c --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/result + +/.direnv/ + +*.log +*.json +*.jsonl diff --git a/README.md b/README.md new file mode 100644 index 0000000..02fadf5 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# Modern Logging Example + +Threaded logging only works in Python 3.12+. + +## Resources + +* [Modern Python Logging by mCoding](https://youtu.be/9L77QExPmI0) diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..da4ec42 --- /dev/null +++ b/flake.lock @@ -0,0 +1,141 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "poetry2nix-lib", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1703863825, + "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1709703039, + "narHash": "sha256-6hqgQ8OK6gsMu1VtcGKBxKQInRLHtzulDo9Z5jxHEFY=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "9df3e30ce24fd28c7b3e2de0d986769db5d6225d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "poetry2nix-lib": { + "inputs": { + "flake-utils": "flake-utils", + "nix-github-actions": "nix-github-actions", + "nixpkgs": [ + "nixpkgs" + ], + "systems": "systems_2", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1708589824, + "narHash": "sha256-2GOiFTkvs5MtVF65sC78KNVxQSmsxtk0WmV1wJ9V2ck=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "3c92540611f42d3fb2d0d084a6c694cd6544b609", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "poetry2nix-lib": "poetry2nix-lib" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "poetry2nix-lib", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1708335038, + "narHash": "sha256-ETLZNFBVCabo7lJrpjD6cAbnE11eDOjaQnznmg/6hAE=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "e504621290a1fd896631ddbc5e9c16f4366c9f65", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..558c8d4 --- /dev/null +++ b/flake.nix @@ -0,0 +1,49 @@ +{ + description = "PLACEHOLDER"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + poetry2nix-lib = { + url = "github:nix-community/poetry2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = { + self, + nixpkgs, + poetry2nix-lib, + }: let + supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + pkgs = forAllSystems (system: nixpkgs.legacyPackages.${system}); + poetry2nix = forAllSystems (system: poetry2nix-lib.lib.mkPoetry2Nix {pkgs = pkgs.${system};}); + in { + # `nix build` + packages = forAllSystems (system: { + default = poetry2nix.${system}.mkPoetryApplication { + projectDir = self; + }; + }); + + # `nix fmt` + formatter = forAllSystems (system: pkgs.${system}.alejandra); + + # `nix develop` + devShells = forAllSystems (system: { + default = let + poetryEnv = + if builtins.pathExists ./poetry.lock + then poetry2nix.${system}.mkPoetryEnv {projectDir = self;} + else null; + in + pkgs.${system}.mkShellNoCC { + packages = with pkgs.${system}; + [ + poetry + ] + ++ [poetryEnv]; + }; + }); + }; +} diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..86a4a80 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +package = [] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "81b2fa642d7f2d1219cf80112ace12d689d053d81be7f7addb98144d56fc0fb2" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fa816a9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "main" +version = "0.1.0" +description = "" +authors = ["Your Name "] +readme = "README.md" +packages = [{ include = "src" }] + +[tool.poetry.dependencies] +python = "^3.11" + +[tool.poetry.scripts] +main = "src.main:main" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/filter.py b/src/filter.py new file mode 100644 index 0000000..e69de29 diff --git a/src/helper.py b/src/helper.py new file mode 100644 index 0000000..3a464b9 --- /dev/null +++ b/src/helper.py @@ -0,0 +1,77 @@ +import json +import logging +from datetime import datetime, timezone + + +LOG_RECORD_BUILTIN_ATTRS = { + "args", + "asctime", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "message", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack_info", + "thread", + "threadName", + "taskName", +} + + +class JSONFormatter(logging.Formatter): + + def __init__(self, *, fmt_keys: dict[str, str] | None = None): + super().__init__() + self.fmt_keys = fmt_keys if fmt_keys is not None else {} + + # @override + def format(self, record: logging.LogRecord) -> str: + message = self._prepare_log_dict(record) + return json.dumps(message, default=str) + + def _prepare_log_dict(self, record: logging.LogRecord) -> None: + always_fields = { + 'message': record.getMessage(), + 'timestamp': datetime.fromtimestamp( + record.created, tz=timezone.utc + ).isoformat() + } + + if record.exc_info is not None: + always_fields['exc_info'] = self.formatException(record.exc_info) + + if record.stack_info is not None: + always_fields['stack_info'] = self.formatStack(record.stack_info) + + message = { + key: msg_value + if (msg_value := always_fields.pop(value, None)) is not None + else getattr(record, value) + for key, value in self.fmt_keys.items() + } + + message.update(always_fields) + + for key, value in record.__dict__.items(): + if key not in LOG_RECORD_BUILTIN_ATTRS: + message[key] = value + + return message + + +class NonErrorFilter(logging.Filter): + # @override + def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord: + return record.levelno <= logging.INFO diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 0000000..f1c52c8 --- /dev/null +++ b/src/logger.py @@ -0,0 +1,89 @@ +import sys +import logging.config + +logger_config = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + "no_errors": { + "()": "src.helper.NonErrorFilter" + } + }, + 'formatters': { + 'simple': { + 'format': '[%(asctime)s][%(levelname)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + }, + 'detailed': { + 'format': '[%(asctime)s][%(levelname)s] %(message)s', + 'datefmt': '%Y-%m-%dT%H:%M:%S%z' # ISO-8601 Timestamp + }, + 'json': { + '()': 'src.helper.JSONFormatter', + 'fmt_keys': { + 'timestamp': 'timestamp', + 'level': 'levelname', + 'message': 'message', + 'logger': 'name', + 'module': 'module', + 'function': 'funcName', + 'line': 'lineno', + 'thread_name': 'threadName' + }, + } + }, + 'handlers': { + 'stdout': { + 'class': 'logging.StreamHandler', + 'filters': ['no_errors'], + 'formatter': 'simple', + 'stream': 'ext://sys.stdout' + }, + 'stderr': { + 'class': 'logging.StreamHandler', + 'level': 'WARNING', + 'formatter': 'simple', + 'stream': 'ext://sys.stderr' + }, + 'file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'level': 'DEBUG', + 'formatter': 'json', + 'filename': 'log.jsonl', + 'maxBytes': 1024 * 1024 * 10, # 10 MB + 'backupCount': 3 + }, + }, + 'loggers': { + 'root': { + 'level': 'DEBUG', + 'handlers': [ + 'stdout', + 'stderr', + 'file' + ], + } + } +} + +if sys.version_info >= (3, 12): # Python 3.12+ + logger_config['handlers']['queue_handler']: { + 'class': 'logging.handlers.QueueHandler', + 'handlers': [ + 'stdout', + 'stderr', + 'file' + ] + } + + logger_config['loggers']['root']['handlers'] = ['queue_handler'] + + +def setup_logging() -> None: + logging.config.dictConfig(logger_config) + + if sys.version_info >= (3, 12): # Python 3.12+ + queue_handler = logging.getHandlerByName('queue_handler') + if queue_handler is not None: + queue_handler.listener.start() + atextit.register(queue_handler.listener.stop) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..71a5393 --- /dev/null +++ b/src/main.py @@ -0,0 +1,16 @@ +import logging + +from .logger import setup_logging + + +def main(): + setup_logging() + + LOGGER = logging.getLogger('main') + LOGGER.warn('This is a warning') + + LOGGER.debug('Another log message.', extra={'x': 'Hello'}) + + +if __name__ == '__main__': + main()