commit a5372cf079542bcb482282059a7e1ffe59f277d8 Author: Kristian Krsnik Date: Sat Mar 9 00:20:52 2024 +0100 intitial commit diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..488901c --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/result + +/.direnv/ + +*.log +*.json +*.jsonl diff --git a/README.md b/README.md new file mode 100644 index 0000000..02fadf5 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# Modern Logging Example + +Threaded logging only works in Python 3.12+. + +## Resources + +* [Modern Python Logging by mCoding](https://youtu.be/9L77QExPmI0) diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..da4ec42 --- /dev/null +++ b/flake.lock @@ -0,0 +1,141 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1705309234, + "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "poetry2nix-lib", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1703863825, + "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1709703039, + "narHash": "sha256-6hqgQ8OK6gsMu1VtcGKBxKQInRLHtzulDo9Z5jxHEFY=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "9df3e30ce24fd28c7b3e2de0d986769db5d6225d", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "poetry2nix-lib": { + "inputs": { + "flake-utils": "flake-utils", + "nix-github-actions": "nix-github-actions", + "nixpkgs": [ + "nixpkgs" + ], + "systems": "systems_2", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1708589824, + "narHash": "sha256-2GOiFTkvs5MtVF65sC78KNVxQSmsxtk0WmV1wJ9V2ck=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "3c92540611f42d3fb2d0d084a6c694cd6544b609", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "poetry2nix-lib": "poetry2nix-lib" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "poetry2nix-lib", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1708335038, + "narHash": "sha256-ETLZNFBVCabo7lJrpjD6cAbnE11eDOjaQnznmg/6hAE=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "e504621290a1fd896631ddbc5e9c16f4366c9f65", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..558c8d4 --- /dev/null +++ b/flake.nix @@ -0,0 +1,49 @@ +{ + description = "PLACEHOLDER"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + poetry2nix-lib = { + url = "github:nix-community/poetry2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + }; + + outputs = { + self, + nixpkgs, + poetry2nix-lib, + }: let + supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + pkgs = forAllSystems (system: nixpkgs.legacyPackages.${system}); + poetry2nix = forAllSystems (system: poetry2nix-lib.lib.mkPoetry2Nix {pkgs = pkgs.${system};}); + in { + # `nix build` + packages = forAllSystems (system: { + default = poetry2nix.${system}.mkPoetryApplication { + projectDir = self; + }; + }); + + # `nix fmt` + formatter = forAllSystems (system: pkgs.${system}.alejandra); + + # `nix develop` + devShells = forAllSystems (system: { + default = let + poetryEnv = + if builtins.pathExists ./poetry.lock + then poetry2nix.${system}.mkPoetryEnv {projectDir = self;} + else null; + in + pkgs.${system}.mkShellNoCC { + packages = with pkgs.${system}; + [ + poetry + ] + ++ [poetryEnv]; + }; + }); + }; +} diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..86a4a80 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +package = [] + +[metadata] +lock-version = "2.0" +python-versions = "^3.11" +content-hash = "81b2fa642d7f2d1219cf80112ace12d689d053d81be7f7addb98144d56fc0fb2" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fa816a9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "main" +version = "0.1.0" +description = "" +authors = ["Your Name "] +readme = "README.md" +packages = [{ include = "src" }] + +[tool.poetry.dependencies] +python = "^3.11" + +[tool.poetry.scripts] +main = "src.main:main" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/filter.py b/src/filter.py new file mode 100644 index 0000000..e69de29 diff --git a/src/helper.py b/src/helper.py new file mode 100644 index 0000000..3a464b9 --- /dev/null +++ b/src/helper.py @@ -0,0 +1,77 @@ +import json +import logging +from datetime import datetime, timezone + + +LOG_RECORD_BUILTIN_ATTRS = { + "args", + "asctime", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "message", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack_info", + "thread", + "threadName", + "taskName", +} + + +class JSONFormatter(logging.Formatter): + + def __init__(self, *, fmt_keys: dict[str, str] | None = None): + super().__init__() + self.fmt_keys = fmt_keys if fmt_keys is not None else {} + + # @override + def format(self, record: logging.LogRecord) -> str: + message = self._prepare_log_dict(record) + return json.dumps(message, default=str) + + def _prepare_log_dict(self, record: logging.LogRecord) -> None: + always_fields = { + 'message': record.getMessage(), + 'timestamp': datetime.fromtimestamp( + record.created, tz=timezone.utc + ).isoformat() + } + + if record.exc_info is not None: + always_fields['exc_info'] = self.formatException(record.exc_info) + + if record.stack_info is not None: + always_fields['stack_info'] = self.formatStack(record.stack_info) + + message = { + key: msg_value + if (msg_value := always_fields.pop(value, None)) is not None + else getattr(record, value) + for key, value in self.fmt_keys.items() + } + + message.update(always_fields) + + for key, value in record.__dict__.items(): + if key not in LOG_RECORD_BUILTIN_ATTRS: + message[key] = value + + return message + + +class NonErrorFilter(logging.Filter): + # @override + def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord: + return record.levelno <= logging.INFO diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 0000000..f1c52c8 --- /dev/null +++ b/src/logger.py @@ -0,0 +1,89 @@ +import sys +import logging.config + +logger_config = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + "no_errors": { + "()": "src.helper.NonErrorFilter" + } + }, + 'formatters': { + 'simple': { + 'format': '[%(asctime)s][%(levelname)s] %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + }, + 'detailed': { + 'format': '[%(asctime)s][%(levelname)s] %(message)s', + 'datefmt': '%Y-%m-%dT%H:%M:%S%z' # ISO-8601 Timestamp + }, + 'json': { + '()': 'src.helper.JSONFormatter', + 'fmt_keys': { + 'timestamp': 'timestamp', + 'level': 'levelname', + 'message': 'message', + 'logger': 'name', + 'module': 'module', + 'function': 'funcName', + 'line': 'lineno', + 'thread_name': 'threadName' + }, + } + }, + 'handlers': { + 'stdout': { + 'class': 'logging.StreamHandler', + 'filters': ['no_errors'], + 'formatter': 'simple', + 'stream': 'ext://sys.stdout' + }, + 'stderr': { + 'class': 'logging.StreamHandler', + 'level': 'WARNING', + 'formatter': 'simple', + 'stream': 'ext://sys.stderr' + }, + 'file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'level': 'DEBUG', + 'formatter': 'json', + 'filename': 'log.jsonl', + 'maxBytes': 1024 * 1024 * 10, # 10 MB + 'backupCount': 3 + }, + }, + 'loggers': { + 'root': { + 'level': 'DEBUG', + 'handlers': [ + 'stdout', + 'stderr', + 'file' + ], + } + } +} + +if sys.version_info >= (3, 12): # Python 3.12+ + logger_config['handlers']['queue_handler']: { + 'class': 'logging.handlers.QueueHandler', + 'handlers': [ + 'stdout', + 'stderr', + 'file' + ] + } + + logger_config['loggers']['root']['handlers'] = ['queue_handler'] + + +def setup_logging() -> None: + logging.config.dictConfig(logger_config) + + if sys.version_info >= (3, 12): # Python 3.12+ + queue_handler = logging.getHandlerByName('queue_handler') + if queue_handler is not None: + queue_handler.listener.start() + atextit.register(queue_handler.listener.stop) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..71a5393 --- /dev/null +++ b/src/main.py @@ -0,0 +1,16 @@ +import logging + +from .logger import setup_logging + + +def main(): + setup_logging() + + LOGGER = logging.getLogger('main') + LOGGER.warn('This is a warning') + + LOGGER.debug('Another log message.', extra={'x': 'Hello'}) + + +if __name__ == '__main__': + main()