From cd4977e84a7f4a0152c4b22d5818e27af150fc81 Mon Sep 17 00:00:00 2001 From: "bar.harel" Date: Tue, 10 Sep 2024 19:14:30 +0300 Subject: [PATCH 1/7] defaults added --- docs/changelog.md | 5 +++++ pyproject.toml | 2 +- src/pythonjsonlogger/core.py | 10 ++++++++-- tests/test_formatters.py | 25 +++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 720708c..e3c2ff5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.2.0](https://github.com/nhairs/python-json-logger/compare/v3.1.0...v3.2.0) - 2024-09-10 + +### Changed +- `defaults` parameter is no longer ignored and now conforms to the standard library. Setting a defaults dictionary will add the specified keys if the those keys do not exist in a record or weren't passed by the `extra` parameter when logging a message. + ## [3.1.0](https://github.com/nhairs/python-json-logger/compare/v3.0.1...v3.1.0) - 2023-05-28 This splits common funcitonality out to allow supporting other JSON encoders. Although this is a large refactor, backwards compatibility has been maintained. diff --git a/pyproject.toml b/pyproject.toml index 40ec377..45a4b2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python-json-logger" -version = "3.1.0" +version = "3.2.0" description = "JSON Log Formatter for the Python Logging Package" authors = [ {name = "Zakaria Zajac", email = "zak@madzak.com"}, diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 820aa94..85f972d 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -102,6 +102,7 @@ def merge_record_extra( target: Dict, reserved: Container[str], rename_fields: Optional[Dict[str, str]] = None, + defaults: Optional[Dict[str, Any]] = None, ) -> Dict: """ Merges extra attributes from LogRecord object into target dictionary @@ -117,7 +118,8 @@ def merge_record_extra( """ if rename_fields is None: rename_fields = {} - for key, value in record.__dict__.items(): + record_dict = record.__dict__ if not defaults else (defaults | record.__dict__) + for key, value in record_dict.items(): # this allows to have numeric keys if key not in reserved and not (hasattr(key, "startswith") and key.startswith("_")): target[rename_fields.get(key, key)] = value @@ -161,7 +163,9 @@ def __init__( style: how to extract log fields from `fmt` validate: validate `fmt` against style, if implementing a custom `style` you must set this to `False`. - defaults: ignored - kept for compatibility with python 3.10+ + defaults: a dictionary containing default values for unspecified + extras. {"key": 1234} will add the key to the json if + unspecified in the extras while logging a message. prefix: an optional string prefix added at the beginning of the formatted string rename_fields: an optional dict, used to rename field names in the output. @@ -215,6 +219,7 @@ def __init__( self._required_fields = self.parse() self._skip_fields = set(self._required_fields) self._skip_fields.update(self.reserved_attrs) + self.defaults = defaults return def format(self, record: logging.LogRecord) -> str: @@ -322,6 +327,7 @@ def add_fields( log_record, reserved=self._skip_fields, rename_fields=self.rename_fields, + defaults=self.defaults, ) if self.timestamp: diff --git a/tests/test_formatters.py b/tests/test_formatters.py index cbaf886..d4f43c7 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -174,6 +174,18 @@ def test_percentage_format(env: LoggingEnvironment, class_: type[BaseJsonFormatt return +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_defaults_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): + env.set_formatter(class_(defaults={"first": 1, "second": 2})) + + env.logger.info("testing defaults field", extra={"first": 1234}) + log_json = env.load_json() + + assert log_json["first"] == 1234 + assert log_json["second"] == 2 + return + + @pytest.mark.parametrize("class_", ALL_FORMATTERS) def test_rename_base_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_(rename_fields={"message": "@message"})) @@ -186,6 +198,19 @@ def test_rename_base_field(env: LoggingEnvironment, class_: type[BaseJsonFormatt return +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_rename_with_defaults(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): + """Make sure that the default fields are also renamed.""" + env.set_formatter(class_(rename_fields={"custom": "@custom"}, defaults={"custom": 1234})) + + msg = "testing rename with defaults" + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["@custom"] == 1234 + return + + @pytest.mark.parametrize("class_", ALL_FORMATTERS) def test_rename_missing(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_(rename_fields={"missing_field": "new_field"})) From ac4a4b4249fa0f1599ce602e4d8806205ddeea28 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Mon, 21 Oct 2024 21:42:19 +1100 Subject: [PATCH 2/7] Improve handling of optional packages (#27) Improves handling of optional packages by: - No importing them just to check if available - Raises a more specific type of error (and message) ### Test Plan - Run unit tests --- docs/changelog.md | 4 +- src/pythonjsonlogger/__init__.py | 17 ++------ src/pythonjsonlogger/exception.py | 27 +++++++++++++ src/pythonjsonlogger/msgspec.py | 6 ++- src/pythonjsonlogger/orjson.py | 6 ++- src/pythonjsonlogger/utils.py | 40 ++++++++++++++++++ tests/test_missing.py | 67 +++++++++++++++++++++++++++++++ 7 files changed, 150 insertions(+), 17 deletions(-) create mode 100644 src/pythonjsonlogger/exception.py create mode 100644 src/pythonjsonlogger/utils.py create mode 100644 tests/test_missing.py diff --git a/docs/changelog.md b/docs/changelog.md index e3c2ff5..5c23b27 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,9 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.2.0](https://github.com/nhairs/python-json-logger/compare/v3.1.0...v3.2.0) - 2024-09-10 +## [3.2.0](https://github.com/nhairs/python-json-logger/compare/v3.1.0...v3.2.0) - UNRELEASED ### Changed +- `pythonjsonlogger.[ORJSON,MSGSPEC]_AVAILABLE` no longer imports the respective package when determining availability. +- `pythonjsonlogger.[orjson,msgspec]` now throws a `pythonjsonlogger.exception.MissingPackageError` when required libraries are not available. These contain more information about what is missing whilst still being an `ImportError`. - `defaults` parameter is no longer ignored and now conforms to the standard library. Setting a defaults dictionary will add the specified keys if the those keys do not exist in a record or weren't passed by the `extra` parameter when logging a message. ## [3.1.0](https://github.com/nhairs/python-json-logger/compare/v3.0.1...v3.1.0) - 2023-05-28 diff --git a/src/pythonjsonlogger/__init__.py b/src/pythonjsonlogger/__init__.py index ed3ae60..2eee544 100644 --- a/src/pythonjsonlogger/__init__.py +++ b/src/pythonjsonlogger/__init__.py @@ -9,23 +9,12 @@ ## Application import pythonjsonlogger.json +import pythonjsonlogger.utils ### CONSTANTS ### ============================================================================ -try: - import orjson - - ORJSON_AVAILABLE = True -except ImportError: - ORJSON_AVAILABLE = False - - -try: - import msgspec - - MSGSPEC_AVAILABLE = True -except ImportError: - MSGSPEC_AVAILABLE = False +ORJSON_AVAILABLE = pythonjsonlogger.utils.package_is_available("orjson") +MSGSPEC_AVAILABLE = pythonjsonlogger.utils.package_is_available("msgspec") ### DEPRECATED COMPATIBILITY diff --git a/src/pythonjsonlogger/exception.py b/src/pythonjsonlogger/exception.py new file mode 100644 index 0000000..1233f1a --- /dev/null +++ b/src/pythonjsonlogger/exception.py @@ -0,0 +1,27 @@ +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library + +## Installed + +## Application + + +### CLASSES +### ============================================================================ +class PythonJsonLoggerError(Exception): + "Generic base clas for all Python JSON Logger exceptions" + + +class MissingPackageError(ImportError, PythonJsonLoggerError): + "A required package is missing" + + def __init__(self, name: str, extras_name: str | None = None) -> None: + msg = f"The {name!r} package is required but could not be found." + if extras_name is not None: + msg += f" It can be installed using 'python-json-logger[{extras_name}]'." + super().__init__(msg) + return diff --git a/src/pythonjsonlogger/msgspec.py b/src/pythonjsonlogger/msgspec.py index 9208240..8646f85 100644 --- a/src/pythonjsonlogger/msgspec.py +++ b/src/pythonjsonlogger/msgspec.py @@ -9,11 +9,15 @@ from typing import Any ## Installed -import msgspec.json ## Application from . import core from . import defaults as d +from .utils import package_is_available + +# We import msgspec after checking it is available +package_is_available("msgspec", throw_error=True) +import msgspec.json # pylint: disable=wrong-import-position,wrong-import-order ### FUNCTIONS diff --git a/src/pythonjsonlogger/orjson.py b/src/pythonjsonlogger/orjson.py index 3e9ea30..16db842 100644 --- a/src/pythonjsonlogger/orjson.py +++ b/src/pythonjsonlogger/orjson.py @@ -9,11 +9,15 @@ from typing import Any ## Installed -import orjson ## Application from . import core from . import defaults as d +from .utils import package_is_available + +# We import msgspec after checking it is available +package_is_available("orjson", throw_error=True) +import orjson # pylint: disable=wrong-import-position,wrong-import-order ### FUNCTIONS diff --git a/src/pythonjsonlogger/utils.py b/src/pythonjsonlogger/utils.py new file mode 100644 index 0000000..d810a13 --- /dev/null +++ b/src/pythonjsonlogger/utils.py @@ -0,0 +1,40 @@ +"""Utilities for Python JSON Logger""" + +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library +import importlib.util + +## Installed + +## Application +from .exception import MissingPackageError + + +### FUNCTIONS +### ============================================================================ +def package_is_available( + name: str, *, throw_error: bool = False, extras_name: str | None = None +) -> bool: + """Determine if the given package is available for import. + + Args: + name: Import name of the package to check. + throw_error: Throw an error if the package is unavailable. + extras_name: Extra dependency name to use in `throw_error`'s message. + + Raises: + MissingPackageError: When `throw_error` is `True` and the return value would be `False` + + Returns: + If the package is available for import. + """ + available = importlib.util.find_spec(name) is not None + + if not available and throw_error: + raise MissingPackageError(name, extras_name) + + return available diff --git a/tests/test_missing.py b/tests/test_missing.py new file mode 100644 index 0000000..0878014 --- /dev/null +++ b/tests/test_missing.py @@ -0,0 +1,67 @@ +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library + +## Installed +import pytest + +## Application +import pythonjsonlogger +from pythonjsonlogger.utils import package_is_available +from pythonjsonlogger.exception import MissingPackageError + +### CONSTANTS +### ============================================================================ +MISSING_PACKAGE_NAME = "package_name_is_definintely_not_available" +MISSING_PACKAGE_EXTRA = "package_extra_that_is_unique" + + +### TESTS +### ============================================================================ +def test_package_is_available(): + assert package_is_available("json") + return + + +def test_package_not_available(): + assert not package_is_available(MISSING_PACKAGE_NAME) + return + + +def test_package_not_available_throw(): + with pytest.raises(MissingPackageError) as e: + package_is_available(MISSING_PACKAGE_NAME, throw_error=True) + assert MISSING_PACKAGE_NAME in e.value.msg + assert MISSING_PACKAGE_EXTRA not in e.value.msg + return + + +def test_package_not_available_throw_extras(): + with pytest.raises(MissingPackageError) as e: + package_is_available( + MISSING_PACKAGE_NAME, throw_error=True, extras_name=MISSING_PACKAGE_EXTRA + ) + assert MISSING_PACKAGE_NAME in e.value.msg + assert MISSING_PACKAGE_EXTRA in e.value.msg + return + + +## Python JSON Logger Specific +## ----------------------------------------------------------------------------- +if not pythonjsonlogger.ORJSON_AVAILABLE: + + def test_orjson_import_error(): + with pytest.raises(MissingPackageError, match="orjson"): + import pythonjsonlogger.orjson + return + + +if not pythonjsonlogger.MSGSPEC_AVAILABLE: + + def test_msgspec_import_error(): + with pytest.raises(MissingPackageError, match="msgspec"): + import pythonjsonlogger.msgspec + return From c78b015f0c97e6b6074de1bed1baa225990b005c Mon Sep 17 00:00:00 2001 From: "bar.harel" Date: Tue, 22 Oct 2024 15:19:39 +0100 Subject: [PATCH 3/7] cr fixes --- src/pythonjsonlogger/core.py | 9 ++++----- tests/test_formatters.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 85f972d..79d12da 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -102,7 +102,6 @@ def merge_record_extra( target: Dict, reserved: Container[str], rename_fields: Optional[Dict[str, str]] = None, - defaults: Optional[Dict[str, Any]] = None, ) -> Dict: """ Merges extra attributes from LogRecord object into target dictionary @@ -118,8 +117,7 @@ def merge_record_extra( """ if rename_fields is None: rename_fields = {} - record_dict = record.__dict__ if not defaults else (defaults | record.__dict__) - for key, value in record_dict.items(): + for key, value in record.__dict__.items(): # this allows to have numeric keys if key not in reserved and not (hasattr(key, "startswith") and key.startswith("_")): target[rename_fields.get(key, key)] = value @@ -166,6 +164,7 @@ def __init__( defaults: a dictionary containing default values for unspecified extras. {"key": 1234} will add the key to the json if unspecified in the extras while logging a message. + These fields are added prior to renaming. prefix: an optional string prefix added at the beginning of the formatted string rename_fields: an optional dict, used to rename field names in the output. @@ -219,7 +218,7 @@ def __init__( self._required_fields = self.parse() self._skip_fields = set(self._required_fields) self._skip_fields.update(self.reserved_attrs) - self.defaults = defaults + self.defaults = defaults if defaults is not None else {} return def format(self, record: logging.LogRecord) -> str: @@ -253,7 +252,7 @@ def format(self, record: logging.LogRecord) -> str: if record.stack_info and not message_dict.get("stack_info"): message_dict["stack_info"] = self.formatStack(record.stack_info) - log_record: LogRecord = {} + log_record: LogRecord = self.defaults.copy() self.add_fields(log_record, record, message_dict) log_record = self.process_log_record(log_record) diff --git a/tests/test_formatters.py b/tests/test_formatters.py index d4f43c7..719c569 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -346,6 +346,20 @@ def test_log_dict(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): return +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_log_dict_defaults(env: LoggingEnvironment, + class_: type[BaseJsonFormatter]): + env.set_formatter(class_(defaults={"d1": 1234, "d2": "hello"})) + + msg = {"d2": "world"} + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["d1"] == 1234 + assert log_json["d2"] == "world" + return + + @pytest.mark.parametrize("class_", ALL_FORMATTERS) def test_log_extra(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_()) From b22af5685155f114f57142ec9b086a1fdc3515a0 Mon Sep 17 00:00:00 2001 From: "bar.harel" Date: Tue, 22 Oct 2024 15:40:53 +0100 Subject: [PATCH 4/7] pr fix --- src/pythonjsonlogger/core.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 79d12da..5abb58b 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -252,7 +252,7 @@ def format(self, record: logging.LogRecord) -> str: if record.stack_info and not message_dict.get("stack_info"): message_dict["stack_info"] = self.formatStack(record.stack_info) - log_record: LogRecord = self.defaults.copy() + log_record: LogRecord = {} self.add_fields(log_record, record, message_dict) log_record = self.process_log_record(log_record) @@ -314,6 +314,9 @@ def add_fields( message_dict: dictionary that was logged instead of a message. e.g `logger.info({"is_this_message_dict": True})` """ + for field in self.defaults: + log_record[self._get_rename(field)] = self.defaults[field] + for field in self._required_fields: log_record[self._get_rename(field)] = record.__dict__.get(field) @@ -326,7 +329,6 @@ def add_fields( log_record, reserved=self._skip_fields, rename_fields=self.rename_fields, - defaults=self.defaults, ) if self.timestamp: From 8ced132b0fc1fcc3e2f22777b0e2750276111ff7 Mon Sep 17 00:00:00 2001 From: "bar.harel" Date: Wed, 23 Oct 2024 17:11:50 +0100 Subject: [PATCH 5/7] black formatting --- tests/test_formatters.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_formatters.py b/tests/test_formatters.py index 719c569..01295ab 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -347,8 +347,7 @@ def test_log_dict(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_log_dict_defaults(env: LoggingEnvironment, - class_: type[BaseJsonFormatter]): +def test_log_dict_defaults(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_(defaults={"d1": 1234, "d2": "hello"})) msg = {"d2": "world"} From e5a96085c17b5f57b543f371d2b351e2d464a94a Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 26 Oct 2024 12:58:49 +1100 Subject: [PATCH 6/7] Apply suggestions from code review --- pyproject.toml | 2 +- src/pythonjsonlogger/core.py | 6 ++---- tests/test_formatters.py | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 45a4b2f..e97c197 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python-json-logger" -version = "3.2.0" +version = "3.2.0.dev1" description = "JSON Log Formatter for the Python Logging Package" authors = [ {name = "Zakaria Zajac", email = "zak@madzak.com"}, diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 5abb58b..2b11d31 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -161,10 +161,8 @@ def __init__( style: how to extract log fields from `fmt` validate: validate `fmt` against style, if implementing a custom `style` you must set this to `False`. - defaults: a dictionary containing default values for unspecified - extras. {"key": 1234} will add the key to the json if - unspecified in the extras while logging a message. - These fields are added prior to renaming. + defaults: a dictionary containing default fields that are added before all other fields and + may be overridden. The supplied fields are still subject to `rename_fields`. prefix: an optional string prefix added at the beginning of the formatted string rename_fields: an optional dict, used to rename field names in the output. diff --git a/tests/test_formatters.py b/tests/test_formatters.py index 01295ab..2212429 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -208,6 +208,7 @@ def test_rename_with_defaults(env: LoggingEnvironment, class_: type[BaseJsonForm log_json = env.load_json() assert log_json["@custom"] == 1234 + assert "custom" not in log_json return From 5131b70923251f5cd651356a49e0093158c3c7f0 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 10 Nov 2024 15:10:01 +1100 Subject: [PATCH 7/7] Update docs --- docs/quickstart.md | 12 ++++++++++++ src/pythonjsonlogger/core.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/docs/quickstart.md b/docs/quickstart.md index 3b78e5e..dc7a032 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -78,6 +78,18 @@ logger.info( Finally, any non-standard attributes added to a `LogRecord` will also be included in the logged data. See [Cookbook: Request / Trace IDs](cookbook.md#request-trace-ids) for an example. +#### Default Fields + +Default fields that are added to every log record prior to any other field can be set using the `default` argument. + +```python +formatter = JsonFormatter( + defaults={"environment": "dev"} +) +# ... +logger.info("this overwrites the environment field", extras={"environment": "dev"}) +``` + #### Static Fields Static fields that are added to every log record can be set using the `static_fields` argument. diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 2b11d31..27501b2 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -132,6 +132,8 @@ class BaseJsonFormatter(logging.Formatter): Must not be used directly. *New in 3.1* + + *Changed in 3.2*: `defaults` argument is no longer ignored. """ _style: Union[logging.PercentStyle, str] # type: ignore[assignment]