fix: structured log handler drops reserved fields in json_fields (#634)

Author: daniel-sanche

google/cloud/logging_v2/handlers/structured_log.py

@@ -36,6 +36,26 @@
     "}"
 )
 
+# reserved fields taken from Structured Logging documentation:
+# https://cloud.google.com/logging/docs/structured-logging
+GCP_STRUCTURED_LOGGING_FIELDS = frozenset(
+    {
+        "severity",
+        "httpRequest",
+        "time",
+        "timestamp",
+        "timestampSeconds",
+        "timestampNanos",
+        "logging.googleapis.com/insertId",
+        "logging.googleapis.com/labels",
+        "logging.googleapis.com/operation",
+        "logging.googleapis.com/sourceLocation",
+        "logging.googleapis.com/spanId",
+        "logging.googleapis.com/trace",
+        "logging.googleapis.com/trace_sampled",
+    }
+)
+
 
 class StructuredLogHandler(logging.StreamHandler):
     """Handler to format logs into the Cloud Logging structured log format,
@@ -70,6 +90,10 @@ def format(self, record):
         message = _format_and_parse_message(record, super(StructuredLogHandler, self))
 
         if isinstance(message, collections.abc.Mapping):
+            # remove any special fields
+            for key in list(message.keys()):
+                if key in GCP_STRUCTURED_LOGGING_FIELDS:
+                    del message[key]
             # if input is a dictionary, encode it as a json string
             encoded_msg = json.dumps(message, ensure_ascii=False)
             # strip out open and close parentheses

tests/environment

@@ -207,6 +207,54 @@ def test_format_with_custom_formatter(self):
         self.assertIn(expected_result, result)
         self.assertIn("message", result)
 
+    def test_format_with_reserved_json_field(self):
+        # drop json_field data with reserved names
+        # related issue: https://github.com/googleapis/python-logging/issues/543
+        import logging
+        import json
+
+        handler = self._make_one()
+        message = "hello world"
+        extra = "still here"
+        json_fields = {
+            "message": "override?",
+            "severity": "error",
+            "logging.googleapis.com/trace_sampled": True,
+            "time": "none",
+            "extra": extra,
+            "SEVERITY": "error",
+        }
+        record = logging.LogRecord(
+            None,
+            logging.INFO,
+            None,
+            None,
+            message,
+            None,
+            None,
+        )
+        record.created = None
+        setattr(record, "json_fields", json_fields)
+        expected_payload = {
+            "message": message,
+            "severity": "INFO",
+            "SEVERITY": "error",
+            "logging.googleapis.com/trace": "",
+            "logging.googleapis.com/spanId": "",
+            "logging.googleapis.com/trace_sampled": False,
+            "logging.googleapis.com/sourceLocation": {},
+            "httpRequest": {},
+            "logging.googleapis.com/labels": {},
+            "extra": extra,
+        }
+        handler.filter(record)
+        result = json.loads(handler.format(record))
+        self.assertEqual(set(expected_payload.keys()), set(result.keys()))
+        for (key, value) in expected_payload.items():
+            self.assertEqual(
+                value, result[key], f"expected_payload[{key}] != result[{key}]"
+            )
+
     def test_dict(self):
         """
         Handler should parse json encoded as a string