fix: disambiguate missing policy tags from explicitly unset policy tags (#983)

Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:
- [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code!  That way we can discuss the change, evaluate designs, and agree on the general idea
- [ ] Ensure the tests and linter pass
- [ ] Code coverage does not decrease (if any source code was changed)
- [ ] Appropriate docs were updated (if necessary)

Fixes #981 
Fixes #982
Towards googleapis/python-bigquery-pandas#387
🦕

Author: tswast

docs/reference.rst

@@ -128,6 +128,7 @@ Schema
     :toctree: generated
 
     schema.SchemaField
+    schema.PolicyTagList
 
 
 Query

google/cloud/bigquery/__init__.py

@@ -88,6 +88,7 @@
 from google.cloud.bigquery.routine import RoutineReference
 from google.cloud.bigquery.routine import RoutineType
 from google.cloud.bigquery.schema import SchemaField
+from google.cloud.bigquery.schema import PolicyTagList
 from google.cloud.bigquery.table import PartitionRange
 from google.cloud.bigquery.table import RangePartitioning
 from google.cloud.bigquery.table import Row
@@ -140,6 +141,7 @@
     "RoutineReference",
     # Shared helpers
     "SchemaField",
+    "PolicyTagList",
     "UDFResource",
     "ExternalConfig",
     "BigtableOptions",

google/cloud/bigquery/schema.py

@@ -15,12 +15,12 @@
 """Schemas for BigQuery tables / queries."""
 
 import collections
-from typing import Optional
+import enum
+from typing import Iterable, Union
 
 from google.cloud.bigquery_v2 import types
 
 
-_DEFAULT_VALUE = object()
 _STRUCT_TYPES = ("RECORD", "STRUCT")
 
 # SQL types reference:
@@ -49,47 +49,62 @@
 """String names of the legacy SQL types to integer codes of Standard SQL types."""
 
 
+class _DefaultSentinel(enum.Enum):
+    """Object used as 'sentinel' indicating default value should be used.
+
+    Uses enum so that pytype/mypy knows that this is the only possible value.
+    https://stackoverflow.com/a/60605919/101923
+
+    Literal[_DEFAULT_VALUE] is an alternative, but only added in Python 3.8.
+    https://docs.python.org/3/library/typing.html#typing.Literal
+    """
+
+    DEFAULT_VALUE = object()
+
+
+_DEFAULT_VALUE = _DefaultSentinel.DEFAULT_VALUE
+
+
 class SchemaField(object):
     """Describe a single field within a table schema.
 
     Args:
-        name (str): The name of the field.
+        name: The name of the field.
 
-        field_type (str): The type of the field. See
+        field_type:
+            The type of the field. See
             https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableFieldSchema.FIELDS.type
 
-        mode (Optional[str]): The mode of the field.  See
+        mode:
+            Defaults to ``'NULLABLE'``. The mode of the field. See
             https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#TableFieldSchema.FIELDS.mode
 
-        description (Optional[str]): Description for the field.
+        description: Description for the field.
 
-        fields (Optional[Tuple[google.cloud.bigquery.schema.SchemaField]]):
-            Subfields (requires ``field_type`` of 'RECORD').
+        fields: Subfields (requires ``field_type`` of 'RECORD').
 
-        policy_tags (Optional[PolicyTagList]): The policy tag list for the field.
+        policy_tags: The policy tag list for the field.
 
-        precision (Optional[int]):
+        precision:
             Precison (number of digits) of fields with NUMERIC or BIGNUMERIC type.
 
-        scale (Optional[int]):
+        scale:
             Scale (digits after decimal) of fields with NUMERIC or BIGNUMERIC type.
 
-        max_length (Optional[int]):
-            Maximim length of fields with STRING or BYTES type.
-
+        max_length: Maximum length of fields with STRING or BYTES type.
     """
 
     def __init__(
         self,
-        name,
-        field_type,
-        mode="NULLABLE",
-        description=_DEFAULT_VALUE,
-        fields=(),
-        policy_tags=None,
-        precision=_DEFAULT_VALUE,
-        scale=_DEFAULT_VALUE,
-        max_length=_DEFAULT_VALUE,
+        name: str,
+        field_type: str,
+        mode: str = "NULLABLE",
+        description: Union[str, _DefaultSentinel] = _DEFAULT_VALUE,
+        fields: Iterable["SchemaField"] = (),
+        policy_tags: Union["PolicyTagList", None, _DefaultSentinel] = _DEFAULT_VALUE,
+        precision: Union[int, _DefaultSentinel] = _DEFAULT_VALUE,
+        scale: Union[int, _DefaultSentinel] = _DEFAULT_VALUE,
+        max_length: Union[int, _DefaultSentinel] = _DEFAULT_VALUE,
     ):
         self._properties = {
             "name": name,
@@ -105,28 +120,12 @@ def __init__(
             self._properties["scale"] = scale
         if max_length is not _DEFAULT_VALUE:
             self._properties["maxLength"] = max_length
+        if policy_tags is not _DEFAULT_VALUE:
+            self._properties["policyTags"] = (
+                policy_tags.to_api_repr() if policy_tags is not None else None
+            )
         self._fields = tuple(fields)
 
-        self._policy_tags = self._determine_policy_tags(field_type, policy_tags)
-
-    @staticmethod
-    def _determine_policy_tags(
-        field_type: str, given_policy_tags: Optional["PolicyTagList"]
-    ) -> Optional["PolicyTagList"]:
-        """Return the given policy tags, or their suitable representation if `None`.
-
-        Args:
-            field_type: The type of the schema field.
-            given_policy_tags: The policy tags to maybe ajdust.
-        """
-        if given_policy_tags is not None:
-            return given_policy_tags
-
-        if field_type is not None and field_type.upper() in _STRUCT_TYPES:
-            return None
-
-        return PolicyTagList()
-
     @staticmethod
     def __get_int(api_repr, name):
         v = api_repr.get(name, _DEFAULT_VALUE)
@@ -152,10 +151,10 @@ def from_api_repr(cls, api_repr: dict) -> "SchemaField":
         mode = api_repr.get("mode", "NULLABLE")
         description = api_repr.get("description", _DEFAULT_VALUE)
         fields = api_repr.get("fields", ())
+        policy_tags = api_repr.get("policyTags", _DEFAULT_VALUE)
 
-        policy_tags = cls._determine_policy_tags(
-            field_type, PolicyTagList.from_api_repr(api_repr.get("policyTags"))
-        )
+        if policy_tags is not None and policy_tags is not _DEFAULT_VALUE:
+            policy_tags = PolicyTagList.from_api_repr(policy_tags)
 
         return cls(
             field_type=field_type,
@@ -230,7 +229,8 @@ def policy_tags(self):
         """Optional[google.cloud.bigquery.schema.PolicyTagList]: Policy tag list
         definition for this field.
         """
-        return self._policy_tags
+        resource = self._properties.get("policyTags")
+        return PolicyTagList.from_api_repr(resource) if resource is not None else None
 
     def to_api_repr(self) -> dict:
         """Return a dictionary representing this schema field.
@@ -244,10 +244,6 @@ def to_api_repr(self) -> dict:
         # add this to the serialized representation.
         if self.field_type.upper() in _STRUCT_TYPES:
             answer["fields"] = [f.to_api_repr() for f in self.fields]
-        else:
-            # Explicitly include policy tag definition (we must not do it for RECORD
-            # fields, because those are not leaf fields).
-            answer["policyTags"] = self.policy_tags.to_api_repr()
 
         # Done; return the serialized dictionary.
         return answer
@@ -272,7 +268,7 @@ def _key(self):
                     field_type = f"{field_type}({self.precision})"
 
         policy_tags = (
-            () if self._policy_tags is None else tuple(sorted(self._policy_tags.names))
+            () if self.policy_tags is None else tuple(sorted(self.policy_tags.names))
         )
 
         return (

tests/system/test_client.py

@@ -673,14 +673,15 @@ def test_unset_table_schema_attributes(self):
             mode=old_field.mode,
             description=None,
             fields=old_field.fields,
-            policy_tags=None,
+            policy_tags=PolicyTagList(),
         )
 
         table.schema = new_schema
         updated_table = Config.CLIENT.update_table(table, ["schema"])
 
         self.assertFalse(updated_table.schema[1].description)  # Empty string or None.
-        self.assertEqual(updated_table.schema[1].policy_tags.names, ())
+        # policyTags key expected to be missing from response.
+        self.assertIsNone(updated_table.schema[1].policy_tags)
 
     def test_update_table_clustering_configuration(self):
         dataset = self.temp_dataset(_make_dataset_id("update_table"))

tests/unit/job/test_load_config.py

@@ -484,13 +484,11 @@ def test_schema_setter_fields(self):
             "name": "full_name",
             "type": "STRING",
             "mode": "REQUIRED",
-            "policyTags": {"names": []},
         }
         age_repr = {
             "name": "age",
             "type": "INTEGER",
             "mode": "REQUIRED",
-            "policyTags": {"names": []},
         }
         self.assertEqual(
             config._properties["load"]["schema"], {"fields": [full_name_repr, age_repr]}
@@ -503,13 +501,11 @@ def test_schema_setter_valid_mappings_list(self):
             "name": "full_name",
             "type": "STRING",
             "mode": "REQUIRED",
-            "policyTags": {"names": []},
         }
         age_repr = {
             "name": "age",
             "type": "INTEGER",
             "mode": "REQUIRED",
-            "policyTags": {"names": []},
         }
         schema = [full_name_repr, age_repr]
         config.schema = schema