Switch uuidRepresentation to 'unspecified' - aimed for 1.0

docs/changelog.rst

@@ -17,6 +17,42 @@ Changes in 1.0.0
     - make sure to read https://www.mongodb.com/docs/manual/core/transactions-in-applications/#callback-api-vs-core-api
     - run_in_transaction context manager relies on Pymongo coreAPI, it will retry automatically in case of ``UnknownTransactionCommitResult`` but not ``TransientTransactionError`` exceptions
     - Using .count() in a transaction will always use Collection.count_document (as estimated_document_count is not supported in transactions)
+
+- BREAKING CHANGE: The default UUID representation used to be ``pythonLegacy`` and is now ``unspecified``.
+  This prevents MongoEngine from silently choosing how to encode native
+  ``uuid.UUID`` values. Applications using the default binary
+  ``UUIDField`` must explicitly select a representation::
+
+      # Keep reading and writing existing Python legacy UUID data.
+      connect(uuidRepresentation="pythonLegacy")
+
+      # Use this for new databases or after migrating existing UUID data.
+      connect(uuidRepresentation="standard")
+
+  Recommendation is that applications with existing UUID data should use ``pythonLegacy``.
+
+  With ``unspecified``, writing or querying with a native ``uuid.UUID`` raises
+  ``ValueError``. Reading a BSON UUID whose representation does not match the
+  configured representation raises ``ValidationError`` instead of exposing a
+  ``bson.Binary`` value through ``UUIDField``. ``ObjectId`` values and
+  ``UUIDField(binary=False)`` are unaffected.
+
+  To migrate a normal, top-level ``UUIDField`` from Python legacy BSON subtype
+  3 to standard BSON subtype 4, read through a Python legacy client and write through a standard
+  client so PyMongo preserves the UUID value while changing its BSON subtype.
+
+  Migrating documents that have UUID as primary key (i.e UUIDField(primary_key=True)) is more tedious as documents' id cannot be updated in
+  place so you will need to recreate the documents with standard UUID primary keys and update every reference.
+
+  Extended JSON UUID handling remains independent from the connection. The
+  implicit ``json_options`` used by ``Document.to_json()``,
+  ``Document.from_json()``, ``QuerySet.to_json()``, and ``QuerySet.from_json()``
+  now also use ``UNSPECIFIED``. Without explicit JSON options, ``to_json()``
+  raises ``ValueError`` for native ``uuid.UUID`` values, while ``from_json()``
+  raises ``ValidationError`` when a BSON UUID cannot be decoded. Pass JSON
+  options with the intended UUID representation when serializing or
+  deserializing UUID data.
+
 - Add a warning that ``mongoengine.org`` is no longer controlled by the MongoEngine
   project and appears to be an expired domain takeover.
 - Fix querying GenericReferenceField with __in operator #2886

mongoengine/base/document.py

@@ -1,6 +1,5 @@
 import copy
 import numbers
-import warnings
 from functools import partial
 
 import pymongo
@@ -24,7 +23,7 @@
     OperationError,
     ValidationError,
 )
-from mongoengine.pymongo_support import LEGACY_JSON_OPTIONS
+from mongoengine.pymongo_support import DEFAULT_JSON_OPTIONS
 
 __all__ = ("BaseDocument", "NON_FIELD_ERRORS")
 
@@ -447,17 +446,7 @@ def to_json(self, *args, **kwargs):
             Defaults to True.
         """
         use_db_field = kwargs.pop("use_db_field", True)
-        if "json_options" not in kwargs:
-            warnings.warn(
-                "No 'json_options' are specified! Falling back to "
-                "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. "
-                "For use with other MongoDB drivers specify the UUID "
-                "representation to use. This will be changed to "
-                "uuid_representation=UNSPECIFIED in a future release.",
-                DeprecationWarning,
-                stacklevel=2,
-            )
-            kwargs["json_options"] = LEGACY_JSON_OPTIONS
+        kwargs.setdefault("json_options", DEFAULT_JSON_OPTIONS)
         return json_util.dumps(self.to_mongo(use_db_field), *args, **kwargs)
 
     @classmethod
@@ -480,17 +469,7 @@ def from_json(cls, json_data, created=False, **kwargs):
         # TODO should `created` default to False? If the object already exists
         # in the DB, you would likely retrieve it from MongoDB itself through
         # a query, not load it from JSON data.
-        if "json_options" not in kwargs:
-            warnings.warn(
-                "No 'json_options' are specified! Falling back to "
-                "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. "
-                "For use with other MongoDB drivers specify the UUID "
-                "representation to use. This will be changed to "
-                "uuid_representation=UNSPECIFIED in a future release.",
-                DeprecationWarning,
-                stacklevel=2,
-            )
-            kwargs["json_options"] = LEGACY_JSON_OPTIONS
+        kwargs.setdefault("json_options", DEFAULT_JSON_OPTIONS)
         return cls._from_son(json_util.loads(json_data, **kwargs), created=created)
 
     def __expand_dynamic_values(self, name, value):

mongoengine/connection.py

@@ -1,6 +1,5 @@
 import collections
 import threading
-import warnings
 
 from pymongo import MongoClient, ReadPreference, uri_parser
 from pymongo.common import _UUID_REPRESENTATIONS
@@ -196,7 +195,7 @@ def _get_connection_settings(
                 REV_UUID_REPRESENTATIONS = {
                     v: k for k, v in _UUID_REPRESENTATIONS.items()
                 }
-                conn_settings["uuidrepresentation"] = REV_UUID_REPRESENTATIONS[
+                conn_settings["uuidRepresentation"] = REV_UUID_REPRESENTATIONS[
                     normalized_uri_options["uuidrepresentation"]
                 ]
         else:
@@ -207,21 +206,10 @@ def _get_connection_settings(
     kwargs.pop("slaves", None)
     kwargs.pop("is_slave", None)
 
-    keys = {
-        key.lower() for key in kwargs.keys()
-    }  # pymongo options are case insensitive
-    if "uuidrepresentation" not in keys and "uuidrepresentation" not in conn_settings:
-        warnings.warn(
-            "No uuidRepresentation is specified! Falling back to "
-            "'pythonLegacy' which is the default for pymongo 3.x. "
-            "For compatibility with other MongoDB drivers this should be "
-            "specified as 'standard' or '{java,csharp}Legacy' to work with "
-            "older drivers in those languages. This will be changed to "
-            "'unspecified' in a future release.",
-            DeprecationWarning,
-            stacklevel=3,
-        )
-        kwargs["uuidRepresentation"] = "pythonLegacy"
+    for key in tuple(kwargs):
+        if key.lower() == "uuidrepresentation":
+            conn_settings["uuidRepresentation"] = kwargs.pop(key)
+    conn_settings.setdefault("uuidRepresentation", "unspecified")
 
     conn_settings.update(kwargs)
     return conn_settings

mongoengine/fields.py

@@ -13,6 +13,7 @@
 import gridfs
 import pymongo
 from bson import SON, Binary, DBRef, ObjectId
+from bson.binary import OLD_UUID_SUBTYPE, UUID_SUBTYPE
 from bson.decimal128 import Decimal128, create_decimal128_context
 from pymongo import ReturnDocument
 
@@ -2220,6 +2221,16 @@ def to_python(self, value):
                 return uuid.UUID(value)
             except (ValueError, TypeError, AttributeError):
                 return original_value
+        elif isinstance(value, Binary) and value.subtype in (
+            OLD_UUID_SUBTYPE,
+            UUID_SUBTYPE,
+        ):
+            # The default uuidRepresentation was switched to UNSPECIFIED in MongoEngine 1.0.
+            # This error may mean the user did not handle the required migration.
+            self.error(
+                "BSON UUID could not be decoded to a native uuid.UUID. "
+                "Configure uuidRepresentation to match the stored UUID representation."
+            )
         return value
 
     def to_mongo(self, value):

mongoengine/pymongo_support.py

@@ -10,14 +10,9 @@
 
 PYMONGO_VERSION = tuple(pymongo.version_tuple[:2])
 
-# This will be changed to UuidRepresentation.UNSPECIFIED in a future
-# (breaking) release.
-if PYMONGO_VERSION >= (4,):
-    LEGACY_JSON_OPTIONS = json_util.LEGACY_JSON_OPTIONS.with_options(
-        uuid_representation=binary.UuidRepresentation.PYTHON_LEGACY,
-    )
-else:
-    LEGACY_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS
+DEFAULT_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS.with_options(
+    uuid_representation=binary.UuidRepresentation.UNSPECIFIED,
+)
 
 
 def count_documents(

mongoengine/queryset/base.py

@@ -30,7 +30,7 @@
     OperationError,
 )
 from mongoengine.pymongo_support import (
-    LEGACY_JSON_OPTIONS,
+    DEFAULT_JSON_OPTIONS,
     count_documents,
 )
 from mongoengine.queryset import transform
@@ -1336,22 +1336,13 @@ def max_time_ms(self, ms):
 
     def to_json(self, *args, **kwargs):
         """Converts a queryset to JSON"""
-        if "json_options" not in kwargs:
-            warnings.warn(
-                "No 'json_options' are specified! Falling back to "
-                "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. "
-                "For use with other MongoDB drivers specify the UUID "
-                "representation to use. This will be changed to "
-                "uuid_representation=UNSPECIFIED in a future release.",
-                DeprecationWarning,
-                stacklevel=2,
-            )
-            kwargs["json_options"] = LEGACY_JSON_OPTIONS
+        kwargs.setdefault("json_options", DEFAULT_JSON_OPTIONS)
         return json_util.dumps(self.as_pymongo(), *args, **kwargs)
 
-    def from_json(self, json_data):
+    def from_json(self, json_data, **kwargs):
         """Converts json data to unsaved objects"""
-        son_data = json_util.loads(json_data)
+        kwargs.setdefault("json_options", DEFAULT_JSON_OPTIONS)
+        son_data = json_util.loads(json_data, **kwargs)
         return [self._document._from_son(data) for data in son_data]
 
     def aggregate(self, pipeline, **kwargs):

setup.cfg

@@ -8,7 +8,6 @@ max-complexity=47
 # avoids that it runs for instance the benchmark
 testpaths = tests
 filterwarnings =
-    ignore:No uuidRepresentation is specified!:DeprecationWarning
     ignore:Multiple Document classes named .* were registered:UserWarning
 
 [isort]

tests/document/test_json_serialisation.py

@@ -2,11 +2,17 @@
 import uuid
 from datetime import datetime
 
-from bson import ObjectId
+import pytest
+from bson import ObjectId, json_util
+from bson.binary import UuidRepresentation
 
 from mongoengine import *
 from tests.utils import MongoDBTestCase
 
+PYTHON_LEGACY_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS.with_options(
+    uuid_representation=UuidRepresentation.PYTHON_LEGACY,
+)
+
 
 class TestJson(MongoDBTestCase):
     def test_json_names(self):
@@ -96,10 +102,29 @@ class Doc(Document):
             def __eq__(self, other):
                 import json
 
-                return json.loads(self.to_json()) == json.loads(other.to_json())
+                return json.loads(
+                    self.to_json(json_options=PYTHON_LEGACY_JSON_OPTIONS)
+                ) == json.loads(other.to_json(json_options=PYTHON_LEGACY_JSON_OPTIONS))
 
         doc = Doc()
-        assert doc == Doc.from_json(doc.to_json())
+        json_data = doc.to_json(json_options=PYTHON_LEGACY_JSON_OPTIONS)
+        assert doc == Doc.from_json(
+            json_data,
+            json_options=PYTHON_LEGACY_JSON_OPTIONS,
+        )
+
+    def test_json_uuid__representation_is_omitted__fails(self):
+        class Doc(Document):
+            uuid_field = UUIDField()
+
+        doc = Doc(uuid_field=uuid.uuid4())
+
+        with pytest.raises(ValueError, match="cannot encode native uuid.UUID"):
+            doc.to_json()
+
+        legacy_json = doc.to_json(json_options=PYTHON_LEGACY_JSON_OPTIONS)
+        with pytest.raises(ValidationError, match="BSON UUID"):
+            Doc.from_json(legacy_json)
 
 
 if __name__ == "__main__":