diff --git a/docs/changelog.rst b/docs/changelog.rst index ca9ad6c80..212f8fd04 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -17,6 +17,42 @@ Changes in 1.0.0 - make sure to read https://www.mongodb.com/docs/manual/core/transactions-in-applications/#callback-api-vs-core-api - run_in_transaction context manager relies on Pymongo coreAPI, it will retry automatically in case of ``UnknownTransactionCommitResult`` but not ``TransientTransactionError`` exceptions - Using .count() in a transaction will always use Collection.count_document (as estimated_document_count is not supported in transactions) + +- BREAKING CHANGE: The default UUID representation used to be ``pythonLegacy`` and is now ``unspecified``. + This prevents MongoEngine from silently choosing how to encode native + ``uuid.UUID`` values. Applications using the default binary + ``UUIDField`` must explicitly select a representation:: + + # Keep reading and writing existing Python legacy UUID data. + connect(uuidRepresentation="pythonLegacy") + + # Use this for new databases or after migrating existing UUID data. + connect(uuidRepresentation="standard") + + Recommendation is that applications with existing UUID data should use ``pythonLegacy``. + + With ``unspecified``, writing or querying with a native ``uuid.UUID`` raises + ``ValueError``. Reading a BSON UUID whose representation does not match the + configured representation raises ``ValidationError`` instead of exposing a + ``bson.Binary`` value through ``UUIDField``. ``ObjectId`` values and + ``UUIDField(binary=False)`` are unaffected. + + To migrate a normal, top-level ``UUIDField`` from Python legacy BSON subtype + 3 to standard BSON subtype 4, read through a Python legacy client and write through a standard + client so PyMongo preserves the UUID value while changing its BSON subtype. + + Migrating documents that have UUID as primary key (i.e UUIDField(primary_key=True)) is more tedious as documents' id cannot be updated in + place so you will need to recreate the documents with standard UUID primary keys and update every reference. + + Extended JSON UUID handling remains independent from the connection. The + implicit ``json_options`` used by ``Document.to_json()``, + ``Document.from_json()``, ``QuerySet.to_json()``, and ``QuerySet.from_json()`` + now also use ``UNSPECIFIED``. Without explicit JSON options, ``to_json()`` + raises ``ValueError`` for native ``uuid.UUID`` values, while ``from_json()`` + raises ``ValidationError`` when a BSON UUID cannot be decoded. Pass JSON + options with the intended UUID representation when serializing or + deserializing UUID data. + - Add a warning that ``mongoengine.org`` is no longer controlled by the MongoEngine project and appears to be an expired domain takeover. - Fix querying GenericReferenceField with __in operator #2886 diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 400fb7931..7cad5d5e9 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -1,6 +1,5 @@ import copy import numbers -import warnings from functools import partial import pymongo @@ -24,7 +23,7 @@ OperationError, ValidationError, ) -from mongoengine.pymongo_support import LEGACY_JSON_OPTIONS +from mongoengine.pymongo_support import DEFAULT_JSON_OPTIONS __all__ = ("BaseDocument", "NON_FIELD_ERRORS") @@ -447,17 +446,7 @@ def to_json(self, *args, **kwargs): Defaults to True. """ use_db_field = kwargs.pop("use_db_field", True) - if "json_options" not in kwargs: - warnings.warn( - "No 'json_options' are specified! Falling back to " - "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " - "For use with other MongoDB drivers specify the UUID " - "representation to use. This will be changed to " - "uuid_representation=UNSPECIFIED in a future release.", - DeprecationWarning, - stacklevel=2, - ) - kwargs["json_options"] = LEGACY_JSON_OPTIONS + kwargs.setdefault("json_options", DEFAULT_JSON_OPTIONS) return json_util.dumps(self.to_mongo(use_db_field), *args, **kwargs) @classmethod @@ -480,17 +469,7 @@ def from_json(cls, json_data, created=False, **kwargs): # TODO should `created` default to False? If the object already exists # in the DB, you would likely retrieve it from MongoDB itself through # a query, not load it from JSON data. - if "json_options" not in kwargs: - warnings.warn( - "No 'json_options' are specified! Falling back to " - "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " - "For use with other MongoDB drivers specify the UUID " - "representation to use. This will be changed to " - "uuid_representation=UNSPECIFIED in a future release.", - DeprecationWarning, - stacklevel=2, - ) - kwargs["json_options"] = LEGACY_JSON_OPTIONS + kwargs.setdefault("json_options", DEFAULT_JSON_OPTIONS) return cls._from_son(json_util.loads(json_data, **kwargs), created=created) def __expand_dynamic_values(self, name, value): diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 4728cb377..722a3632b 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,6 +1,5 @@ import collections import threading -import warnings from pymongo import MongoClient, ReadPreference, uri_parser from pymongo.common import _UUID_REPRESENTATIONS @@ -196,7 +195,7 @@ def _get_connection_settings( REV_UUID_REPRESENTATIONS = { v: k for k, v in _UUID_REPRESENTATIONS.items() } - conn_settings["uuidrepresentation"] = REV_UUID_REPRESENTATIONS[ + conn_settings["uuidRepresentation"] = REV_UUID_REPRESENTATIONS[ normalized_uri_options["uuidrepresentation"] ] else: @@ -207,21 +206,10 @@ def _get_connection_settings( kwargs.pop("slaves", None) kwargs.pop("is_slave", None) - keys = { - key.lower() for key in kwargs.keys() - } # pymongo options are case insensitive - if "uuidrepresentation" not in keys and "uuidrepresentation" not in conn_settings: - warnings.warn( - "No uuidRepresentation is specified! Falling back to " - "'pythonLegacy' which is the default for pymongo 3.x. " - "For compatibility with other MongoDB drivers this should be " - "specified as 'standard' or '{java,csharp}Legacy' to work with " - "older drivers in those languages. This will be changed to " - "'unspecified' in a future release.", - DeprecationWarning, - stacklevel=3, - ) - kwargs["uuidRepresentation"] = "pythonLegacy" + for key in tuple(kwargs): + if key.lower() == "uuidrepresentation": + conn_settings["uuidRepresentation"] = kwargs.pop(key) + conn_settings.setdefault("uuidRepresentation", "unspecified") conn_settings.update(kwargs) return conn_settings diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0f5ee5402..549d4b039 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -13,6 +13,7 @@ import gridfs import pymongo from bson import SON, Binary, DBRef, ObjectId +from bson.binary import OLD_UUID_SUBTYPE, UUID_SUBTYPE from bson.decimal128 import Decimal128, create_decimal128_context from pymongo import ReturnDocument @@ -2220,6 +2221,16 @@ def to_python(self, value): return uuid.UUID(value) except (ValueError, TypeError, AttributeError): return original_value + elif isinstance(value, Binary) and value.subtype in ( + OLD_UUID_SUBTYPE, + UUID_SUBTYPE, + ): + # The default uuidRepresentation was switched to UNSPECIFIED in MongoEngine 1.0. + # This error may mean the user did not handle the required migration. + self.error( + "BSON UUID could not be decoded to a native uuid.UUID. " + "Configure uuidRepresentation to match the stored UUID representation." + ) return value def to_mongo(self, value): diff --git a/mongoengine/pymongo_support.py b/mongoengine/pymongo_support.py index 3c819610f..24a333e91 100644 --- a/mongoengine/pymongo_support.py +++ b/mongoengine/pymongo_support.py @@ -10,14 +10,9 @@ PYMONGO_VERSION = tuple(pymongo.version_tuple[:2]) -# This will be changed to UuidRepresentation.UNSPECIFIED in a future -# (breaking) release. -if PYMONGO_VERSION >= (4,): - LEGACY_JSON_OPTIONS = json_util.LEGACY_JSON_OPTIONS.with_options( - uuid_representation=binary.UuidRepresentation.PYTHON_LEGACY, - ) -else: - LEGACY_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS +DEFAULT_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS.with_options( + uuid_representation=binary.UuidRepresentation.UNSPECIFIED, +) def count_documents( diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 0242d92e0..15209e7f4 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -30,7 +30,7 @@ OperationError, ) from mongoengine.pymongo_support import ( - LEGACY_JSON_OPTIONS, + DEFAULT_JSON_OPTIONS, count_documents, ) from mongoengine.queryset import transform @@ -1336,22 +1336,13 @@ def max_time_ms(self, ms): def to_json(self, *args, **kwargs): """Converts a queryset to JSON""" - if "json_options" not in kwargs: - warnings.warn( - "No 'json_options' are specified! Falling back to " - "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " - "For use with other MongoDB drivers specify the UUID " - "representation to use. This will be changed to " - "uuid_representation=UNSPECIFIED in a future release.", - DeprecationWarning, - stacklevel=2, - ) - kwargs["json_options"] = LEGACY_JSON_OPTIONS + kwargs.setdefault("json_options", DEFAULT_JSON_OPTIONS) return json_util.dumps(self.as_pymongo(), *args, **kwargs) - def from_json(self, json_data): + def from_json(self, json_data, **kwargs): """Converts json data to unsaved objects""" - son_data = json_util.loads(json_data) + kwargs.setdefault("json_options", DEFAULT_JSON_OPTIONS) + son_data = json_util.loads(json_data, **kwargs) return [self._document._from_son(data) for data in son_data] def aggregate(self, pipeline, **kwargs): diff --git a/setup.cfg b/setup.cfg index cadc59a1b..9723544ca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,6 @@ max-complexity=47 # avoids that it runs for instance the benchmark testpaths = tests filterwarnings = - ignore:No uuidRepresentation is specified!:DeprecationWarning ignore:Multiple Document classes named .* were registered:UserWarning [isort] diff --git a/tests/document/test_json_serialisation.py b/tests/document/test_json_serialisation.py index a8d573548..901237c4f 100644 --- a/tests/document/test_json_serialisation.py +++ b/tests/document/test_json_serialisation.py @@ -2,11 +2,17 @@ import uuid from datetime import datetime -from bson import ObjectId +import pytest +from bson import ObjectId, json_util +from bson.binary import UuidRepresentation from mongoengine import * from tests.utils import MongoDBTestCase +PYTHON_LEGACY_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS.with_options( + uuid_representation=UuidRepresentation.PYTHON_LEGACY, +) + class TestJson(MongoDBTestCase): def test_json_names(self): @@ -96,10 +102,29 @@ class Doc(Document): def __eq__(self, other): import json - return json.loads(self.to_json()) == json.loads(other.to_json()) + return json.loads( + self.to_json(json_options=PYTHON_LEGACY_JSON_OPTIONS) + ) == json.loads(other.to_json(json_options=PYTHON_LEGACY_JSON_OPTIONS)) doc = Doc() - assert doc == Doc.from_json(doc.to_json()) + json_data = doc.to_json(json_options=PYTHON_LEGACY_JSON_OPTIONS) + assert doc == Doc.from_json( + json_data, + json_options=PYTHON_LEGACY_JSON_OPTIONS, + ) + + def test_json_uuid__representation_is_omitted__fails(self): + class Doc(Document): + uuid_field = UUIDField() + + doc = Doc(uuid_field=uuid.uuid4()) + + with pytest.raises(ValueError, match="cannot encode native uuid.UUID"): + doc.to_json() + + legacy_json = doc.to_json(json_options=PYTHON_LEGACY_JSON_OPTIONS) + with pytest.raises(ValidationError, match="BSON UUID"): + Doc.from_json(legacy_json) if __name__ == "__main__": diff --git a/tests/fields/test_uuid_field.py b/tests/fields/test_uuid_field.py index ec81033b0..f1bd70bc0 100644 --- a/tests/fields/test_uuid_field.py +++ b/tests/fields/test_uuid_field.py @@ -1,9 +1,56 @@ import uuid import pytest +from bson.binary import UuidRepresentation from mongoengine import * -from tests.utils import MongoDBTestCase, get_as_pymongo +from mongoengine.connection import disconnect, get_connection, get_db +from tests.utils import MONGO_TEST_DB, MongoDBTestCase, get_as_pymongo + +LEGACY_ALIAS = "uuid-python-legacy" +STANDARD_ALIAS = "uuid-standard" +UNSPECIFIED_ALIAS = "uuid-unspecified" +UUID_COLLECTION = "uuid_representation" +UUID_PRIMARY_KEY_COLLECTION = "uuid_primary_key_representation" + + +class LegacyUUIDDocument(Document): + identifier = UUIDField() + + meta = {"collection": UUID_COLLECTION, "db_alias": LEGACY_ALIAS} + + +class StandardUUIDDocument(Document): + identifier = UUIDField() + + meta = {"collection": UUID_COLLECTION, "db_alias": STANDARD_ALIAS} + + +class UnspecifiedUUIDDocument(Document): + identifier = UUIDField() + + meta = {"collection": UUID_COLLECTION, "db_alias": UNSPECIFIED_ALIAS} + + +class UnspecifiedStringUUIDDocument(Document): + identifier = UUIDField(binary=False) + + meta = {"collection": "uuid_string_representation", "db_alias": UNSPECIFIED_ALIAS} + + +class LegacyUUIDPrimaryKeyDocument(Document): + id = UUIDField(primary_key=True) + + meta = {"collection": UUID_PRIMARY_KEY_COLLECTION, "db_alias": LEGACY_ALIAS} + + +class UnspecifiedUUIDPrimaryKeyDocument(Document): + id = UUIDField(primary_key=True) + + meta = { + "collection": UUID_PRIMARY_KEY_COLLECTION, + "db_alias": UNSPECIFIED_ALIAS, + } class Person(Document): @@ -63,3 +110,88 @@ def test_field_binary(self): person.api_key = api_key with pytest.raises(ValidationError): person.validate() + + +class TestUUIDRepresentation(MongoDBTestCase): + def setUp(self): + connect( + db=MONGO_TEST_DB, + alias=LEGACY_ALIAS, + uuidRepresentation="pythonLegacy", + ) + connect( + db=MONGO_TEST_DB, + alias=STANDARD_ALIAS, + uuidRepresentation="standard", + ) + connect(db=MONGO_TEST_DB, alias=UNSPECIFIED_ALIAS) + + LegacyUUIDDocument.drop_collection() + LegacyUUIDPrimaryKeyDocument.drop_collection() + UnspecifiedStringUUIDDocument.drop_collection() + + self.identifier = uuid.uuid4() + self.document = LegacyUUIDDocument(identifier=self.identifier).save() + + def tearDown(self): + LegacyUUIDDocument.drop_collection() + LegacyUUIDPrimaryKeyDocument.drop_collection() + UnspecifiedStringUUIDDocument.drop_collection() + disconnect(LEGACY_ALIAS) + disconnect(STANDARD_ALIAS) + disconnect(UNSPECIFIED_ALIAS) + + def test_unspecified__legacy_uuid_exists__fails_on_read_write_and_query(self): + connection = get_connection(UNSPECIFIED_ALIAS) + assert ( + connection.options.codec_options.uuid_representation + == UuidRepresentation.UNSPECIFIED + ) + + with pytest.raises(ValidationError, match="BSON UUID"): + UnspecifiedUUIDDocument.objects.first() + + with pytest.raises(ValueError, match="cannot encode native uuid.UUID"): + UnspecifiedUUIDDocument(identifier=uuid.uuid4()).save() + + with pytest.raises(ValueError, match="cannot encode native uuid.UUID"): + UnspecifiedUUIDDocument.objects(identifier=self.identifier).first() + + def test_python_legacy__legacy_uuid_exists__reads_uuid(self): + document = LegacyUUIDDocument.objects.get(id=self.document.id) + + assert document.identifier == self.identifier + assert isinstance(document.identifier, uuid.UUID) + + def test_standard__legacy_uuid_is_migrated__reads_uuid(self): + with pytest.raises(ValidationError, match="BSON UUID"): + StandardUUIDDocument.objects.first() + + legacy_collection = get_db(LEGACY_ALIAS)[UUID_COLLECTION] + standard_collection = get_db(STANDARD_ALIAS)[UUID_COLLECTION] + legacy_document = legacy_collection.find_one({"_id": self.document.id}) + standard_collection.update_one( + {"_id": self.document.id}, + {"$set": {"identifier": legacy_document["identifier"]}}, + ) + + migrated_document = StandardUUIDDocument.objects.get(id=self.document.id) + assert migrated_document.identifier == self.identifier + assert isinstance(migrated_document.identifier, uuid.UUID) + + def test_unspecified__uuid_is_stored_as_string__reads_and_writes_uuid(self): + document = UnspecifiedStringUUIDDocument(identifier=self.identifier).save() + + assert ( + UnspecifiedStringUUIDDocument.objects.get(id=document.id).identifier + == self.identifier + ) + + def test_unspecified__legacy_uuid_primary_key_exists__fails_on_read(self): + LegacyUUIDPrimaryKeyDocument(id=self.identifier).save() + + with pytest.raises(ValidationError, match="BSON UUID"): + UnspecifiedUUIDPrimaryKeyDocument.objects.first() + + with pytest.raises(ValueError, match="cannot encode native uuid.UUID"): + UnspecifiedUUIDPrimaryKeyDocument.objects.get(id=self.identifier) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index d64384670..ce5298e8d 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -5,7 +5,8 @@ import pymongo import pytest -from bson import DBRef, ObjectId +from bson import DBRef, ObjectId, json_util +from bson.binary import UuidRepresentation from pymongo.read_preferences import ReadPreference from pymongo.results import UpdateResult @@ -43,7 +44,7 @@ def get_key_compat(mongo_ver): class TestQueryset(unittest.TestCase): def setUp(self): - connect(db="mongoenginetest") + connect(db="mongoenginetest", uuidRepresentation="pythonLegacy") connect(db="mongoenginetest2", alias="test2") class PersonMeta(EmbeddedDocument): @@ -5226,10 +5227,19 @@ class Doc(Document): Doc.drop_collection() Doc().save() - json_data = Doc.objects.to_json() + with pytest.raises(ValueError, match="cannot encode native uuid.UUID"): + Doc.objects.to_json() + + json_options = json_util.DEFAULT_JSON_OPTIONS.with_options( + uuid_representation=UuidRepresentation.PYTHON_LEGACY, + ) + json_data = Doc.objects.to_json(json_options=json_options) doc_objects = list(Doc.objects) - assert doc_objects == Doc.objects.from_json(json_data) + assert doc_objects == Doc.objects.from_json( + json_data, + json_options=json_options, + ) def test_as_pymongo(self): class LastLogin(EmbeddedDocument): diff --git a/tests/test_connection.py b/tests/test_connection.py index b41ef5826..3055a97aa 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -210,7 +210,7 @@ def test___get_connection_settings(self): "read_preference": read_pref, "replicaSet": "s0", "username": "root", - "uuidrepresentation": "javaLegacy", + "uuidRepresentation": "javaLegacy", } def test_connect_passes_silently_connect_multiple_times_with_same_config(self): @@ -682,6 +682,18 @@ def test_connect_2_databases_uses_different_client_if_different_parameters(self) c2 = connect(alias="testdb2", db="testdb2", username="u2", password="pass") assert c1 is not c2 + def test_connect__equivalent_uuidrepresentation_options__reuses_alias(self): + rand = random_str() + host = f"mongodb://localhost:27017/{rand}?uuidRepresentation=standard" + + first_connection = connect(alias=rand, host=host) + second_connection = connect( + alias=rand, host=host, uuidRepresentation="standard" + ) + + assert second_connection is first_connection + disconnect(rand) + def test_connect_uri_uuidrepresentation_set_in_uri(self): rand = random_str() tmp_conn = connect( @@ -716,14 +728,12 @@ def test_connect_uri_uuidrepresentation_set_both_arg_and_uri_arg_prevail(self): ) disconnect(rand) - def test_connect_uri_uuidrepresentation_default_to_pythonlegacy(self): - # To be changed soon to unspecified + def test_connect_uuidrepresentation_defaults_to_unspecified(self): rand = random_str() - with pytest.warns(DeprecationWarning, match="No uuidRepresentation"): - tmp_conn = connect(alias=rand, db=rand) + tmp_conn = connect(alias=rand, db=rand) assert ( tmp_conn.options.codec_options.uuid_representation - == pymongo.common._UUID_REPRESENTATIONS["pythonLegacy"] + == pymongo.common._UUID_REPRESENTATIONS["unspecified"] ) disconnect(rand) diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index 1333f5574..e03fa73cd 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -599,8 +599,7 @@ class A(Document): @requires_mongodb_gte_40 def test_transaction_updates_across_databases(self): - connect("mongoenginetest") - connect("test2", "test2") + connect("test2", "test2", uuidRepresentation="pythonLegacy") class A(Document): name = StringField() @@ -624,8 +623,7 @@ class B(Document): @requires_mongodb_gte_44 def test_collection_creation_via_upserts_across_databases_in_transaction(self): - connect("mongoenginetest") - connect("test2", "test2") + connect("test2", "test2", uuidRepresentation="pythonLegacy") class A(Document): name = StringField() @@ -658,8 +656,7 @@ class B(Document): def test_an_exception_raised_in_transactions_across_databases_rolls_back_updates( self, ): - connect("mongoenginetest") - connect("test2", "test2") + connect("test2", "test2", uuidRepresentation="pythonLegacy") class A(Document): name = StringField()