Fix DurationField, IntegerField unique constraints for values < or > 32 bits

Whether values less than or greater than 32 bits were problematic
depends on the field.

Author: timgraham

django_mongodb_backend/base.py

@@ -46,11 +46,11 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         "FileField": "string",
         "FilePathField": "string",
         "FloatField": "double",
-        "IntegerField": "int",
+        "IntegerField": "long",
         "BigIntegerField": "long",
         "GenericIPAddressField": "string",
         "JSONField": "object",
-        "PositiveBigIntegerField": "int",
+        "PositiveBigIntegerField": "long",
         "PositiveIntegerField": "long",
         "PositiveSmallIntegerField": "int",
         "SlugField": "string",

django_mongodb_backend/fields/duration.py

@@ -1,3 +1,4 @@
+from bson import Int64
 from django.db.models.fields import DurationField
 
 _get_db_prep_value = DurationField.get_db_prep_value
@@ -8,6 +9,8 @@ def get_db_prep_value(self, value, connection, prepared=False):
     value = _get_db_prep_value(self, value, connection, prepared)
     if connection.vendor == "mongodb" and value is not None:
         value //= 1000
+        # Store value as Int64 (long).
+        value = Int64(value)
     return value
 
 

django_mongodb_backend/operations.py

@@ -4,7 +4,7 @@
 import uuid
 from decimal import Decimal
 
-from bson.decimal128 import Decimal128
+from bson import Decimal128, Int64
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 from django.db import DataError
@@ -66,6 +66,14 @@ def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None):
             return None
         return Decimal128(value)
 
+    def adapt_integerfield_value(self, value, internal_type):
+        """Store all IntegerField variants as Int64 (long)."""
+        if value is None:
+            return None
+        if "Small" not in internal_type:
+            return Int64(value)
+        return value
+
     def adapt_json_value(self, value, encoder):
         if encoder is None:
             return value
@@ -131,8 +139,16 @@ def get_db_converters(self, expression):
                 converters.append(self.convert_timefield_value)
         elif internal_type == "UUIDField":
             converters.append(self.convert_uuidfield_value)
+        elif "IntegerField" in internal_type and "Small" not in internal_type:
+            converters.append(self.convert_integerfield_value)
         return converters
 
+    def convert_integerfield_value(self, value, expression, connection):
+        if value is not None:
+            # from Int64 to int
+            value = int(value)
+        return value
+
     def convert_datefield_value(self, value, expression, connection):
         if value is not None:
             value = value.date()

docs/source/ref/models/fields.rst

@@ -27,6 +27,17 @@ A few notes about some of the other fields:
   prohibit inserting values outside of the supported range and unique
   constraints don't work for values outside of the 32-bit range of the BSON
   ``int`` type.
+- :class:`~django.db.models.IntegerField`,
+  :class:`~django.db.models.BigIntegerField` and
+  :class:`~django.db.models.PositiveSmallIntegerField`, and
+  :class:`~django.db.models.PositiveBigIntegerField` support 64 bit values
+  (ranges ``(-9223372036854775808, 9223372036854775807)`` and
+  ``(0, 9223372036854775807)``, respectively), validated by forms and model
+  validation. If you're inserting data outside of the ORM, you must cast all
+  values to :class:`bson.int64.Int64`, otherwise values less then 32 bits will
+  be stored as ``int`` and won't be validated by unique constraints.
+- Similarly, all :class:`~django.db.models.DurationField` values are stored as
+  :class:`bson.int64.Int64`.
 
 MongoDB-specific model fields
 =============================

docs/source/releases/5.2.x.rst

@@ -36,6 +36,17 @@ Bug fixes
   databases.
 - :meth:`QuerySet.explain() <django.db.models.query.QuerySet.explain>` now
   :ref:`returns a string that can be parsed as JSON <queryset-explain>`.
+- Fixed unique constraint generation for :class:`~django.db.models.IntegerField`
+  and :class:`~django.db.models.PositiveBigIntegerField`, which incorrectly
+  allowed duplicate values larger than 32 bits. Existing unique constraints
+  must be recreated to use ``$type: long`` instead of ``int``.
+- :class:`~django.db.models.IntegerField`,
+  :class:`~django.db.models.BigIntegerField` (as well as the
+  ``Positive`` versions of these fields), and
+  :class:`~django.db.models.DurationField` values are now sent to MongoDB as
+  :class:`bson.int64.Int64`, which fixes unique constraints on values less than
+  32 bits for ``BigIntegerField``, ``PositiveIntegerField``, and
+  ``DurationField``. Existing data must be converted to ``Int64``.
 
 Performance improvements
 ------------------------

tests/model_fields_/models.py

@@ -15,7 +15,12 @@
 
 class UniqueIntegers(models.Model):
     small = models.SmallIntegerField(unique=True, blank=True, null=True)
+    plain = models.IntegerField(unique=True, blank=True, null=True)
+    big = models.BigIntegerField(unique=True, blank=True, null=True)
     positive_small = models.PositiveSmallIntegerField(unique=True, blank=True, null=True)
+    positive = models.PositiveIntegerField(unique=True, blank=True, null=True)
+    positive_big = models.PositiveBigIntegerField(unique=True, blank=True, null=True)
+    duration = models.DurationField(unique=True, blank=True, null=True)
 
 
 # ObjectIdField

tests/model_fields_/test_durationfield.py

@@ -0,0 +1,27 @@
+from datetime import timedelta
+
+from django.db import IntegrityError
+from django.test import TestCase
+
+from .models import Integers
+
+
+class UniqueTests(TestCase):
+    def test_small_value(self):
+        """
+        Duplicate values < 32 bits are prohibited. This confirms DurationField values
+        are cast to Int64 so MongoDB stores them as long. Otherwise, the
+        partialFilterExpression: {$type: long} unique constraint doesn't work.
+        """
+        Integers.objects.create(duration=timedelta(1))
+        with self.assertRaises(IntegrityError):
+            Integers.objects.create(duration=timedelta(1))
+
+    def test_large_value(self):
+        """
+        Duplicate values > 32 bits are prohibited. This confirms DurationField uses the
+        long db_type() rather than the 32 bit int type.
+        """
+        Integers.objects.create(duration=timedelta(1000000))
+        with self.assertRaises(IntegrityError):
+            Integers.objects.create(duration=timedelta(1000000))

tests/model_fields_/test_integerfield.py

@@ -73,3 +73,62 @@ def test_validate_min_value(self):
         msg = "{'positive_small': ['Ensure this value is greater than or equal to 0.']"
         with self.assertRaisesMessage(ValidationError, msg):
             UniqueIntegers(positive_small=self.min_value - 1).full_clean()
+
+
+class SmallUniqueTests(TestCase):
+    """
+    Duplicate values < 32 bits are prohibited. This confirms integer field
+    values are cast to Int64 so MongoDB stores it as long. Otherwise, the
+    partialFilterExpression: {$type: long} unique constraint doesn't work.
+    """
+
+    test_value = 123
+
+    def test_integerfield(self):
+        UniqueIntegers.objects.create(plain=self.test_value)
+        with self.assertRaises(IntegrityError):
+            UniqueIntegers.objects.create(plain=self.test_value)
+
+    def test_bigintegerfield(self):
+        UniqueIntegers.objects.create(big=self.test_value)
+        with self.assertRaises(IntegrityError):
+            UniqueIntegers.objects.create(big=self.test_value)
+
+    def test_positiveintegerfield(self):
+        UniqueIntegers.objects.create(positive=self.test_value)
+        with self.assertRaises(IntegrityError):
+            UniqueIntegers.objects.create(positive=self.test_value)
+
+    def test_positivebigintegerfield(self):
+        UniqueIntegers.objects.create(positive_big=self.test_value)
+        with self.assertRaises(IntegrityError):
+            UniqueIntegers.objects.create(positive_big=self.test_value)
+
+
+class LargeUniqueTests(TestCase):
+    """
+    Duplicate values > 32 bits are prohibited. This confirms each field uses
+    the long db_type() rather than the 32 bit int type.
+    """
+
+    test_value = 2**63 - 1
+
+    def test_integerfield(self):
+        UniqueIntegers.objects.create(plain=self.test_value)
+        with self.assertRaises(IntegrityError):
+            UniqueIntegers.objects.create(plain=self.test_value)
+
+    def test_bigintegerfield(self):
+        UniqueIntegers.objects.create(big=self.test_value)
+        with self.assertRaises(IntegrityError):
+            UniqueIntegers.objects.create(big=self.test_value)
+
+    def test_positiveintegerfield(self):
+        UniqueIntegers.objects.create(positive=self.test_value)
+        with self.assertRaises(IntegrityError):
+            UniqueIntegers.objects.create(positive=self.test_value)
+
+    def test_positivebigintegerfield(self):
+        UniqueIntegers.objects.create(positive_big=self.test_value)
+        with self.assertRaises(IntegrityError):
+            UniqueIntegers.objects.create(positive_big=self.test_value)