Fix for issue pandas-dev#21150, using a simple lock to prevent an issue with multiple threads accessing an Index

Author: kinow

pandas/_libs/index.pyx

@@ -26,6 +26,18 @@ from pandas._libs import algos, hashtable as _hash
 from pandas._libs.tslibs import Timestamp, Timedelta, period as periodlib
 from pandas._libs.missing import checknull
 
+# Python 2 vs Python 3
+try:
+    from thread import allocate_lock as _thread_allocate_lock
+except:
+    try:
+        from _thread import allocate_lock as _thread_allocate_lock
+    except:
+        try:
+            from dummy_thread import allocate_lock as _thread_allocate_lock
+        except:
+            from _dummy_thread import allocate_lock as _thread_allocate_lock
+
 cdef int64_t iNaT = util.get_nat()
 
 
@@ -72,6 +84,9 @@ cpdef object get_value_box(ndarray arr, object loc):
 # Don't populate hash tables in monotonic indexes larger than this
 _SIZE_CUTOFF = 1000000
 
+# Used in _ensure_mapping_populated to ensure is_unique behaves correctly
+# in multi-threaded code, see gh-21150
+_mapping_populated_lock = _thread_allocate_lock()
 
 cdef class IndexEngine:
 
@@ -258,17 +273,17 @@ cdef class IndexEngine:
 
     cdef inline _ensure_mapping_populated(self):
         # this populates the mapping
-        # if its not already populated
+        # if it is not already populated
         # also satisfies the need_unique_check
 
-        if not self.is_mapping_populated:
-
-            values = self._get_index_values()
-            self.mapping = self._make_hash_table(len(values))
-            self._call_map_locations(values)
+        with _mapping_populated_lock:
+            if not self.is_mapping_populated:
+                values = self._get_index_values()
+                self.mapping = self._make_hash_table(len(values))
+                self._call_map_locations(values)
 
-            if len(self.mapping) == len(values):
-                self.unique = 1
+                if len(self.mapping) == len(values):
+                    self.unique = 1
 
         self.need_unique_check = 0
 

pandas/tests/indexes/test_threadsafety.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+Testing that indexes (and its engine) work correclty with multiple threads
+"""
+
+import pytest
+import pandas as pd
+
+from concurrent.futures import ThreadPoolExecutor
+
+
+class TestThreadSafety(object):
+
+    @pytest.mark.slow
+    @pytest.mark.parametrize('execution_number', range(7))
+    def test_isunique(self, execution_number):
+        """This test is executed seven times, each time it uses a pool of
+        two threads to run a test that is very likely to fail without the
+        fix for gh-21150. It is not a deterministic test, as there is
+        still a chance it will pass even though the bug exists. But
+        with the fix, it must always work with not issues."""
+        x = pd.date_range('2001', '2020')
+        with ThreadPoolExecutor(2) as p:
+            assert all(p.map(lambda x: x.is_unique, [x] * 2))