diff --git a/doc/source/whatsnew/v0.24.0.txt b/doc/source/whatsnew/v0.24.0.txt
index fa748cccc0d65..7afe24880eadc 100644
--- a/doc/source/whatsnew/v0.24.0.txt
+++ b/doc/source/whatsnew/v0.24.0.txt
@@ -1210,6 +1210,7 @@ Indexing
 - :class:`Index` no longer mangles ``None``, ``NaN`` and ``NaT``, i.e. they are treated as three different keys. However, for numeric Index all three are still coerced to a ``NaN`` (:issue:`22332`)
 - Bug in `scalar in Index` if scalar is a float while the ``Index`` is of integer dtype (:issue:`22085`)
 - Bug in `MultiIndex.set_levels` when levels value is not subscriptable (:issue:`23273`)
+- Bug where setting a timedelta column by ``Index`` causes it to be casted to double, and therefore lose precision (:issue:`23511`)
 
 Missing
 ^^^^^^^
diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py
index 5ce8a9103f008..e84953f3dab56 100644
--- a/pandas/core/internals/blocks.py
+++ b/pandas/core/internals/blocks.py
@@ -2104,9 +2104,9 @@ def _box_func(self):
     def _can_hold_element(self, element):
         tipo = maybe_infer_dtype_type(element)
         if tipo is not None:
-            return issubclass(tipo.type, np.timedelta64)
+            return issubclass(tipo.type, (np.timedelta64, np.int64))
         return is_integer(element) or isinstance(
-            element, (timedelta, np.timedelta64))
+            element, (timedelta, np.timedelta64, np.int64))
 
     def fillna(self, value, **kwargs):
 
diff --git a/pandas/tests/indexing/test_timedelta.py b/pandas/tests/indexing/test_timedelta.py
index 29031c908bda4..acd8bee3e5663 100644
--- a/pandas/tests/indexing/test_timedelta.py
+++ b/pandas/tests/indexing/test_timedelta.py
@@ -80,3 +80,18 @@ def test_numpy_timedelta_scalar_indexing(self, start, stop,
         result = s.loc[slice(start, stop)]
         expected = s.iloc[expected_slice]
         tm.assert_series_equal(result, expected)
+
+    def test_roundtrip_thru_setitem(self):
+        # PR 23462
+        dt1 = pd.Timedelta(0)
+        dt2 = pd.Timedelta(28767471428571405)
+        df = pd.DataFrame({'dt': pd.Series([dt1, dt2])})
+        df_copy = df.copy()
+        s = pd.Series([dt1])
+
+        expected = df['dt'].iloc[1].value
+        df.loc[[True, False]] = s
+        result = df['dt'].iloc[1].value
+
+        assert expected == result
+        tm.assert_frame_equal(df, df_copy)