diff --git a/pandas/core/arrays/datetimelike.py b/pandas/core/arrays/datetimelike.py
index 407daf15d5cce..b13a682ef3985 100644
--- a/pandas/core/arrays/datetimelike.py
+++ b/pandas/core/arrays/datetimelike.py
@@ -410,7 +410,7 @@ def ceil(self, freq, ambiguous="raise", nonexistent="raise"):
 
     def _with_freq(self, freq):
         """
-        Helper to set our freq in-place, returning self to allow method chaining.
+        Helper to get a view on the same data, with a new freq.
 
         Parameters
         ----------
@@ -418,7 +418,7 @@ def _with_freq(self, freq):
 
         Returns
         -------
-        self
+        Same type as self
         """
         # GH#29843
         if freq is None:
@@ -433,8 +433,9 @@ def _with_freq(self, freq):
             assert freq == "infer"
             freq = frequencies.to_offset(self.inferred_freq)
 
-        self._freq = freq
-        return self
+        arr = self.view()
+        arr._freq = freq
+        return arr
 
 
 class DatetimeLikeArrayMixin(
diff --git a/pandas/core/indexes/datetimelike.py b/pandas/core/indexes/datetimelike.py
index f41044db2b49c..ae119e72e37e1 100644
--- a/pandas/core/indexes/datetimelike.py
+++ b/pandas/core/indexes/datetimelike.py
@@ -44,7 +44,7 @@
 from pandas.core.ops import get_op_result_name
 from pandas.core.tools.timedeltas import to_timedelta
 
-from pandas.tseries.frequencies import DateOffset, to_offset
+from pandas.tseries.frequencies import DateOffset
 from pandas.tseries.offsets import Tick
 
 _index_doc_kwargs = dict(ibase._index_doc_kwargs)
@@ -678,20 +678,8 @@ def freqstr(self):
         return self.freq.freqstr
 
     def _with_freq(self, freq):
-        index = self.copy(deep=False)
-        if freq is None:
-            # Even if we _can_ have a freq, we might want to set it to None
-            index._freq = None
-        elif len(self) == 0 and isinstance(freq, DateOffset):
-            # Always valid.  In the TimedeltaArray case, we assume this
-            #  is a Tick offset.
-            index._freq = freq
-        else:
-            assert freq == "infer", freq
-            freq = to_offset(self.inferred_freq)
-            index._freq = freq
-
-        return index
+        arr = self._data._with_freq(freq)
+        return type(self)._simple_new(arr, name=self.name)
 
     def _shallow_copy(self, values=None, name: Label = lib.no_default):
         name = self.name if name is lib.no_default else name
diff --git a/pandas/tests/indexes/datetimes/test_datetime.py b/pandas/tests/indexes/datetimes/test_datetime.py
index 81fa1a27ac911..e0e5beaf48e20 100644
--- a/pandas/tests/indexes/datetimes/test_datetime.py
+++ b/pandas/tests/indexes/datetimes/test_datetime.py
@@ -38,6 +38,13 @@ def test_pickle(self):
         idx_p = tm.round_trip_pickle(idx)
         tm.assert_index_equal(idx, idx_p)
 
+    def test_pickle_after_set_freq(self):
+        dti = date_range("20130101", periods=3, tz="US/Eastern", name="foo")
+        dti = dti._with_freq(None)
+
+        res = tm.round_trip_pickle(dti)
+        tm.assert_index_equal(res, dti)
+
     def test_reindex_preserves_tz_if_target_is_empty_list_or_array(self):
         # GH7774
         index = date_range("20130101", periods=3, tz="US/Eastern")
diff --git a/pandas/tests/indexes/timedeltas/test_timedelta.py b/pandas/tests/indexes/timedeltas/test_timedelta.py
index 637a2629dda8a..5efa1a75700e0 100644
--- a/pandas/tests/indexes/timedeltas/test_timedelta.py
+++ b/pandas/tests/indexes/timedeltas/test_timedelta.py
@@ -47,6 +47,13 @@ def test_shift(self):
     def test_pickle_compat_construction(self):
         pass
 
+    def test_pickle_after_set_freq(self):
+        tdi = timedelta_range("1 day", periods=4, freq="s")
+        tdi = tdi._with_freq(None)
+
+        res = tm.round_trip_pickle(tdi)
+        tm.assert_index_equal(res, tdi)
+
     def test_isin(self):
 
         index = tm.makeTimedeltaIndex(4)