diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst
index 6eedf9dee5266..ba2ebfff79ad9 100644
--- a/doc/source/whatsnew/v1.1.0.rst
+++ b/doc/source/whatsnew/v1.1.0.rst
@@ -636,6 +636,7 @@ Indexing
 - Bug in :meth:`Series.__getitem__` allowing missing labels with ``np.ndarray``, :class:`Index`, :class:`Series` indexers but not ``list``, these now all raise ``KeyError`` (:issue:`33646`)
 - Bug in :meth:`DataFrame.truncate` and :meth:`Series.truncate` where index was assumed to be monotone increasing (:issue:`33756`)
 - Indexing with a list of strings representing datetimes failed on :class:`DatetimeIndex` or :class:`PeriodIndex`(:issue:`11278`)
+- Bug in :meth:`Index.is_all_dates` incorrectly returning ``False`` when inferred type of index was other dates than datetime (:issue:`19204`)
 
 Missing
 ^^^^^^^
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 79af28dc5f2ce..f98aa55ac7708 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -9,7 +9,7 @@
 
 from pandas._libs import algos as libalgos, index as libindex, lib
 import pandas._libs.join as libjoin
-from pandas._libs.lib import is_datetime_array, no_default
+from pandas._libs.lib import no_default
 from pandas._libs.tslibs import OutOfBoundsDatetime, Timestamp
 from pandas._libs.tslibs.period import IncompatibleFrequency
 from pandas._libs.tslibs.timezones import tz_compare
@@ -25,7 +25,6 @@
 )
 from pandas.core.dtypes.common import (
     ensure_int64,
-    ensure_object,
     ensure_platform_int,
     is_bool,
     is_bool_dtype,
@@ -1951,7 +1950,14 @@ def is_all_dates(self) -> bool:
         """
         Whether or not the index values only consist of dates.
         """
-        return is_datetime_array(ensure_object(self._values))
+        return self.inferred_type in [
+            "datetime64",
+            "datetime",
+            "date",
+            "timedelta64",
+            "timedelta",
+            "period",
+        ]
 
     # --------------------------------------------------------------------
     # Pickle Methods
diff --git a/pandas/tests/indexes/test_base.py b/pandas/tests/indexes/test_base.py
index 9f235dcdbb295..ae1df002a6bd9 100644
--- a/pandas/tests/indexes/test_base.py
+++ b/pandas/tests/indexes/test_base.py
@@ -1152,14 +1152,26 @@ def test_is_object(self, indices, expected):
             ("bool", False),
             ("categorical", False),
             ("int", False),
-            ("datetime", True),
             ("float", False),
+            ("datetime", True),
+            ("datetime-tz", True),
+            ("period", True),
+            ("timedelta", True),
+            ("empty", False),
         ],
         indirect=["indices"],
     )
     def test_is_all_dates(self, indices, expected):
         assert indices.is_all_dates is expected
 
+    @pytest.mark.parametrize(
+        "index", ["datetime", "datetime-tz", "period", "timedelta"], indirect=["index"],
+    )
+    def test_is_all_dates_consistency(self, index):
+        # GH 19204
+        non_date = pd.Index(["not a date"])
+        assert index.is_all_dates == index.append(non_date)[:-1].is_all_dates
+
     def test_summary(self, indices):
         self._check_method_works(Index._summary, indices)