diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py
index e83f5556369..d381897a29d 100644
--- a/xarray/backends/zarr.py
+++ b/xarray/backends/zarr.py
@@ -375,6 +375,11 @@ def _get_zarr_dims_and_attrs(zarr_obj, dimension_key, try_nczarr):
         pass
     else:
         attributes = dict(zarr_obj.attrs)
+        if len(zarr_obj.shape) != len(dimensions):
+            raise KeyError(
+                "Zarr object is missing the `dimension_names` metadata which is "
+                "required for xarray to determine variable dimensions."
+            )
         return dimensions, attributes
 
     # Zarr arrays do not have dimensions. To get around this problem, we add
@@ -393,7 +398,13 @@ def _get_zarr_dims_and_attrs(zarr_obj, dimension_key, try_nczarr):
 
         # NCZarr defines dimensions through metadata in .zarray
         zarray_path = os.path.join(zarr_obj.path, ".zarray")
-        zarray = json.loads(zarr_obj.store[zarray_path])
+        if _zarr_v3():
+            import asyncio
+
+            zarray_str = asyncio.run(zarr_obj.store.get(zarray_path)).to_bytes()
+        else:
+            zarray_str = zarr_obj.store.get(zarray_path)
+        zarray = json.loads(zarray_str)
         try:
             # NCZarr uses Fully Qualified Names
             dimensions = [
diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index a6df4d7b0cb..5b1901ab86b 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -6153,6 +6153,18 @@ def test_zarr_closing_internal_zip_store():
         assert_identical(original_da, loaded_da)
 
 
+@requires_zarr
+@pytest.mark.usefixtures("default_zarr_format")
+def test_raises_key_error_on_invalid_zarr_store(tmp_path):
+    root = zarr.open_group(tmp_path / "tmp.zarr")
+    if Version(zarr.__version__) < Version("3.0.0"):
+        root.create_dataset("bar", shape=(3, 5), dtype=np.float32)
+    else:
+        root.create_array("bar", shape=(3, 5), dtype=np.float32)
+    with pytest.raises(KeyError, match=r"xarray to determine variable dimensions"):
+        xr.open_zarr(tmp_path / "tmp.zarr", consolidated=False)
+
+
 @requires_zarr
 @pytest.mark.usefixtures("default_zarr_format")
 class TestZarrRegionAuto: