Remove convert_to_numpy

conftest.py

@@ -55,16 +55,13 @@ def sat_data_source(sat_filename: Path):
         forecast_minutes=5,
         channels=("HRV",),
         n_timesteps_per_batch=2,
-        convert_to_numpy=True,
     )
 
 
 @pytest.fixture
 def general_data_source():
 
-    return MetadataDataSource(
-        history_minutes=0, forecast_minutes=5, object_at_center="GSP", convert_to_numpy=True
-    )
+    return MetadataDataSource(history_minutes=0, forecast_minutes=5, object_at_center="GSP")
 
 
 @pytest.fixture
@@ -75,7 +72,6 @@ def gsp_data_source():
         filename=Path(__file__).parent.absolute() / "tests" / "data" / "gsp" / "test.zarr",
         history_minutes=0,
         forecast_minutes=30,
-        convert_to_numpy=True,
     )
 
 

notebooks/2021-09/2021-09-07/sat_data.py

@@ -7,7 +7,6 @@
     "all_zarr_int16_single_timestep.zarr",
     history_len=6,
     forecast_len=12,
-    convert_to_numpy=True,
     image_size_pixels=64,
     meters_per_pixel=2000,
     n_timesteps_per_batch=32,

nowcasting_dataset/data_sources/data_source.py

@@ -29,7 +29,6 @@ class DataSource:
         Does NOT include t0.  If forecast_minutes = 0 then the example will end
         at t0.  If both history_minutes and forecast_minutes are 0, then the example
         will consist of a single timestep at t0.
-      convert_to_numpy: Whether or not to convert each example to numpy.
       sample_period_minutes: The time delta between each data point
 
     Attributes ending in `_length` are sequence lengths represented as integer numbers of timesteps.
@@ -38,7 +37,6 @@ class DataSource:
 
     history_minutes: int
     forecast_minutes: int
-    convert_to_numpy: bool
 
     def __post_init__(self):
         """ Post Init """

nowcasting_dataset/dataset/datamodule.py

@@ -66,7 +66,6 @@ class NowcastingDataModule(pl.LightningDataModule):
     sun_filename: Optional[Union[str, Path]] = None
     nwp_image_size_pixels: int = 2  #: Passed to Data Sources.
     meters_per_pixel: int = 2000  #: Passed to Data Sources.
-    convert_to_numpy: bool = True  #: Passed to Data Sources.
     pin_memory: bool = True  #: Passed to DataLoader.
     num_workers: int = 16  #: Passed to DataLoader.
     prefetch_factor: int = 64  #: Passed to DataLoader.
@@ -117,7 +116,6 @@ def prepare_data(self) -> None:
             forecast_minutes=self.forecast_minutes,
             channels=self.sat_channels,
             n_timesteps_per_batch=n_timesteps_per_batch,
-            convert_to_numpy=self.convert_to_numpy,
         )
 
         self.data_sources = [self.sat_data_source]
@@ -133,7 +131,6 @@ def prepare_data(self) -> None:
                 end_dt=sat_datetimes[-1],
                 history_minutes=self.history_minutes,
                 forecast_minutes=self.forecast_minutes,
-                convert_to_numpy=self.convert_to_numpy,
                 image_size_pixels=self.satellite_image_size_pixels,
                 meters_per_pixel=self.meters_per_pixel,
                 get_center=False,
@@ -149,7 +146,6 @@ def prepare_data(self) -> None:
                 end_dt=sat_datetimes[-1],
                 history_minutes=self.history_minutes,
                 forecast_minutes=self.forecast_minutes,
-                convert_to_numpy=self.convert_to_numpy,
                 image_size_pixels=self.satellite_image_size_pixels,
                 meters_per_pixel=self.meters_per_pixel,
                 get_center=True,
@@ -169,7 +165,6 @@ def prepare_data(self) -> None:
                 forecast_minutes=self.forecast_minutes,
                 channels=self.nwp_channels,
                 n_timesteps_per_batch=n_timesteps_per_batch,
-                convert_to_numpy=self.convert_to_numpy,
             )
 
             self.data_sources.append(self.nwp_data_source)
@@ -182,7 +177,6 @@ def prepare_data(self) -> None:
                 meters_per_pixel=self.meters_per_pixel,
                 history_minutes=self.history_minutes,
                 forecast_minutes=self.forecast_minutes,
-                convert_to_numpy=self.convert_to_numpy,
             )
 
             self.data_sources.append(self.topo_data_source)
@@ -193,22 +187,19 @@ def prepare_data(self) -> None:
                 filename=self.sun_filename,
                 history_minutes=self.history_minutes,
                 forecast_minutes=self.forecast_minutes,
-                convert_to_numpy=self.convert_to_numpy,
             )
             self.data_sources.append(self.sun_data_source)
 
         self.datetime_data_source = data_sources.DatetimeDataSource(
             history_minutes=self.history_minutes,
             forecast_minutes=self.forecast_minutes,
-            convert_to_numpy=self.convert_to_numpy,
         )
         self.data_sources.append(self.datetime_data_source)
 
         self.data_sources.append(
             MetadataDataSource(
                 history_minutes=self.history_minutes,
                 forecast_minutes=self.forecast_minutes,
-                convert_to_numpy=self.convert_to_numpy,
                 object_at_center="GSP",
             )
         )

scripts/generate_data_for_tests/get_test_data.py

@@ -104,7 +104,6 @@
 #                                  "all_zarr_int16_single_timestep.zarr",
 #                           history_length=6,
 #                           forecast_length=12,
-#                           convert_to_numpy=True,
 #                           image_size_pixels=64,
 #                           meters_per_pixel=2000,
 #                           n_timesteps_per_batch=32)

scripts/prepare_ml_data.py

@@ -146,7 +146,6 @@ def get_data_module():
         n_validation_batches_per_epoch=1_008,
         n_test_batches_per_epoch=1_008,
         collate_fn=lambda x: x,
-        convert_to_numpy=False,  #: Leave data as Pandas / Xarray for pre-preparing.
         skip_n_train_batches=maximum_batch_id_train // num_workers,
         skip_n_validation_batches=maximum_batch_id_validation // num_workers,
         skip_n_test_batches=maximum_batch_id_test // num_workers,

tests/data_sources/gsp/test_gsp_data_source.py

@@ -15,7 +15,6 @@ def test_gsp_pv_data_source_init():
         end_dt=datetime(2019, 1, 2),
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
         image_size_pixels=64,
         meters_per_pixel=2000,
     )
@@ -30,7 +29,6 @@ def test_gsp_pv_data_source_get_locations_for_batch():
         end_dt=datetime(2019, 1, 2),
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
         image_size_pixels=64,
         meters_per_pixel=2000,
     )
@@ -59,7 +57,6 @@ def test_gsp_pv_data_source_get_example():
         end_dt=datetime(2019, 1, 2),
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
         image_size_pixels=64,
         meters_per_pixel=2000,
     )
@@ -85,7 +82,6 @@ def test_gsp_pv_data_source_get_batch():
         history_minutes=30,
         forecast_minutes=60,
         sample_period_minutes=30,
-        convert_to_numpy=True,
         image_size_pixels=64,
         meters_per_pixel=2000,
     )

tests/data_sources/sun/test_sun_data_source.py

@@ -5,17 +5,13 @@
 def test_init(test_data_folder):
     filename = test_data_folder + "/sun/test.zarr"
 
-    _ = SunDataSource(
-        filename=filename, history_minutes=30, forecast_minutes=60, convert_to_numpy=True
-    )
+    _ = SunDataSource(filename=filename, history_minutes=30, forecast_minutes=60)
 
 
 def test_get_example(test_data_folder):
     filename = test_data_folder + "/sun/test.zarr"
 
-    sun_data_source = SunDataSource(
-        filename=filename, history_minutes=30, forecast_minutes=60, convert_to_numpy=True
-    )
+    sun_data_source = SunDataSource(filename=filename, history_minutes=30, forecast_minutes=60)
 
     x = 256895.63164759654
     y = 666180.3018829626
@@ -30,9 +26,7 @@ def test_get_example(test_data_folder):
 def test_get_example_different_year(test_data_folder):
     filename = test_data_folder + "/sun/test.zarr"
 
-    sun_data_source = SunDataSource(
-        filename=filename, history_minutes=30, forecast_minutes=60, convert_to_numpy=True
-    )
+    sun_data_source = SunDataSource(filename=filename, history_minutes=30, forecast_minutes=60)
 
     x = 256895.63164759654
     y = 666180.3018829626

tests/data_sources/test_data_source.py

@@ -7,5 +7,4 @@ def test_image_data_source():
         meters_per_pixel=2000,
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
     )

tests/data_sources/test_datetime.py

@@ -5,7 +5,6 @@
 
 def test_datetime_source():
     datetime_source = DatetimeDataSource(
-        convert_to_numpy=True,
         forecast_minutes=300,
         history_minutes=10,
     )
@@ -15,7 +14,6 @@ def test_datetime_source():
 
 def test_datetime_source_batch():
     datetime_source = DatetimeDataSource(
-        convert_to_numpy=True,
         forecast_minutes=300,
         history_minutes=10,
     )

tests/data_sources/test_nwp_data_source.py

@@ -16,7 +16,6 @@ def test_nwp_data_source_init():
         filename=NWP_FILENAME,
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
         n_timesteps_per_batch=8,
     )
 
@@ -26,7 +25,6 @@ def test_nwp_data_source_open():
         filename=NWP_FILENAME,
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
         n_timesteps_per_batch=8,
         channels=["t"],
     )
@@ -39,7 +37,6 @@ def test_nwp_data_source_batch():
         filename=NWP_FILENAME,
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
         n_timesteps_per_batch=8,
         channels=["t"],
     )
@@ -60,7 +57,6 @@ def test_nwp_get_contiguous_time_periods():
         filename=NWP_FILENAME,
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
         n_timesteps_per_batch=8,
         channels=["t"],
     )

tests/data_sources/test_pv_data_source.py

@@ -24,7 +24,6 @@ def test_get_example_and_batch():
     pv_data_source = PVDataSource(
         history_minutes=30,
         forecast_minutes=60,
-        convert_to_numpy=True,
         image_size_pixels=64,
         meters_per_pixel=2000,
         filename=PV_DATA_FILENAME,

tests/data_sources/test_topographic_data_source.py

@@ -25,7 +25,6 @@ def test_get_example_2km(x, y, left, right, top, bottom):
         filename="tests/data/europe_dem_2km_osgb.tif",
         image_size_pixels=128,
         meters_per_pixel=size,
-        convert_to_numpy=True,
         forecast_minutes=300,
         history_minutes=10,
     )
@@ -53,7 +52,6 @@ def test_get_example_gcs():
         filename=filename,
         image_size_pixels=128,
         meters_per_pixel=size,
-        convert_to_numpy=True,
         forecast_minutes=300,
         history_minutes=10,
     )

tests/test_datamodule.py

@@ -99,7 +99,6 @@ def test_data_module(config_filename):
         n_training_batches_per_epoch=200,  # Add pre-fetch factor!
         n_validation_batches_per_epoch=200,
         collate_fn=lambda x: x,
-        convert_to_numpy=False,  #: Leave data as Pandas / Xarray for pre-preparing.
         skip_n_train_batches=0,
         skip_n_validation_batches=0,
         train_validation_percentage_split=50,
@@ -153,7 +152,6 @@ def test_batch_to_batch_to_dataset():
         n_training_batches_per_epoch=200,  # Add pre-fetch factor!
         n_validation_batches_per_epoch=200,
         collate_fn=lambda x: x,
-        convert_to_numpy=False,  #: Leave data as Pandas / Xarray for pre-preparing.
         skip_n_train_batches=0,
         skip_n_validation_batches=0,
         train_validation_percentage_split=50,