FedCurv algorithm

openfl-tutorials/interactive_api/PyTorch_Histology/envoy/start_envoy.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -e
 
-fx envoy start -n env_one --disable-tls --envoy-config-path envoy_config.yaml -dh localhost -dp 50051
+fx envoy start -n env_one --disable-tls --envoy-config-path envoy_config.yaml -dh localhost -dp 50051

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/README.md

@@ -0,0 +1,28 @@
+# PyTorch tutorial for FedCurv Federated Learning method on Histology dataset
+
+To show results on non-iid data distribution, this tutorial contains shard descriptor with custom data splitter where data is split log-normally. Federation consists of 8 envoys.
+
+Your Python environment must have OpenFL installed.
+
+1. Run Director instance:
+```
+cd director
+bash start_director.sh
+```
+
+2. In a separate terminal, execute:
+```
+cd envoy
+bash populate_envoys.sh # This creates all envoys folders in current directory
+bash start_envoys.sh # This launches all envoy instances 
+```
+
+3. In a separate terminal, launch a Jupyter Lab:
+```
+cd workspace
+jupyter lab
+```
+
+4. Open your browser at corresponding port and open `pytorch_histology.ipynb` from Jupyter web interface. 
+
+5. Execute all cells in order.

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/director/director_config.yaml

@@ -0,0 +1,5 @@
+settings:
+  listen_host: localhost
+  listen_port: 50051
+  sample_shape: ['150', '150']
+  target_shape: ['1']

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/director/start_director.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+fx director start --disable-tls -c director_config.yaml

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/director/start_director_with_tls.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+FQDN=$1
+fx director start -c director_config.yaml -rc cert/root_ca.crt -pk cert/"${FQDN}".key -oc cert/"${FQDN}".crt

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/envoy/.gitignore

@@ -0,0 +1 @@
+histology_data

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/envoy/envoy_config.yaml

@@ -0,0 +1,10 @@
+params:
+  cuda_devices: []
+
+optional_plugin_components: {}
+
+shard_descriptor:
+  template: histology_shard_descriptor.HistologyShardDescriptor
+  params:
+    data_folder: histology_data
+    rank_worldsize: 1,8

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/envoy/histology_shard_descriptor.py

@@ -0,0 +1,138 @@
+# Copyright (C) 2020-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Histology Shard Descriptor."""
+
+import logging
+import os
+from pathlib import Path
+from typing import Tuple
+from urllib.request import urlretrieve
+from zipfile import ZipFile
+
+import numpy as np
+from PIL import Image
+
+from openfl.interface.interactive_api.shard_descriptor import ShardDataset
+from openfl.interface.interactive_api.shard_descriptor import ShardDescriptor
+from openfl.utilities import tqdm_report_hook
+from openfl.utilities import validate_file_hash
+from openfl.utilities.data_splitters.numpy import LogNormalNumPyDataSplitter
+
+
+logger = logging.getLogger(__name__)
+
+
+class HistologyShardDataset(ShardDataset):
+    """Histology shard dataset class."""
+
+    TRAIN_SPLIT_RATIO = 0.8
+
+    def __init__(self, data_folder: Path, data_type='train', rank=1, worldsize=1):
+        """Histology shard dataset class."""
+        self.data_type = data_type
+        root = Path(data_folder) / 'Kather_texture_2016_image_tiles_5000'
+        classes = [d.name for d in root.iterdir() if d.is_dir()]
+        class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)}
+        self.samples = []
+        root = root.absolute()
+        for target_class in sorted(class_to_idx.keys()):
+            class_index = class_to_idx[target_class]
+            target_dir = root / target_class
+            for path in sorted(target_dir.glob('*')):
+                item = path, class_index
+                self.samples.append(item)
+        np.random.seed(0)
+        np.random.shuffle(self.samples)
+        idx_range = list(range(len(self.samples)))
+        idx_sep = int(len(idx_range) * HistologyShardDataset.TRAIN_SPLIT_RATIO)
+        train_idx, test_idx = np.split(idx_range, [idx_sep])
+        data_splitter = LogNormalNumPyDataSplitter(
+            mu=0,
+            sigma=2,
+            num_classes=8,
+            classes_per_col=2,
+            min_samples_per_class=5)
+        if data_type == 'train':
+            labels = np.array(self.samples)[train_idx][:, 1].astype(int)
+            self.idx = data_splitter.split(labels, worldsize)[rank - 1]
+        else:
+            labels = np.array(self.samples)[test_idx][:, 1].astype(int)
+            self.idx = data_splitter.split(labels, worldsize)[rank - 1]
+
+    def __len__(self) -> int:
+        """Return the len of the shard dataset."""
+        return len(self.idx)
+
+    def load_pil(self, path):
+        """Load image."""
+        with open(path, 'rb') as f:
+            img = Image.open(f)
+            return img.convert('RGB')
+
+    def __getitem__(self, index: int) -> Tuple['Image', int]:
+        """Return an item by the index."""
+        path, target = self.samples[self.idx[index]]
+        sample = self.load_pil(path)
+        return sample, target
+
+
+class HistologyShardDescriptor(ShardDescriptor):
+    """Shard descriptor class."""
+
+    URL = ('https://zenodo.org/record/53169/files/Kather_'
+           'texture_2016_image_tiles_5000.zip?download=1')
+    FILENAME = 'Kather_texture_2016_image_tiles_5000.zip'
+    ZIP_SHA384 = ('7d86abe1d04e68b77c055820c2a4c582a1d25d2983e38ab724e'
+                  'ac75affce8b7cb2cbf5ba68848dcfd9d84005d87d6790')
+    DEFAULT_PATH = Path('.') / 'data'
+
+    def __init__(
+            self,
+            data_folder: Path = DEFAULT_PATH,
+            rank_worldsize: str = '1,1',
+            **kwargs
+    ):
+        """Initialize HistologyShardDescriptor."""
+        self.data_folder = Path.cwd() / data_folder
+        self.download_data()
+        self.rank, self.worldsize = tuple(int(num) for num in rank_worldsize.split(','))
+
+    def download_data(self):
+        """Download prepared shard dataset."""
+        os.makedirs(self.data_folder, exist_ok=True)
+        filepath = self.data_folder / HistologyShardDescriptor.FILENAME
+        if not filepath.exists():
+            reporthook = tqdm_report_hook()
+            urlretrieve(HistologyShardDescriptor.URL, filepath, reporthook)  # nosec
+            validate_file_hash(filepath, HistologyShardDescriptor.ZIP_SHA384)
+            with ZipFile(filepath, 'r') as f:
+                f.extractall(self.data_folder)
+
+    def get_dataset(self, dataset_type):
+        """Return a shard dataset by type."""
+        return HistologyShardDataset(
+            data_folder=self.data_folder,
+            data_type=dataset_type,
+            rank=self.rank,
+            worldsize=self.worldsize
+        )
+
+    @property
+    def sample_shape(self):
+        """Return the sample shape info."""
+        shape = self.get_dataset('train')[0][0].size
+        return [str(dim) for dim in shape]
+
+    @property
+    def target_shape(self):
+        """Return the target shape info."""
+        target = self.get_dataset('train')[0][1]
+        shape = np.array([target]).shape
+        return [str(dim) for dim in shape]
+
+    @property
+    def dataset_description(self) -> str:
+        """Return the shard dataset description."""
+        return (f'Histology dataset, shard number {self.rank}'
+                f' out of {self.worldsize}')

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/envoy/populate_envoys.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+DIRECTOR_HOST=${1:-'localhost'}
+DIRECTOR_PORT=${2:-'50051'}
+PYTHON=${3:-'python3.8'}
+
+for i in {1..8}
+do
+    mkdir $i
+    cd $i
+    echo "shard_descriptor:
+    template: histology_shard_descriptor.HistologyShardDescriptor
+    params:
+        data_folder: histology_data
+        rank_worldsize: $i,8
+" > envoy_config.yaml
+
+    eval ${PYTHON} '-m venv venv'
+    echo "source venv/bin/activate
+    pip install ../../../../.. # install OpenFL
+    pip install -r requirements.txt
+    fx envoy start -n env_$i --disable-tls --envoy-config-path envoy_config.yaml -dh ${DIRECTOR_HOST} -dp ${DIRECTOR_PORT}
+    " > start_envoy.sh
+    cp ../requirements.txt .
+    cp ../histology_shard_descriptor.py .
+    cd ..
+done

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/envoy/requirements.txt

@@ -0,0 +1,3 @@
+Pillow==8.3.2
+tqdm==4.48.2
+numpy==1.19.1

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/envoy/start_envoy.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+fx envoy start -n env_one --disable-tls --envoy-config-path envoy_config.yaml -dh localhost -dp 50051

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/envoy/start_envoy_with_tls.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+ENVOY_NAME=$1
+DIRECTOR_FQDN=$2
+
+fx envoy start -n "$ENVOY_NAME" --envoy-config-path envoy_config.yaml -dh "$DIRECTOR_FQDN" -dp 50051 -rc cert/root_ca.crt -pk cert/"$ENVOY_NAME".key -oc cert/"$ENVOY_NAME".crt

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/envoy/start_envoys.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+cd 1 && bash start_envoy.sh &
+cd 2 && bash start_envoy.sh &
+cd 3 && bash start_envoy.sh &
+cd 4 && bash start_envoy.sh &
+cd 5 && bash start_envoy.sh &
+cd 6 && bash start_envoy.sh &
+cd 7 && bash start_envoy.sh &
+cd 8 && bash start_envoy.sh 

openfl-tutorials/interactive_api/PyTorch_Histology_FedCurv/workspace/.gitignore

@@ -0,0 +1 @@
+logs