Skip to content

Commit 6126cc3

Browse files
Fiona-WatersFiona-Waters
committed
updating training script in basic_interactive notebook
1 parent 2a43d9e commit 6126cc3

File tree

6 files changed

+286
-212
lines changed

6 files changed

+286
-212
lines changed

‎demo-notebooks/additional-demos/hf_interactive.ipynb

+70-51
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,15 @@
1313
"id": "d4acfb10-1aa1-445d-947e-396ea5ebed1a",
1414
"metadata": {},
1515
"source": [
16-
"In this notebook you will learn how to leverage the **[huggingface](https://huggingface.co/)** support in ray ecosystem to carry out a text classification task using transfer learning. We will be referencing the example **[here](https://huggingface.co/docs/transformers/tasks/sequence_classification)**"
16+
"In this notebook you will learn how to leverage the **[huggingface](https://huggingface.co/)** support in ray ecosystem to carry out a text classification task using transfer learning. We will be referencing the example **[here](https://huggingface.co/docs/transformers/tasks/sequence_classification)** and **[here](https://docs.ray.io/en/latest/train/getting-started-transformers.html)**."
1717
]
1818
},
1919
{
2020
"cell_type": "markdown",
2121
"id": "70b77929-e96c-434e-ada3-8b14795bfbb1",
2222
"metadata": {},
2323
"source": [
24-
"The example carries out a text classification task on **[imdb dataset](https://huggingface.co/datasets/imdb)** and tries to classify the movie reviews as positive or negative. Huggingface library provides an easy way to build a model and the dataset to carry out this classification task. In this case we will be using **distilbert-base-uncased** model which is a **BERT** based model.\n",
25-
"\n",
26-
"Huggingface has a **[built in support for ray ecosystem](https://docs.ray.io/en/releases-1.13.0/_modules/ray/ml/train/integrations/huggingface/huggingface_trainer.html)** which allows the huggingface trainer to scale on CodeFlare and can scale the training as we add additional gpus and can run distributed training across multiple GPUs that will help scale out the training.\n"
24+
"The example carries out a text classification task on **[imdb dataset](https://huggingface.co/datasets/imdb)** and tries to classify the movie reviews as positive or negative. Huggingface library provides an easy way to build a model and the dataset to carry out this classification task. In this case we will be using **distilbert-base-uncased** model which is a **BERT** based model."
2725
]
2826
},
2927
{
@@ -323,7 +321,7 @@
323321
"# establish connection to ray cluster\n",
324322
"\n",
325323
"#install additional libraries that will be required for this training\n",
326-
"runtime_env = {\"pip\": [\"transformers\", \"datasets\", \"evaluate\", \"pyarrow<7.0.0\", \"accelerate\"]}\n",
324+
"runtime_env = {\"pip\": [\"pytorch_lightning==1.5.10\", \"ray_lightning\", \"torchmetrics==0.9.1\", \"torchvision==0.12.0\"]}\n",
327325
"\n",
328326
"# NOTE: This will work for in-cluster notebook servers (RHODS/ODH), but not for local machines\n",
329327
"# To see how to connect from your laptop, go to demo-notebooks/additional-demos/local_interactive.ipynb\n",
@@ -365,66 +363,87 @@
365363
"source": [
366364
"@ray.remote\n",
367365
"def train_fn():\n",
368-
" from datasets import load_dataset\n",
369-
" import transformers\n",
370-
" from transformers import AutoTokenizer, TrainingArguments\n",
371-
" from transformers import AutoModelForSequenceClassification\n",
366+
" import os\n",
372367
" import numpy as np\n",
373-
" from datasets import load_metric\n",
374-
" import ray\n",
375-
" from ray import tune\n",
376-
" from ray.train.huggingface import HuggingFaceTrainer\n",
377-
"\n",
378-
" dataset = load_dataset(\"imdb\")\n",
379-
" tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
380-
"\n",
381-
" def tokenize_function(examples):\n",
382-
" return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n",
368+
" import evaluate\n",
369+
" from datasets import load_dataset, load_metric\n",
370+
" import transformers\n",
371+
" from transformers import (\n",
372+
" Trainer,\n",
373+
" TrainingArguments,\n",
374+
" AutoTokenizer,\n",
375+
" AutoModelForSequenceClassification,\n",
376+
" )\n",
377+
" import ray.train.huggingface.transformers\n",
378+
" from ray.train import ScalingConfig\n",
379+
" from ray.train.torch import TorchTrainer\n",
383380
"\n",
384-
" tokenized_datasets = dataset.map(tokenize_function, batched=True)\n",
381+
" # For S3 persistent storage replace the following environment variables with your AWS credentials \n",
382+
" # See here for information on how to set up an S3 bucket https://docs.aws.amazon.com/AmazonS3/latest/userguide/creating-bucket.html\n",
383+
" os.environ[\"AWS_ACCESS_KEY_ID\"] = \"XXXXXXXX\"\n",
384+
" os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"XXXXXXXX\"\n",
385+
" os.environ[\"AWS_DEFAULT_REGION\"] = \"XXXXXXXX\"\n",
386+
" \n",
387+
" def train_func():\n",
388+
" # Datasets\n",
389+
" dataset = load_dataset(\"imdb\")\n",
390+
" tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
385391
"\n",
386-
" #using a fraction of dataset but you can run with the full dataset\n",
387-
" small_train_dataset = tokenized_datasets[\"train\"].shuffle(seed=42).select(range(100))\n",
388-
" small_eval_dataset = tokenized_datasets[\"test\"].shuffle(seed=42).select(range(100))\n",
392+
" def tokenize_function(examples):\n",
393+
" return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n",
389394
"\n",
390-
" print(f\"len of train {small_train_dataset} and test {small_eval_dataset}\")\n",
395+
" small_train_dataset = (\n",
396+
" dataset[\"train\"].select(range(100)).map(tokenize_function, batched=True)\n",
397+
" )\n",
398+
" small_eval_dataset = (\n",
399+
" dataset[\"test\"].select(range(100)).map(tokenize_function, batched=True)\n",
400+
" )\n",
391401
"\n",
392-
" ray_train_ds = ray.data.from_huggingface(small_train_dataset)\n",
393-
" ray_evaluation_ds = ray.data.from_huggingface(small_eval_dataset)\n",
402+
" # Model\n",
403+
" model = AutoModelForSequenceClassification.from_pretrained(\n",
404+
" \"distilbert-base-uncased\", num_labels=2\n",
405+
" )\n",
394406
"\n",
395-
" def compute_metrics(eval_pred):\n",
396-
" metric = load_metric(\"accuracy\")\n",
397-
" logits, labels = eval_pred\n",
398-
" predictions = np.argmax(logits, axis=-1)\n",
399-
" return metric.compute(predictions=predictions, references=labels)\n",
407+
" def compute_metrics(eval_pred):\n",
408+
" metric = load_metric(\"accuracy\")\n",
409+
" logits, labels = eval_pred\n",
410+
" predictions = np.argmax(logits, axis=-1)\n",
411+
" return metric.compute(predictions=predictions, references=labels)\n",
400412
"\n",
401-
" def trainer_init_per_worker(train_dataset, eval_dataset, **config):\n",
402-
" model = AutoModelForSequenceClassification.from_pretrained(\"distilbert-base-uncased\", num_labels=2)\n",
413+
" # Hugging Face Trainer\n",
414+
" training_args = TrainingArguments(\n",
415+
" output_dir=\"test_trainer\",\n",
416+
" evaluation_strategy=\"epoch\",\n",
417+
" save_strategy=\"epoch\",\n",
418+
" report_to=\"none\",\n",
419+
" )\n",
403420
"\n",
404-
" training_args = TrainingArguments(\"/tmp/hf_imdb/test\", eval_steps=1, disable_tqdm=True, \n",
405-
" num_train_epochs=1, skip_memory_metrics=True,\n",
406-
" learning_rate=2e-5,\n",
407-
" per_device_train_batch_size=16,\n",
408-
" per_device_eval_batch_size=16, \n",
409-
" weight_decay=0.01,)\n",
410-
" return transformers.Trainer(\n",
421+
" trainer = Trainer(\n",
411422
" model=model,\n",
412423
" args=training_args,\n",
413-
" train_dataset=train_dataset,\n",
414-
" eval_dataset=eval_dataset,\n",
415-
" compute_metrics=compute_metrics\n",
424+
" train_dataset=small_train_dataset,\n",
425+
" eval_dataset=small_eval_dataset,\n",
426+
" compute_metrics=compute_metrics,\n",
416427
" )\n",
417428
"\n",
418-
" scaling_config = ScalingConfig(num_workers=4, use_gpu=True) #num workers is the number of gpus\n",
419429
"\n",
420-
" # we are using the ray native HuggingFaceTrainer, but you can swap out to use non ray Huggingface Trainer. Both have the same method signature. \n",
421-
" # the ray native HFTrainer has built in support for scaling to multiple GPUs\n",
422-
" trainer = HuggingFaceTrainer(\n",
423-
" trainer_init_per_worker=trainer_init_per_worker,\n",
424-
" scaling_config=scaling_config,\n",
425-
" datasets={\"train\": ray_train_ds, \"evaluation\": ray_evaluation_ds},\n",
430+
" callback = ray.train.huggingface.transformers.RayTrainReportCallback()\n",
431+
" trainer.add_callback(callback)\n",
432+
"\n",
433+
" trainer = ray.train.huggingface.transformers.prepare_trainer(trainer)\n",
434+
"\n",
435+
" trainer.train()\n",
436+
"\n",
437+
"\n",
438+
" ray_trainer = TorchTrainer(\n",
439+
" train_func,\n",
440+
" scaling_config=ScalingConfig(num_workers=3, use_gpu=True),\n",
441+
" # Configure the run's persistent storage that is accessible across \n",
442+
" # all worker nodes\n",
443+
" # update RunConfig below to include your s3 bucket details \n",
444+
" run_config=ray.train.RunConfig(storage_path=\"s3://BUCKET_NAME/SUB_PATH/\", name=\"unique_run_name\"),\n",
426445
" )\n",
427-
" result = trainer.fit()\n"
446+
" result: ray.train.Result = ray_trainer.fit()"
428447
]
429448
},
430449
{

‎demo-notebooks/guided-demos/2_basic_interactive.ipynb

+72-52
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
" min_memory=8,\n",
6969
" max_memory=8,\n",
7070
" num_gpus=1,\n",
71+
" head_gpus=1,\n",
7172
" image=\"quay.io/project-codeflare/ray:latest-py39-cu118\",\n",
7273
" write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources \n",
7374
" # local_queue=\"local-queue-name\" # Specify the local queue manually\n",
@@ -147,19 +148,17 @@
147148
"metadata": {},
148149
"outputs": [],
149150
"source": [
150-
"#before proceeding make sure the cluster exists and the uri is not empty\n",
151+
"# before proceeding make sure the cluster exists and the uri is not empty\n",
151152
"assert ray_cluster_uri, \"Ray cluster needs to be started and set before proceeding\"\n",
152153
"\n",
153154
"import ray\n",
154-
"from ray.air.config import ScalingConfig\n",
155155
"\n",
156156
"# reset the ray context in case there's already one. \n",
157157
"ray.shutdown()\n",
158158
"# establish connection to ray cluster\n",
159159
"\n",
160-
"#install additional libraries that will be required for model training\n",
161-
"runtime_env = {\"pip\": [\"transformers\", \"datasets\", \"evaluate\", \"pyarrow<7.0.0\", \"accelerate\"]}\n",
162-
"\n",
160+
"# install additional libraries that will be required for model training\n",
161+
"runtime_env = {\"pip\": [\"pytorch_lightning==1.5.10\", \"ray_lightning\", \"torchmetrics==0.9.1\", \"torchvision==0.12.0\"]}\n",
163162
"# NOTE: This will work for in-cluster notebook servers (RHODS/ODH), but not for local machines\n",
164163
"# To see how to connect from your laptop, go to demo-notebooks/additional-demos/local_interactive.ipynb\n",
165164
"ray.init(address=ray_cluster_uri, runtime_env=runtime_env)\n",
@@ -172,7 +171,7 @@
172171
"id": "9711030b",
173172
"metadata": {},
174173
"source": [
175-
"Now that we are connected (and have passed in some package requirements), let's try writing some training code for a DistilBERT transformer model via HuggingFace (using IMDB dataset):"
174+
"Now that we are connected (and have passed in some package requirements), let's try writing some training code:"
176175
]
177176
},
178177
{
@@ -184,66 +183,87 @@
184183
"source": [
185184
"@ray.remote\n",
186185
"def train_fn():\n",
187-
" from datasets import load_dataset\n",
188-
" import transformers\n",
189-
" from transformers import AutoTokenizer, TrainingArguments\n",
190-
" from transformers import AutoModelForSequenceClassification\n",
186+
" import os\n",
191187
" import numpy as np\n",
192-
" from datasets import load_metric\n",
193-
" import ray\n",
194-
" from ray import tune\n",
195-
" from ray.train.huggingface import HuggingFaceTrainer\n",
196-
"\n",
197-
" dataset = load_dataset(\"imdb\")\n",
198-
" tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
199-
"\n",
200-
" def tokenize_function(examples):\n",
201-
" return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n",
188+
" import evaluate\n",
189+
" from datasets import load_dataset, load_metric\n",
190+
" import transformers\n",
191+
" from transformers import (\n",
192+
" Trainer,\n",
193+
" TrainingArguments,\n",
194+
" AutoTokenizer,\n",
195+
" AutoModelForSequenceClassification,\n",
196+
" )\n",
197+
" import ray.train.huggingface.transformers\n",
198+
" from ray.train import ScalingConfig\n",
199+
" from ray.train.torch import TorchTrainer\n",
202200
"\n",
203-
" tokenized_datasets = dataset.map(tokenize_function, batched=True)\n",
201+
" # For S3 persistent storage replace the following environment variables with your AWS credentials \n",
202+
" # See here for information on how to set up an S3 bucket https://docs.aws.amazon.com/AmazonS3/latest/userguide/creating-bucket.html\n",
203+
" os.environ[\"AWS_ACCESS_KEY_ID\"] = \"XXXXXXXX\"\n",
204+
" os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"XXXXXXXX\"\n",
205+
" os.environ[\"AWS_DEFAULT_REGION\"] = \"XXXXXXXX\"\n",
206+
" \n",
207+
" def train_func():\n",
208+
" # Datasets\n",
209+
" dataset = load_dataset(\"imdb\")\n",
210+
" tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
204211
"\n",
205-
" #using a fraction of dataset but you can run with the full dataset\n",
206-
" small_train_dataset = tokenized_datasets[\"train\"].shuffle(seed=42).select(range(100))\n",
207-
" small_eval_dataset = tokenized_datasets[\"test\"].shuffle(seed=42).select(range(100))\n",
212+
" def tokenize_function(examples):\n",
213+
" return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n",
208214
"\n",
209-
" print(f\"len of train {small_train_dataset} and test {small_eval_dataset}\")\n",
215+
" small_train_dataset = (\n",
216+
" dataset[\"train\"].select(range(100)).map(tokenize_function, batched=True)\n",
217+
" )\n",
218+
" small_eval_dataset = (\n",
219+
" dataset[\"test\"].select(range(100)).map(tokenize_function, batched=True)\n",
220+
" )\n",
210221
"\n",
211-
" ray_train_ds = ray.data.from_huggingface(small_train_dataset)\n",
212-
" ray_evaluation_ds = ray.data.from_huggingface(small_eval_dataset)\n",
222+
" # Model\n",
223+
" model = AutoModelForSequenceClassification.from_pretrained(\n",
224+
" \"distilbert-base-uncased\", num_labels=2\n",
225+
" )\n",
213226
"\n",
214-
" def compute_metrics(eval_pred):\n",
215-
" metric = load_metric(\"accuracy\")\n",
216-
" logits, labels = eval_pred\n",
217-
" predictions = np.argmax(logits, axis=-1)\n",
218-
" return metric.compute(predictions=predictions, references=labels)\n",
227+
" def compute_metrics(eval_pred):\n",
228+
" metric = load_metric(\"accuracy\")\n",
229+
" logits, labels = eval_pred\n",
230+
" predictions = np.argmax(logits, axis=-1)\n",
231+
" return metric.compute(predictions=predictions, references=labels)\n",
219232
"\n",
220-
" def trainer_init_per_worker(train_dataset, eval_dataset, **config):\n",
221-
" model = AutoModelForSequenceClassification.from_pretrained(\"distilbert-base-uncased\", num_labels=2)\n",
233+
" # Hugging Face Trainer\n",
234+
" training_args = TrainingArguments(\n",
235+
" output_dir=\"test_trainer\",\n",
236+
" evaluation_strategy=\"epoch\",\n",
237+
" save_strategy=\"epoch\",\n",
238+
" report_to=\"none\",\n",
239+
" )\n",
222240
"\n",
223-
" training_args = TrainingArguments(\"/tmp/hf_imdb/test\", eval_steps=1, disable_tqdm=True, \n",
224-
" num_train_epochs=1, skip_memory_metrics=True,\n",
225-
" learning_rate=2e-5,\n",
226-
" per_device_train_batch_size=16,\n",
227-
" per_device_eval_batch_size=16, \n",
228-
" weight_decay=0.01,)\n",
229-
" return transformers.Trainer(\n",
241+
" trainer = Trainer(\n",
230242
" model=model,\n",
231243
" args=training_args,\n",
232-
" train_dataset=train_dataset,\n",
233-
" eval_dataset=eval_dataset,\n",
234-
" compute_metrics=compute_metrics\n",
244+
" train_dataset=small_train_dataset,\n",
245+
" eval_dataset=small_eval_dataset,\n",
246+
" compute_metrics=compute_metrics,\n",
235247
" )\n",
236248
"\n",
237-
" scaling_config = ScalingConfig(num_workers=2, use_gpu=True) #num workers is the number of gpus\n",
238249
"\n",
239-
" # we are using the ray native HuggingFaceTrainer, but you can swap out to use non ray Huggingface Trainer. Both have the same method signature. \n",
240-
" # the ray native HFTrainer has built in support for scaling to multiple GPUs\n",
241-
" trainer = HuggingFaceTrainer(\n",
242-
" trainer_init_per_worker=trainer_init_per_worker,\n",
243-
" scaling_config=scaling_config,\n",
244-
" datasets={\"train\": ray_train_ds, \"evaluation\": ray_evaluation_ds},\n",
250+
" callback = ray.train.huggingface.transformers.RayTrainReportCallback()\n",
251+
" trainer.add_callback(callback)\n",
252+
"\n",
253+
" trainer = ray.train.huggingface.transformers.prepare_trainer(trainer)\n",
254+
"\n",
255+
" trainer.train()\n",
256+
"\n",
257+
"\n",
258+
" ray_trainer = TorchTrainer(\n",
259+
" train_func,\n",
260+
" scaling_config=ScalingConfig(num_workers=3, use_gpu=True),\n",
261+
" # Configure the run's persistent storage that is accessible across \n",
262+
" # all worker nodes\n",
263+
" # update RunConfig below to include your s3 bucket details \n",
264+
" run_config=ray.train.RunConfig(storage_path=\"s3://BUCKET_NAME/SUB_PATH/\", name=\"unique_run_name\"),\n",
245265
" )\n",
246-
" result = trainer.fit()"
266+
" result: ray.train.Result = ray_trainer.fit()"
247267
]
248268
},
249269
{

‎demo-notebooks/guided-demos/mnist_fashion.py

-9
Original file line numberDiff line numberDiff line change
@@ -35,15 +35,6 @@ def forward(self, inputs):
3535
return logits
3636

3737

38-
def get_dataset():
39-
return datasets.FashionMNIST(
40-
root="/tmp/data",
41-
train=True,
42-
download=True,
43-
transform=ToTensor(),
44-
)
45-
46-
4738
def train_func_distributed():
4839
num_epochs = 3
4940
batch_size = 64

0 commit comments

Comments
 (0)