From 05e0ba01aab1e112c601681156b4b25b221474dd Mon Sep 17 00:00:00 2001 From: richardodliu <richardodliu@gmail.com> Date: Fri, 15 Mar 2024 12:14:16 +0000 Subject: [PATCH 1/2] modify the "scheduler" in "finetune/configs/ds_config_zero3.json" to be consistent with the sample shell script to finetune --- finetune/configs/ds_config_zero3.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/finetune/configs/ds_config_zero3.json b/finetune/configs/ds_config_zero3.json index 73f3b5f..c239059 100644 --- a/finetune/configs/ds_config_zero3.json +++ b/finetune/configs/ds_config_zero3.json @@ -13,11 +13,12 @@ }, "scheduler": { - "type": "WarmupLR", + "type": "WarmupCosineLR", "params": { - "warmup_min_lr": "auto", - "warmup_max_lr": "auto", - "warmup_num_steps": "auto" + "total_num_steps": "auto", + "warmup_num_steps": "auto", + "warmup_type": "linear", + "warmup_min_ratio": 0.1 } }, From d26a44f9e121fa5d24fa2533768cd4efe6acccf5 Mon Sep 17 00:00:00 2001 From: richardodliu <richardodliu@gmail.com> Date: Sat, 16 Mar 2024 19:27:06 +0800 Subject: [PATCH 2/2] Update ds_config_zero3.json delete an unnecessary parameter from the configuration file --- finetune/configs/ds_config_zero3.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/finetune/configs/ds_config_zero3.json b/finetune/configs/ds_config_zero3.json index c239059..d107ccc 100644 --- a/finetune/configs/ds_config_zero3.json +++ b/finetune/configs/ds_config_zero3.json @@ -17,8 +17,7 @@ "params": { "total_num_steps": "auto", "warmup_num_steps": "auto", - "warmup_type": "linear", - "warmup_min_ratio": 0.1 + "warmup_type": "linear" } }, @@ -49,4 +48,4 @@ "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "wall_clock_breakdown": false -} \ No newline at end of file +}