PLD Client (#61) (#62)

* test commits in DSE

* Support for porgressive layer dropping

* Minor changes on PLD

* update the finetune script

* PLD client

* Remove theta option

Co-authored-by: Minjia Zhang <[email protected]>

Co-authored-by: Minjia Zhang <[email protected]>

Author: tjruwase

BingBertSquad/turing/modelingpreln_layerdrop.py

@@ -1,5 +1,5 @@
 {
-    "name": "bing_bert_base_lamb_seq",
+    "name": "bing_bert_base_seq",
     "bert_token_file": "bert-base-uncased",
     "bert_model_file": "bert-base-uncased",
     "bert_model_config": {

bing_bert/bert_base.json

@@ -0,0 +1,49 @@
+{
+    "name": "bing_bert_base_seq",
+    "bert_token_file": "bert-base-uncased",
+    "bert_model_file": "bert-base-uncased",
+    "bert_model_config": {
+        "vocab_size_or_config_json_file": 119547,
+        "hidden_size": 768,
+        "num_hidden_layers": 12,
+        "num_attention_heads": 12,
+        "intermediate_size": 3072,
+        "hidden_act": "gelu",
+        "hidden_dropout_prob": 0.1,
+        "attention_probs_dropout_prob": 0.1,
+        "max_position_embeddings": 512,
+        "type_vocab_size": 2,
+        "initializer_range": 0.02
+    },
+    "data": {
+        "flags": {
+            "pretrain_dataset": true,
+            "pretrain_type": "wiki_bc"
+        },
+        "mixed_seq_datasets": {
+            "128": {
+                "wiki_pretrain_dataset": "bnorick_format/128/wiki_pretrain",
+                "bc_pretrain_dataset": "bnorick_format/128/bookcorpus_pretrain"
+            },
+            "512": {
+                "wiki_pretrain_dataset": "bnorick_format/512/wiki_pretrain",
+                "bc_pretrain_dataset": "bnorick_format/512/bookcorpus_pretrain"
+            }
+        }
+    },
+    "mixed_seq_training": {
+        "128": {
+            "num_epochs": 200,
+            "warmup_proportion": 0.02,
+            "learning_rate": 1e-3,
+            "num_workers": 0,
+            "async_worker": true,
+            "decay_rate": 0.99,
+            "decay_step": 1000,
+            "total_training_steps": 200000
+        }
+    },
+    "validation": {
+        "path": "validation_set/"
+    }
+}

bing_bert/bert_base_large_lr.json

@@ -0,0 +1,26 @@
+{
+  "train_batch_size": 4096,
+  "train_micro_batch_size_per_gpu": 16,
+  "steps_per_print": 1000,
+  "prescale_gradients": true,
+  "gradient_predivide_factor": 8,
+  "optimizer": {
+    "type": "Adam",
+    "params": {
+      "lr": 1e-3,
+      "weight_decay": 0.01,
+      "bias_correction": false
+    }
+  },
+  "gradient_clipping": 1.0,
+  "wall_clock_breakdown": false,
+  "fp16": {
+    "enabled": true,
+    "loss_scale": 0
+  },
+  "progressive_layer_drop": {
+    "enabled": true,
+    "theta": 0.5,
+    "gamma": 0.001
+  }
+}

bing_bert/deepspeed_bsz4k_progressive_layer_drop_config_seq128.json

@@ -182,8 +182,8 @@ def train(args,
                     lr_this_step = update_learning_rate(
                         args, config, global_step, optimizer)
 
-                    report_step_metrics(args, lr_this_step, unscaled_loss,
-                                        global_step, current_data_sample_count)
+                report_step_metrics(args, lr_this_step, unscaled_loss,
+                                    global_step, current_data_sample_count)
 
                 model.network.step()
 

bing_bert/deepspeed_train.py

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+base_dir=`pwd`
+
+# Where should we save checkpoints and tensorboard events?
+JOB_NAME=adam_4k_seq128_progressive_layer_drop
+OUTPUT_DIR=${base_dir}/bert_model_outputs
+
+mkdir -p $OUTPUT_DIR
+
+config="--progressive_layer_drop"
+
+NCCL_TREE_THRESHOLD=0 deepspeed \
+${base_dir}/deepspeed_train.py \
+--cf ${base_dir}/bert_base_large_lr.json \
+--max_seq_length 128 \
+--output_dir $OUTPUT_DIR \
+--deepspeed \
+--print_steps 100 \
+--lr_schedule "LE" \
+--job_name $JOB_NAME \
+--deepspeed_config ${base_dir}/deepspeed_bsz4k_progressive_layer_drop_config_seq128.json \
+--data_path_prefix /data/bert \
+${config} \
+&> ${JOB_NAME}.log