Merge branch 'master' of https://github.com/huseinzol05/malaya

mesolitica · Mar 27, 2024 · b6e3160 · b6e3160
2 parents b329192 + 40e9f26
commit b6e3160
Show file tree

Hide file tree

Showing 2 changed files with 496 additions and 0 deletions.
diff --git a/pretrained-model/deberta-v3/run-base-v2.sh b/pretrained-model/deberta-v3/run-base-v2.sh
@@ -0,0 +1,21 @@
+rm -rf /dev/shm/*
+WANDB_PROJECT=deberta-base \
+~/.local/bin/torchrun --nproc_per_node 4 \
+-m run-mlm \
+--tokenizer_name malaysia-ai/bpe-tokenizer \
+--config_name microsoft/deberta-v3-base \
+--per_device_train_batch_size 75 \
+--do_train \
+--max_seq_len 512 \
+--output_dir debertav2-base-malaysian-v2 \
+--mlm_probability 0.15 \
+--train_file "mosaic-combine-512" \
+--logging_steps="1" \
+--save_steps="1000" \
+--bf16 \
+--learning_rate 2e-4 \
+--warmup_steps 10000 \
+--do_train \
+--do_eval false \
+--num_train_epochs 10 \
+--save_total_limit 2