conda create -n nmt python=3.6
conda install pytorch cudatoolkit=11.1 -c pytorch -c nvidia
pip install OpenNMT-py
pip install sacrebleu
python download_wmt16_ro_en_data.py
cd NMT_Vanilla
onmt_build_vocab -config wmt16_ro_en.yaml -n_sample -1
onmt_train -config wmt16_ro_en.yaml
onmt_translate -model ../data/wmt16_ro_en/run/model_step_100000.pt -src ../data/wmt16_ro_en/test.en -tgt ../data/wmt16_ro_en/test.ro -output ../data/wmt16_ro_en/pred.txt -gpu 0 -verbose
sacrebleu ../data/wmt16_ro_en/pred.txt < ../data/wmt16_ro_en/test.ro
python eval_diversity.py "../data/wmt16_ro_en/pred.txt"
python download_wmt16_ro_en_data.py
Install SentencePiece using instructions on its repo
cd NMT_SentencePiece
./prepare_wmt16_ro_en.sh
onmt_build_vocab -config wmt16_ro_en.yaml -n_sample -1
onmt_train -config wmt16_ro_en.yaml
onmt_translate -model ../data/wmt16_ro_en/run/model_step_100000.pt -src ../data/wmt16_ro_en/test.en.sp -tgt ../data/wmt16_ro_en/test.ro.sp -output ../data/wmt16_ro_en/run/test.ro.hyp_model_step_100000.sp -gpu 0 -verbose
spm_decode -model ../data/wmt16_ro_en/wmtenro.model -input_format piece < ../data/wmt16_ro_en/run/test.ro.hyp_model_step_100000.sp > ../data/wmt16_ro_en/run/test.ro.hyp_model_step_100000
sacrebleu ../data/wmt16_ro_en/test.ro < ../data/wmt16_ro_en/run/test.ro.hyp_model_step_100000
python download_wmt16_en_de_data.py
git clone https://github.com/pytorch/fairseq
cd fairseq
pip install --editable ./
cd NMT_FairSeq
python fairseq_translate.py # default config
# Other sample configs:
--sampling --sampling-topk 10
--sampling --sampling-topp 0.8
--beam 10
sacrebleu data/wmt16_en_de/pred_topk_0_topp_0_beam_5.txt < data/wmt16_en_de/test.de
python eval_diversity.py "data/wmt16_en_de/pred_topk_0_topp_0_beam_5.txt"
./script_fairseq_translate.py
./script_fairseq_eval.sh
./script_diversity_fairseq.sh