add direct support for multilingual eval in run.sh (#42)

benchmark/datasets_multilingual.txt

@@ -0,0 +1,12 @@
+multilingual_mscoco_captions,es
+multilingual_mscoco_captions,it
+multilingual_mscoco_captions,ko
+multilingual_mscoco_captions,pl
+multilingual_mscoco_captions,ru
+multilingual_mscoco_captions,tr
+multilingual_mscoco_captions,zh
+multilingual_mscoco_captions,en
+imagenet1k,zh
+imagenet1k,it
+imagenet1k,jp
+imagenet1k,en

benchmark/run.sh

@@ -6,9 +6,14 @@ WORKERS=4
 MODEL=$1
 PRETRAINED=$2
 ROOT=clip_benchmark_datasets
-LANGUAGE=en
 for ds in $DATASETS;do
+    LANGUAGE=$(echo $ds|cut -d, -f2)
+    ds=$(echo $ds|cut -d, -f1)
     ds_name=`echo $ds|tr '/' '_'`
+    if [[ "$LANGUAGE" == "" ]]
+    then
+	$LANGUAGE="en"
+    fi
     for model in $MODELS;do
        arch=$(echo $model|cut -d, -f1)
        pretrained=$(echo $model|cut -d, -f2)
@@ -20,6 +25,7 @@ for ds in $DATASETS;do
                TASK=zeroshot_classification
        fi
        echo "$ds_name"
+       echo "$LANGUAGE"
        echo $TASK
        $RUN --dataset=$ds --language=$LANGUAGE --annotation_file=$ROOT/$ds/captions.txt --dataset_root $ROOT/$ds --task=$TASK --pretrained=$pretrained --model=$arch --output="${ds_name}_${name}_${arch}_${LANGUAGE}.json"  --batch_size=$BS --num_workers=$WORKERS
     done