-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile.submit
112 lines (94 loc) · 3.45 KB
/
Makefile.submit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# -*-makefile-*-
#--------------------------------------------------------------------
# Joerg Tiedemann 2017-06-07
#
# generic target to submit a job via sbatch
# works on CSC only so far
#--------------------------------------------------------------------
# enable e-mail notification by setting EMAIL
WHOAMI = $(shell whoami)
ifeq ("$(WHOAMI)","tiedeman")
EMAIL = [email protected]
endif
# job-specific settings (overwrite if necessary)
# HPC_EXTRA: additional SBATCH commands
HPC_MEM = 4g
HPC_TIME = 72:00
HPC_NODES = 1
HPC_CORES = 1
HPC_QUEUE = small
# HPC_MODULES = nlpl-opus python-env/3.4.1
# HPC_MODULES = parallel python-env nlpl-efmaral nlpl-opus nlpl-moses
# HPC_MODULES = parallel python-env nlpl-efmaral nlpl-moses
# HPC_MODULES = parallel git nlpl-cwb nlpl-opus nlpl-moses
HPC_MODULES = perl parallel git nlpl-cwb nlpl-moses
HPC_EXTRA =
SLURM_MAX_NR_JOBS := 200
%.submit:
while [ `squeue -u ${WHOAMI}` -lt ${SLURM_MAX_NR_JOBS} ]; do echo "waiting for space in the queue";sleep 1; done
echo '#!/bin/bash -l' > $@
echo '#SBATCH -J "${@:.submit=}"' >>$@
echo '#SBATCH -o ${@:.submit=}.out.%j' >> $@
echo '#SBATCH -e ${@:.submit=}.err.%j' >> $@
echo '#SBATCH --mem=${HPC_MEM}' >> $@
ifeq (${shell hostname --domain},bullx)
ifdef HPC_DISK
echo '#SBATCH --gres=nvme:${HPC_DISK}' >> $@
endif
echo '#SBATCH --account=${CSC_PROJECT}' >> $@
endif
ifeq (${shell hostname --domain},mahti.csc.fi)
echo '#SBATCH --account=${CSC_PROJECT}' >> $@
endif
ifneq (${EMAIL},)
echo '#SBATCH --mail-type=END' >> $@
echo '#SBATCH --mail-user=${EMAIL}' >> $@
endif
echo '#SBATCH -n ${HPC_CORES}' >> $@
echo '#SBATCH -N ${HPC_NODES}' >> $@
echo '#SBATCH -p ${HPC_QUEUE}' >> $@
echo '#SBATCH -t ${HPC_TIME}:00' >> $@
echo '${HPC_EXTRA}' >> $@
echo 'module use -a /proj/nlpl/modules' >> $@
echo 'module use -a /projappl/nlpl/software/modules/etc' >> $@
for m in ${HPC_MODULES}; do \
echo "module load $$m" >> $@; \
done
echo 'module list' >> $@
echo 'cd $${SLURM_SUBMIT_DIR:-.}' >> $@
echo 'pwd' >> $@
echo 'echo "Starting at `date`"' >> $@
echo '${MAKE} -j ${HPC_CORES} ${MAKEARGS} ${@:.submit=}' >> $@
echo 'echo "Finishing at `date`"' >> $@
sbatch ${SBATCH_ARGS} $@
mv $@ $@-${shell date "+%Y-%m-%d"}
# echo 'module purge' >> $@
## run in a local tmp dir instead
## --> reduce I/O! but requires massive copying ...
## --> requires enough space in TMPDIR!
## TODO: should we use --ignore-errors in make call?
%-local:
mkdir -p ${TMPDIR}/OPUS/corpus/${CORPUS}
-rsync ${OPUSCORPORA}/Makefile* ${TMPDIR}/OPUS/corpus/
-ln -s ${OPUSTOOLS} ${TMPDIR}/OPUS/tools
-rsync -av ${CORPUSHOME} ${TMPDIR}/OPUS/corpus/
-${MAKE} -C $(patsubst ${OPUSHOME}%,${TMPDIR}/OPUS%,${PWD}) \
--keep-going \
OPUSHOME=${TMPDIR}/OPUS ${@:-local=}
-rsync -zav ${TMPDIR}/OPUS/corpus/${CORPUS} ${OPUSHOME}/corpus/
-rsync -zav ${TMPDIR}/OPUS/web/${CORPUS} ${OPUSHOME}/web/
-rsync -zav ${TMPDIR}/OPUS/web/${CORPUS}*php ${OPUSHOME}/web/
-rm -fr ${TMPDIR}/OPUS/corpus/${CORPUS}
-rm -fr ${TMPDIR}/OPUS/web/${CORPUS}*
# run a recipe on a local copy in TMPDIR
# NOTE: may overwrite existing files
# when copying back!!!
%-runlocal:
mkdir -p ${TMPDIR}/OPUS/corpus
rsync -a ${CORPUSHOME} ${TMPDIR}/OPUS/corpus/
-ln -s ${OPUSHOME}/* ${TMPDIR}/OPUS/
-ln -s ${OPUSCORPORA}/* ${TMPDIR}/OPUS/corpus/
${MAKE} -C ${patsubst ${OPUSHOME}/%,${TMPDIR}/OPUS/%,${shell pwd}} \
OPUSHOME=${TMPDIR}/OPUS ${@:-runlocal=}
rsync -a ${TMPDIR}/OPUS/corpus/${CORPUS} ${OPUSCORPORA}/