-
Notifications
You must be signed in to change notification settings - Fork 0
/
interpol_losses_job
86 lines (75 loc) · 3.23 KB
/
interpol_losses_job
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
#!/bin/sh
#SBATCH --partition=maxgpu ## or allgpu / cms / cms-uhh / maxgpu
#SBATCH --time=03:00:00
#SBATCH --nodes=2
#SBATCH --job-name interpol_losses # give job unique name
#SBATCH --output ./interpol_losses_log/%j-std.out # terminal output
#SBATCH --error ./interpol_losses_log/%j-std.err
#S BATCH --mail-type END
#S BATCH --mail-user [email protected]
#SBATCH --constraint=GPU
# #SBATCH --nodelist=max-cmsg004 # you can select specific nodes, if necessary
#S BATCH --constraint="P100|V100" # ask for specific types of GPUs
# (GPU drivers, anaconda, .bashrc, etc) --> likely not necessary if you're installing your own cudatoolkit & cuDNN versions
#source /etc/profile.d/modules.sh
#module load maxwell
#module load cuda
#module load anaconda/3
#module load mpi/mpich-x86_64
module load mpi/openmpi3-x86_64
. ~/.bashrc
export PATH="/home/kaemmlel/miniconda3/envs/torch/bin:$PATH"
set -x
# export PATH="/home/kaemmlel/miniconda3/envs/torch/bin:$PATH"
export PYTHONPATH="" # openmpi module adds python 2 packages to pythonpath
# Recipe for gloo protocol
#export MASTER_HOST=$(hostname -i)
#export MASTER_PORT=$(python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()')
# go to your folder with your python scripts
cd ~/Projects/amazing_ai
# START=s
# END=b
# --file-end @events_${END}_sr.h5 \
# --a-to-a \
# --interpol-radius 50 \
MPI4PY_RC_THREADS=0 time mpirun --use-hwthread-cpus \
python -m amazing_ai.scripts.interpol_losses \
--file-start @events_b_sr.h5 @events_s_sr.h5 \
--n-starts 1000 1000 \
--file-end @events_b_sr.h5 \
--n-ends 1000 \
--interpol-method linear --interpol-radius 5 \
-o @interpol_{interpol_method}/{interpol_radius}/interpol_{jobid}_{n_starts}_{n_ends}_ir={interpol_radius}_im={interpol_method}{flags}.h5 \
--steps 100 \
--npix 42 \
--block-size 50 \
--model-batch-size 10000 \
--model models/13554747_model=1_npix=42_ts=0.6_data=@events_b_sr_images_42_center_hist.h5_noise=0.0008_ls=40 \
--mpi
# MPI4PY_RC_THREADS=0 time mpirun --use-hwthread-cpus \
# python -m amazing_ai.scripts.interpol_losses \
# --file-start @events_b_sr.h5 @events_s_sr.h5 \
# --n-starts 10000 10000 \
# --file-end @events_b_sr.h5 \
# --n-ends 1000 \
# --interpol-method emd \
# -o @interpol_losses_any-b_{n_starts}_{n_ends}_ir={interpol_radius}_im={interpol_method}{flags}.h5 \
# --steps 100 \
# --npix 42 \
# --block-size 50 \
# --model-batch-size 10000 \
# --model models/13554747_model=1_npix=42_ts=0.6_data=@events_b_sr_images_42_center_hist.h5_noise=0.0008_ls=40 \
# --mpi
# MPI4PY_RC_THREADS=0 time mpirun --use-hwthread-cpus \
# python -m amazing_ai.scripts.interpol_losses \
# --file-start @events_b_sr.h5 @events_s_sr.h5 \
# --n-starts 10000 10000 \
# --file-end @events_b_sr.h5 \
# --n-ends 600 \
# --interpol-method linear \
# -o @interpol_losses_any-b_{n_starts}_{n_ends}_im={interpol_method}{flags}.h5 \
# --steps 100 \
# --block-size 50 \
# --model-batch-size 10000 \
# --model out/12139736*/ \
# --mpi