From 8cc4f4a0278406fb3b2ad5a3f9f5d4b5fd329daf Mon Sep 17 00:00:00 2001 From: init-22 Date: Thu, 31 Oct 2024 22:26:53 +0530 Subject: [PATCH 1/6] default dropout rates for workloads are added --- DOCUMENTATION.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 607f47ead..851d85dbc 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -400,6 +400,22 @@ Submissions will be scored based on their performance on the [fixed workload](#f Furthermore, a less computationally expensive subset of the fixed workloads is collected with the [qualification set](#qualification-set). Submitters without enough compute resources to self-report on the full set of fixed and held-out workloads can instead self-report on this smaller qualification set. Well-performing submissions can thereby qualify for computational resources provided by sponsors of the benchmark to be scored on the full benchmark set. +#### Default Dropout Values for Different Workloads: + +| Workload | Dropout Values | +|------------------------|------------------------------------------------------------------------------------------------------| +| cifar | dropout not used | +| criteo 1tb | dropout_rate: 0.0 | +| fastmri | dropout_rate: 0.0 | +| imagenet_resnet | dropout not used | +| imagenet_vit | dropout_rate: 0.0 | +| librispeech_conformer | attention_dropout_rate: 0.0
attention_residual_dropout_rate: 0.1
conv_residual_dropout_rate: 0.0
feed_forward_dropout_rate: 0.0
feed_forward_residual_dropout_rate: 0.1
input_dropout_rate: 0.1 | +| librispeech_deepspeech | input_dropout_rate: 0.1
feed_forward_dropout_rate: 0.1
(Only for JAX - dropout_rate in CudnnLSTM class: 0.0) | +| mnist | dropout not used | +| ogbg | dropout_rate: 0.1 | +| wmt | dropout_rate: 0.1
attention_dropout_rate: 0.1 | + + NOTE: Submitters are no longer required to self-report results for AlgoPerf competition v0.5. #### Fixed workloads From a6fc879e119cc805bafd98ecd086b1243b9a42c7 Mon Sep 17 00:00:00 2001 From: init-22 Date: Thu, 31 Oct 2024 22:45:21 +0530 Subject: [PATCH 2/6] adding the dropout info in fixed workload section --- DOCUMENTATION.md | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 851d85dbc..2decbcb46 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -400,22 +400,6 @@ Submissions will be scored based on their performance on the [fixed workload](#f Furthermore, a less computationally expensive subset of the fixed workloads is collected with the [qualification set](#qualification-set). Submitters without enough compute resources to self-report on the full set of fixed and held-out workloads can instead self-report on this smaller qualification set. Well-performing submissions can thereby qualify for computational resources provided by sponsors of the benchmark to be scored on the full benchmark set. -#### Default Dropout Values for Different Workloads: - -| Workload | Dropout Values | -|------------------------|------------------------------------------------------------------------------------------------------| -| cifar | dropout not used | -| criteo 1tb | dropout_rate: 0.0 | -| fastmri | dropout_rate: 0.0 | -| imagenet_resnet | dropout not used | -| imagenet_vit | dropout_rate: 0.0 | -| librispeech_conformer | attention_dropout_rate: 0.0
attention_residual_dropout_rate: 0.1
conv_residual_dropout_rate: 0.0
feed_forward_dropout_rate: 0.0
feed_forward_residual_dropout_rate: 0.1
input_dropout_rate: 0.1 | -| librispeech_deepspeech | input_dropout_rate: 0.1
feed_forward_dropout_rate: 0.1
(Only for JAX - dropout_rate in CudnnLSTM class: 0.0) | -| mnist | dropout not used | -| ogbg | dropout_rate: 0.1 | -| wmt | dropout_rate: 0.1
attention_dropout_rate: 0.1 | - - NOTE: Submitters are no longer required to self-report results for AlgoPerf competition v0.5. #### Fixed workloads @@ -433,6 +417,23 @@ The currently eight fixed workloads are: | **7** | Molecular property prediction | OGBG | GNN | CE | mAP | 0.28098 | 0.268729 | 18,477 | | **8** | Translation | WMT | Transformer | CE | BLEU | 30.8491 | 30.7219 | 48,151 | +#### Default Dropout Values for Different Workloads: + +| Workload | Dropout Values | +|------------------------|------------------------------------------------------------------------------------------------------| +| cifar | dropout not used | +| criteo 1tb | dropout_rate: 0.0 | +| fastmri | dropout_rate: 0.0 | +| imagenet_resnet | dropout not used | +| imagenet_vit | dropout_rate: 0.0 | +| librispeech_conformer | attention_dropout_rate: 0.0
attention_residual_dropout_rate: 0.1
conv_residual_dropout_rate: 0.0
feed_forward_dropout_rate: 0.0
feed_forward_residual_dropout_rate: 0.1
input_dropout_rate: 0.1 | +| librispeech_deepspeech | input_dropout_rate: 0.1
feed_forward_dropout_rate: 0.1
(Only for JAX - dropout_rate in CudnnLSTM class: 0.0) | +| mnist | dropout not used | +| ogbg | dropout_rate: 0.1 | +| wmt | dropout_rate: 0.1
attention_dropout_rate: 0.1 | + + + #### Randomized workloads In addition to the [fixed and known workloads](#fixed-workloads), there will also be randomized workloads in our benchmark. These randomized workloads will introduce minor modifications to a fixed workload (e.g. small model changes). The exact instances of these randomized workloads will only be created after the submission deadline and are thus unknown to both the submitters as well as the benchmark organizers. The instructions for creating them, i.e. providing a set or distribution of workloads to sample from, will be defined by this working group and made public with the call for submissions, to allow the members of this working group to submit as well as ensure that they do not possess any additional information compared to other submitters. We will refer to the unspecific workloads as *randomized workloads*, e.g. the set or distribution. The specific instance of such a randomized workload we call a *held-out workload*. That is, a held-out workload is a specific sample of a randomized workload that is used for one iteration of the benchmark. While we may reuse randomized workloads between iterations of the benchmark, new held-out workloads will be sampled for each new benchmark iteration. From 19838992f8edb766860f655670215c037ddcc834 Mon Sep 17 00:00:00 2001 From: init-22 Date: Thu, 31 Oct 2024 22:47:07 +0530 Subject: [PATCH 3/6] removing bold headings --- DOCUMENTATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 2decbcb46..0c9c429c6 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -417,7 +417,7 @@ The currently eight fixed workloads are: | **7** | Molecular property prediction | OGBG | GNN | CE | mAP | 0.28098 | 0.268729 | 18,477 | | **8** | Translation | WMT | Transformer | CE | BLEU | 30.8491 | 30.7219 | 48,151 | -#### Default Dropout Values for Different Workloads: +Default Dropout Values for Different Workloads: | Workload | Dropout Values | |------------------------|------------------------------------------------------------------------------------------------------| From 76b084b556af6bd58d1fbf40d5215cce510146b9 Mon Sep 17 00:00:00 2001 From: init-22 Date: Fri, 15 Nov 2024 00:55:52 +0530 Subject: [PATCH 4/6] fix: removed cifar10 and mnist --- DOCUMENTATION.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 0c9c429c6..990656d38 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -421,19 +421,15 @@ Default Dropout Values for Different Workloads: | Workload | Dropout Values | |------------------------|------------------------------------------------------------------------------------------------------| -| cifar | dropout not used | | criteo 1tb | dropout_rate: 0.0 | | fastmri | dropout_rate: 0.0 | | imagenet_resnet | dropout not used | | imagenet_vit | dropout_rate: 0.0 | | librispeech_conformer | attention_dropout_rate: 0.0
attention_residual_dropout_rate: 0.1
conv_residual_dropout_rate: 0.0
feed_forward_dropout_rate: 0.0
feed_forward_residual_dropout_rate: 0.1
input_dropout_rate: 0.1 | | librispeech_deepspeech | input_dropout_rate: 0.1
feed_forward_dropout_rate: 0.1
(Only for JAX - dropout_rate in CudnnLSTM class: 0.0) | -| mnist | dropout not used | | ogbg | dropout_rate: 0.1 | | wmt | dropout_rate: 0.1
attention_dropout_rate: 0.1 | - - #### Randomized workloads In addition to the [fixed and known workloads](#fixed-workloads), there will also be randomized workloads in our benchmark. These randomized workloads will introduce minor modifications to a fixed workload (e.g. small model changes). The exact instances of these randomized workloads will only be created after the submission deadline and are thus unknown to both the submitters as well as the benchmark organizers. The instructions for creating them, i.e. providing a set or distribution of workloads to sample from, will be defined by this working group and made public with the call for submissions, to allow the members of this working group to submit as well as ensure that they do not possess any additional information compared to other submitters. We will refer to the unspecific workloads as *randomized workloads*, e.g. the set or distribution. The specific instance of such a randomized workload we call a *held-out workload*. That is, a held-out workload is a specific sample of a randomized workload that is used for one iteration of the benchmark. While we may reuse randomized workloads between iterations of the benchmark, new held-out workloads will be sampled for each new benchmark iteration. From f72028f1236ab562a6d1d50f43ec099de9325bbb Mon Sep 17 00:00:00 2001 From: init-22 Date: Tue, 19 Nov 2024 21:07:46 +0530 Subject: [PATCH 5/6] fix: ran yapf for passing the checks --- scoring/score_submissions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py index 59295b686..8cc06b15f 100644 --- a/scoring/score_submissions.py +++ b/scoring/score_submissions.py @@ -211,7 +211,8 @@ def main(_): verbosity=0, self_tuning_ruleset=FLAGS.self_tuning_ruleset, strict=FLAGS.strict, - output_dir=FLAGS.output_dir,) + output_dir=FLAGS.output_dir, + ) if not os.path.exists(FLAGS.output_dir): os.mkdir(FLAGS.output_dir) performance_profile.plot_performance_profiles( From d8f07b73c7a6b0d049c513e4b846696ed7df1da8 Mon Sep 17 00:00:00 2001 From: init-22 Date: Fri, 22 Nov 2024 22:15:01 +0530 Subject: [PATCH 6/6] fix: triggering the checks again --- scoring/performance_profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scoring/performance_profile.py b/scoring/performance_profile.py index 0d5ca9770..f4f2d5679 100644 --- a/scoring/performance_profile.py +++ b/scoring/performance_profile.py @@ -321,7 +321,7 @@ def compute_performance_profiles(submissions, df = df[BASE_WORKLOADS + HELDOUT_WORKLOADS] # Sort workloads alphabetically (for better display) df = df.reindex(sorted(df.columns), axis=1) - + # Save time to target dataframe df.to_csv(os.path.join(output_dir, 'time_to_targets.csv')) # For each held-out workload set to inf if the base workload is inf or nan