Skip to content

Commit 5bc16bf

Browse files
mwawrzosmmarcinkiewicz
mwawrzos
and
mmarcinkiewicz
authored
ComplianceChecker update (#83)
* new keys according to #78 and #80 * RNN-T constants * RNN-T update constants * [compliance_checker] update to rules 1.0 * [compliance_checker] add gradient_accumulation_steps * update constants * [compliance_checker] RNN-T rules * add rnnt and unet3d benchmarks * Revert "RNN-T update constants" This reverts commit 03b986a. * Revert "RNN-T constants" This reverts commit b550182. * [compliance_checker] check weights_initialization based on metadata * Add unet3d * [compliance_checker][RNN-T] missing weights initialization check * [compliance_checker][Unet3D] target 0.91 -> 0.908 after mlcommons/training@149c2b8 Co-authored-by: michalm <[email protected]>
1 parent d4da635 commit 5bc16bf

26 files changed

+897
-1
lines changed

‎mlperf_logging/compliance_checker/1.0.0/closed_bert.yaml

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
2+
- KEY:
3+
NAME: global_batch_size
4+
REQ: EXACTLY_ONE
5+
POST: >
6+
s['global_batch_size'] = v['value']
7+
8+
- KEY:
9+
NAME: opt_base_learning_rate
10+
REQ: EXACTLY_ONE
11+
12+
- KEY:
13+
NAME: opt_epsilon
14+
REQ: EXACTLY_ONE
15+
16+
- KEY:
17+
NAME: opt_learning_rate_training_steps
18+
REQ: EXACTLY_ONE
19+
20+
- KEY:
21+
NAME: opt_learning_rate_warmup_steps
22+
REQ: EXACTLY_ONE
23+
24+
- KEY:
25+
NAME: num_warmup_steps
26+
REQ: EXACTLY_ONE
27+
28+
- KEY:
29+
NAME: start_warmup_step
30+
REQ: EXACTLY_ONE
31+
32+
- KEY:
33+
NAME: opt_lamb_beta_1
34+
REQ: EXACTLY_ONE
35+
36+
- KEY:
37+
NAME: opt_lamb_beta_2
38+
REQ: EXACTLY_ONE
39+
40+
- KEY:
41+
NAME: opt_lamb_weight_decay_rate
42+
REQ: EXACTLY_ONE
43+
44+
- KEY:
45+
NAME: eval_accuracy
46+
REQ: AT_LEAST_ONE
47+
CHECK:
48+
- "'epoch_num' in v['metadata']"
49+
ATLEAST_ONE_CHECK: "(v['value'] >= 0.712 if s['global_batch_size'] < 8192 else v['value'] >= 0.706) and v['value'] < 1.0"

‎mlperf_logging/compliance_checker/1.0.0/closed_common.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
2+
- KEY:
3+
NAME: submission_benchmark
4+
REQ: EXACTLY_ONE
5+
CHECK: " v['value'] in ['resnet', 'ssd', 'maskrcnn', 'transformer', 'gnmt', 'minigo', 'dlrm', 'bert', 'rnnt', 'unet3d'] "
6+
POST: " enqueue_config('1.0.0/closed_{}.yaml'.format(v['value'])) "
7+
8+
- KEY:
9+
NAME: gradient_accumulation_steps
10+
REQ: EXACTLY_ONE
11+
CHECK: " v['value'] > 0 "

‎mlperf_logging/compliance_checker/1.0.0/closed_dlrm.yaml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
2+
- KEY:
3+
NAME: global_batch_size
4+
REQ: EXACTLY_ONE
5+
6+
- KEY:
7+
NAME: opt_base_learning_rate
8+
REQ: EXACTLY_ONE
9+
10+
- KEY:
11+
NAME: opt_learning_rate_warmup_steps
12+
REQ: EXACTLY_ONE
13+
14+
- KEY:
15+
NAME: lr_decay_start_steps
16+
REQ: EXACTLY_ONE
17+
18+
- KEY:
19+
NAME: sgd_opt_base_learning_rate
20+
REQ: EXACTLY_ONE
21+
22+
- KEY:
23+
NAME: sgd_opt_learning_rate_decay_poly_power
24+
REQ: EXACTLY_ONE
25+
CHECK: " v['value'] == 2 "
26+
27+
- KEY:
28+
NAME: sgd_opt_learning_rate_decay_steps
29+
REQ: EXACTLY_ONE
30+
31+
- KEY:
32+
NAME: eval_accuracy
33+
REQ: AT_LEAST_ONE
34+
CHECK:
35+
- "'epoch_num' in v['metadata']"
36+
ATLEAST_ONE_CHECK: "v['value'] >= 0.8025 and v['value'] < 1.0"

‎mlperf_logging/compliance_checker/1.0.0/closed_gnmt.yaml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
- KEY:
2+
NAME: global_batch_size
3+
REQ: EXACTLY_ONE
4+
5+
- KEY:
6+
NAME: opt_base_learning_rate
7+
REQ: EXACTLY_ONE
8+
9+
- KEY:
10+
NAME: opt_learning_rate_alt_decay_func
11+
REQ: EXACTLY_ONE
12+
13+
- KEY:
14+
NAME: opt_learning_rate_decay_factor
15+
REQ: EXACTLY_ONE
16+
17+
- KEY:
18+
NAME: opt_learning_rate_decay_interval
19+
REQ: EXACTLY_ONE
20+
21+
- KEY:
22+
NAME: opt_learning_rate_decay_steps
23+
REQ: EXACTLY_ONE
24+
25+
- KEY:
26+
NAME: opt_learning_rate_remain_steps
27+
REQ: EXACTLY_ONE
28+
29+
- KEY:
30+
NAME: max_sequence_length
31+
REQ: EXACTLY_ONE
32+
33+
- KEY:
34+
NAME: opt_learning_rate_alt_warmup_func
35+
REQ: EXACTLY_ONE
36+
37+
- KEY:
38+
NAME: opt_learning_rate_warmup_steps
39+
REQ: EXACTLY_ONE
40+
41+
- KEY:
42+
NAME: eval_accuracy
43+
REQ: AT_LEAST_ONE
44+
CHECK:
45+
- "'epoch_num' in v['metadata']"
46+
ATLEAST_ONE_CHECK: "v['value'] >= 0.240 and v['value'] < 1"
47+

‎mlperf_logging/compliance_checker/1.0.0/closed_maskrcnn.yaml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
2+
- KEY:
3+
NAME: global_batch_size
4+
REQ: EXACTLY_ONE
5+
CHECK: " v['value'] > 0"
6+
7+
- KEY:
8+
NAME: opt_base_learning_rate
9+
REQ: EXACTLY_ONE
10+
CHECK: " is_integer(v['value'] / 0.02) "
11+
12+
- KEY:
13+
NAME: opt_learning_rate_warmup_steps
14+
REQ: EXACTLY_ONE
15+
CHECK: " v['value'] >= 0"
16+
17+
- KEY:
18+
NAME: opt_learning_rate_warmup_factor
19+
REQ: EXACTLY_ONE
20+
CHECK: " v['value'] > 0"
21+
22+
- KEY:
23+
NAME: num_image_candidates
24+
REQ: EXACTLY_ONE
25+
CHECK: " is_integer(v['value'] / 1000.0)"
26+
27+
- KEY:
28+
NAME: opt_learning_rate_decay_factor
29+
REQ: EXACTLY_ONE
30+
CHECK: " v['value'] == 0.1 "
31+
32+
- KEY:
33+
NAME: opt_learning_rate_decay_steps
34+
REQ: EXACTLY_ONE
35+
CHECK: " len(v['value']) == 2 "
36+
37+
- KEY:
38+
NAME: min_image_size
39+
REQ: EXACTLY_ONE
40+
CHECK: " v['value'] >= 0 "
41+
42+
- KEY:
43+
NAME: max_image_size
44+
REQ: EXACTLY_ONE
45+
CHECK: " v['value'] >= 0 "
46+
47+
- KEY:
48+
NAME: eval_accuracy
49+
REQ: AT_LEAST_ONE
50+
CHECK:
51+
- "'BBOX' in v['value']"
52+
- "'SEGM' in v['value']"
53+
- "'epoch_num' in v['metadata']"
54+
POST: " s['accuracy_printed'] = True "
55+
ATLEAST_ONE_CHECK: "v['value']['BBOX'] >= 0.377 and v['value']['BBOX'] < 1.0 and v['value']['SEGM'] >= 0.339 and v['value']['SEGM'] < 1.0"
56+
57+

‎mlperf_logging/compliance_checker/1.0.0/closed_minigo.yaml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
- KEY:
2+
NAME: train_batch_size
3+
REQ: EXACTLY_ONE
4+
5+
- KEY:
6+
NAME: lr_boundaries
7+
REQ: EXACTLY_ONE
8+
9+
- KEY:
10+
NAME: lr_rates
11+
REQ: EXACTLY_ONE
12+
13+
- KEY:
14+
NAME: actual_selfplay_games_per_generation
15+
REQ: AT_LEAST_ONE
16+
CHECK:
17+
- "v['value'] >= 8192"
18+
19+
- KEY:
20+
NAME: min_selfplay_games_per_generation
21+
REQ: EXACTLY_ONE
22+
CHECK:
23+
- "v['value'] == 8192"
24+
25+
- KEY:
26+
NAME: eval_accuracy
27+
REQ: AT_LEAST_ONE
28+
CHECK:
29+
- "'epoch_num' in v['metadata']"
30+
ATLEAST_ONE_CHECK: "v['value'] >= 0.5 and v['value'] < 1.0"
31+
32+
- KEY:
33+
NAME: block_start
34+
35+
- KEY:
36+
NAME: block_stop
37+
38+
- KEY:
39+
NAME: train_samples
40+
41+
- KEY:
42+
NAME: eval_samples
43+

‎mlperf_logging/compliance_checker/1.0.0/closed_resnet.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
2+
- KEY:
3+
NAME: global_batch_size
4+
REQ: EXACTLY_ONE
5+
CHECK: " v['value'] > 0"
6+
7+
- KEY:
8+
NAME: opt_name
9+
REQ: EXACTLY_ONE
10+
CHECK: " v['value'] in ['sgd', 'lars'] "
11+
POST: " enqueue_config('1.0.0/closed_resnet_{}.yaml'.format(v['value'])) "
12+
13+
- KEY:
14+
NAME: eval_accuracy
15+
REQ: AT_LEAST_ONE
16+
CHECK:
17+
- "'epoch_num' in v['metadata']"
18+
ATLEAST_ONE_CHECK: "v['value'] >= 0.7590 and v['value'] < 1.0"

‎mlperf_logging/compliance_checker/1.0.0/closed_resnet_lars.yaml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
2+
- KEY:
3+
NAME: lars_opt_base_learning_rate
4+
REQ: EXACTLY_ONE
5+
6+
- KEY:
7+
NAME: lars_opt_end_learning_rate
8+
REQ: EXACTLY_ONE
9+
CHECK: " v['value'] == 0.0001 "
10+
11+
- KEY:
12+
NAME: lars_opt_learning_rate_decay_poly_power
13+
REQ: EXACTLY_ONE
14+
CHECK: " v['value'] == 2 "
15+
16+
- KEY:
17+
NAME: lars_opt_learning_rate_decay_steps
18+
REQ: EXACTLY_ONE
19+
20+
- KEY:
21+
NAME: lars_epsilon
22+
REQ: EXACTLY_ONE
23+
CHECK: " v['value'] == 0.0 "
24+
25+
- KEY:
26+
NAME: lars_opt_learning_rate_warmup_epochs
27+
REQ: EXACTLY_ONE
28+
29+
- KEY:
30+
NAME: lars_opt_momentum
31+
REQ: EXACTLY_ONE
32+
33+
- KEY:
34+
NAME: lars_opt_weight_decay
35+
REQ: EXACTLY_ONE
36+
CHECK: " is_integer(math.log2(v['value'] / 0.0001)) "
37+

‎mlperf_logging/compliance_checker/1.0.0/closed_resnet_sgd.yaml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
2+
- KEY:
3+
NAME: sgd_opt_base_learning_rate
4+
REQ: EXACTLY_ONE
5+
CHECK: " is_integer(v['value'] / 0.001) "
6+
7+
- KEY:
8+
NAME: sgd_opt_end_learning_rate
9+
REQ: EXACTLY_ONE
10+
CHECK: " v['value'] == 0.0001 "
11+
12+
- KEY:
13+
NAME: sgd_opt_learning_rate_decay_poly_power
14+
REQ: EXACTLY_ONE
15+
16+
- KEY:
17+
NAME: sgd_opt_learning_rate_decay_steps
18+
REQ: EXACTLY_ONE
19+
20+
- KEY:
21+
NAME: sgd_opt_weight_decay
22+
REQ: EXACTLY_ONE
23+
CHECK: " is_integer(math.log2(v['value'] / 0.0001)) "
24+
25+
- KEY:
26+
NAME: sgd_opt_momentum
27+
REQ: EXACTLY_ONE
28+
29+
- KEY:
30+
NAME: model_bn_span
31+
REQ: EXACTLY_ONE
32+
33+
- KEY:
34+
NAME: opt_learning_rate_warmup_epochs
35+
REQ: EXACTLY_ONE
36+

0 commit comments

Comments
 (0)