171 lines
3.9 KiB
JSON
171 lines
3.9 KiB
JSON
|
{
|
||
|
|
"exp": "exp22",
|
||
|
|
"cell": "s1_string",
|
||
|
|
"variant": "direct",
|
||
|
|
"arm": "correct",
|
||
|
|
"seed": 0,
|
||
|
|
"composable": [
|
||
|
|
"interleave_halves",
|
||
|
|
"reverse_blocks3",
|
||
|
|
"reverse_first_half",
|
||
|
|
"swap_pairs"
|
||
|
|
],
|
||
|
|
"distractors": [
|
||
|
|
"charset",
|
||
|
|
"nice_chars"
|
||
|
|
],
|
||
|
|
"atoms": [
|
||
|
|
"charset",
|
||
|
|
"interleave_halves",
|
||
|
|
"nice_chars",
|
||
|
|
"reverse_blocks3",
|
||
|
|
"reverse_first_half",
|
||
|
|
"swap_pairs"
|
||
|
|
],
|
||
|
|
"fourth_atom": "reverse_blocks3",
|
||
|
|
"chain_lengths": [
|
||
|
|
5,
|
||
|
|
7,
|
||
|
|
8
|
||
|
|
],
|
||
|
|
"excluded_pairs": [
|
||
|
|
[
|
||
|
|
"reverse_first_half",
|
||
|
|
"swap_pairs"
|
||
|
|
]
|
||
|
|
],
|
||
|
|
"train_comps": [
|
||
|
|
"reverse_first_half|interleave_halves",
|
||
|
|
"interleave_halves|reverse_first_half",
|
||
|
|
"reverse_blocks3|swap_pairs",
|
||
|
|
"swap_pairs|reverse_blocks3",
|
||
|
|
"reverse_blocks3|interleave_halves",
|
||
|
|
"interleave_halves|reverse_blocks3"
|
||
|
|
],
|
||
|
|
"heldout_comps": [
|
||
|
|
"reverse_blocks3|reverse_first_half",
|
||
|
|
"reverse_first_half|reverse_blocks3",
|
||
|
|
"swap_pairs|interleave_halves",
|
||
|
|
"interleave_halves|swap_pairs"
|
||
|
|
],
|
||
|
|
"heldout_labels": {
|
||
|
|
"reverse_blocks3|reverse_first_half": "never_paired",
|
||
|
|
"reverse_first_half|reverse_blocks3": "never_paired",
|
||
|
|
"swap_pairs|interleave_halves": "never_paired",
|
||
|
|
"interleave_halves|swap_pairs": "never_paired"
|
||
|
|
},
|
||
|
|
"n_comp_train_per_pairing": 200,
|
||
|
|
"total_comp_train": 1200,
|
||
|
|
"compositions": {
|
||
|
|
"reverse_first_half|interleave_halves": {
|
||
|
|
"role": "train",
|
||
|
|
"n_train": 200,
|
||
|
|
"n_seen_eval": 200
|
||
|
|
},
|
||
|
|
"interleave_halves|reverse_first_half": {
|
||
|
|
"role": "train",
|
||
|
|
"n_train": 200,
|
||
|
|
"n_seen_eval": 200
|
||
|
|
},
|
||
|
|
"reverse_blocks3|swap_pairs": {
|
||
|
|
"role": "train",
|
||
|
|
"n_train": 200,
|
||
|
|
"n_seen_eval": 200
|
||
|
|
},
|
||
|
|
"swap_pairs|reverse_blocks3": {
|
||
|
|
"role": "train",
|
||
|
|
"n_train": 200,
|
||
|
|
"n_seen_eval": 200
|
||
|
|
},
|
||
|
|
"reverse_blocks3|interleave_halves": {
|
||
|
|
"role": "train",
|
||
|
|
"n_train": 200,
|
||
|
|
"n_seen_eval": 200
|
||
|
|
},
|
||
|
|
"interleave_halves|reverse_blocks3": {
|
||
|
|
"role": "train",
|
||
|
|
"n_train": 200,
|
||
|
|
"n_seen_eval": 200
|
||
|
|
},
|
||
|
|
"reverse_blocks3|reverse_first_half": {
|
||
|
|
"role": "heldout",
|
||
|
|
"label": "never_paired",
|
||
|
|
"n_eval": 200,
|
||
|
|
"distinct_answers": 200,
|
||
|
|
"top_answer_share": 0.005
|
||
|
|
},
|
||
|
|
"reverse_first_half|reverse_blocks3": {
|
||
|
|
"role": "heldout",
|
||
|
|
"label": "never_paired",
|
||
|
|
"n_eval": 200,
|
||
|
|
"distinct_answers": 200,
|
||
|
|
"top_answer_share": 0.005
|
||
|
|
},
|
||
|
|
"swap_pairs|interleave_halves": {
|
||
|
|
"role": "heldout",
|
||
|
|
"label": "never_paired",
|
||
|
|
"n_eval": 200,
|
||
|
|
"distinct_answers": 200,
|
||
|
|
"top_answer_share": 0.005
|
||
|
|
},
|
||
|
|
"interleave_halves|swap_pairs": {
|
||
|
|
"role": "heldout",
|
||
|
|
"label": "never_paired",
|
||
|
|
"n_eval": 200,
|
||
|
|
"distinct_answers": 200,
|
||
|
|
"top_answer_share": 0.005
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"train_heldout_chain_overlap": [],
|
||
|
|
"heldout_instance_collisions": 0,
|
||
|
|
"coverage": {
|
||
|
|
"train_inner_atoms": [
|
||
|
|
"interleave_halves",
|
||
|
|
"reverse_blocks3",
|
||
|
|
"reverse_first_half",
|
||
|
|
"swap_pairs"
|
||
|
|
],
|
||
|
|
"train_outer_atoms": [
|
||
|
|
"interleave_halves",
|
||
|
|
"reverse_blocks3",
|
||
|
|
"reverse_first_half",
|
||
|
|
"swap_pairs"
|
||
|
|
],
|
||
|
|
"heldout_inner_atoms": [
|
||
|
|
"interleave_halves",
|
||
|
|
"reverse_blocks3",
|
||
|
|
"reverse_first_half",
|
||
|
|
"swap_pairs"
|
||
|
|
],
|
||
|
|
"heldout_outer_atoms": [
|
||
|
|
"interleave_halves",
|
||
|
|
"reverse_blocks3",
|
||
|
|
"reverse_first_half",
|
||
|
|
"swap_pairs"
|
||
|
|
],
|
||
|
|
"heldout_inner_count": 4,
|
||
|
|
"heldout_outer_count": 4,
|
||
|
|
"per_slot_coverage": {
|
||
|
|
"interleave_halves": {
|
||
|
|
"train_as_inner": 2,
|
||
|
|
"train_as_outer": 2
|
||
|
|
},
|
||
|
|
"reverse_blocks3": {
|
||
|
|
"train_as_inner": 2,
|
||
|
|
"train_as_outer": 2
|
||
|
|
},
|
||
|
|
"reverse_first_half": {
|
||
|
|
"train_as_inner": 1,
|
||
|
|
"train_as_outer": 1
|
||
|
|
},
|
||
|
|
"swap_pairs": {
|
||
|
|
"train_as_inner": 1,
|
||
|
|
"train_as_outer": 1
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"train_covers_both_slots": true,
|
||
|
|
"pairing_collisions": []
|
||
|
|
},
|
||
|
|
"collisions_total": 0,
|
||
|
|
"chain_lengths_out_of_range": []
|
||
|
|
}
|