[{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"GPT-4.1-mini","acc_reference":0.7589,"acc_perturbed":0.7139,"acc_change":-0.045,"agreement":0.725,"both_correct":0.5989},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"GPT-4.1-mini","acc_reference":0.7589,"acc_perturbed":0.775,"acc_change":0.0161,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"GPT-4.1-mini","acc_reference":0.7589,"acc_perturbed":0.7911,"acc_change":0.0322,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"GPT-4.1-mini","acc_reference":0.7589,"acc_perturbed":0.755,"acc_change":-0.0039,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"GPT-4.1-mini","acc_reference":0.7589,"acc_perturbed":0.6956,"acc_change":-0.0633,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"GPT-4.1-mini","acc_reference":0.7589,"acc_perturbed":0.7667,"acc_change":0.0078,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"GPT-4.1-mini","acc_reference":0.7589,"acc_perturbed":0.7628,"acc_change":0.0039,"agreement":0.8361,"both_correct":0.6789},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.8983,"acc_change":-0.0322,"agreement":0.8778,"both_correct":0.8533},{"perturbation":"form-reshuffle","label":"Reorder only","family":"Share of premises restated","description":"Premise order and names list reshuffled; no premise restated.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.9317,"acc_change":0.0011,"agreement":0.9256,"both_correct":0.8939},{"perturbation":"form-inv25","label":"Restate 25%","family":"Share of premises restated","description":"25% of the premises restated with the inverse phrase (“B is behind A”); premise order unchanged.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.9256,"acc_change":-0.005,"agreement":0.9128,"both_correct":0.8844},{"perturbation":"form-inv50","label":"Restate 50%","family":"Share of premises restated","description":"50% of the premises restated with the inverse phrase (“B is behind A”); premise order unchanged.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.9033,"acc_change":-0.0272,"agreement":0.8989,"both_correct":0.8667},{"perturbation":"form-inv75","label":"Restate 75%","family":"Share of premises restated","description":"75% of the premises restated with the inverse phrase (“B is behind A”); premise order unchanged.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.905,"acc_change":-0.0256,"agreement":0.9,"both_correct":0.8678},{"perturbation":"form-inv100","label":"Restate 100%","family":"Share of premises restated","description":"Every premise restated with the inverse phrase (“B is behind A”); premise order unchanged.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.8406,"acc_change":-0.09,"agreement":0.8022,"both_correct":0.7867},{"perturbation":"form-converse","label":"Restate 100% + reorder","family":"Share of premises restated","description":"Every premise restated with the inverse phrase, and the premise order and names list reshuffled.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.8411,"acc_change":-0.0894,"agreement":0.7972,"both_correct":0.7844},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.9394,"acc_change":0.0089,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.7094,"acc_change":-0.2211,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.7817,"acc_change":-0.1489,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.7894,"acc_change":-0.1411,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.8906,"acc_change":-0.04,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"GPT-4.1","acc_reference":0.9306,"acc_perturbed":0.9344,"acc_change":0.0039,"agreement":0.9639,"both_correct":0.9144},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"GPT-5-mini","acc_reference":0.7,"acc_perturbed":0.72,"acc_change":0.02,"agreement":0.6711,"both_correct":0.5456},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"GPT-5-mini","acc_reference":0.7,"acc_perturbed":0.7283,"acc_change":0.0283,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"GPT-5-mini","acc_reference":0.7,"acc_perturbed":0.705,"acc_change":0.005,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"GPT-5-mini","acc_reference":0.7,"acc_perturbed":0.6583,"acc_change":-0.0417,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"GPT-5-mini","acc_reference":0.7,"acc_perturbed":0.6672,"acc_change":-0.0328,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"GPT-5-mini","acc_reference":0.7,"acc_perturbed":0.7433,"acc_change":0.0433,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"GPT-5-mini","acc_reference":0.7,"acc_perturbed":0.6978,"acc_change":-0.0022,"agreement":0.7511,"both_correct":0.5744},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8706,"acc_change":-0.0706,"agreement":0.8533,"both_correct":0.8328},{"perturbation":"form-reshuffle","label":"Reorder only","family":"Share of premises restated","description":"Premise order and names list reshuffled; no premise restated.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.9361,"acc_change":-0.005,"agreement":0.91,"both_correct":0.8939},{"perturbation":"form-inv25","label":"Restate 25%","family":"Share of premises restated","description":"25% of the premises restated with the inverse phrase (“B is behind A”); premise order unchanged.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8928,"acc_change":-0.0483,"agreement":0.8756,"both_correct":0.855},{"perturbation":"form-inv50","label":"Restate 50%","family":"Share of premises restated","description":"50% of the premises restated with the inverse phrase (“B is behind A”); premise order unchanged.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8483,"acc_change":-0.0928,"agreement":0.8528,"both_correct":0.8217},{"perturbation":"form-inv75","label":"Restate 75%","family":"Share of premises restated","description":"75% of the premises restated with the inverse phrase (“B is behind A”); premise order unchanged.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8606,"acc_change":-0.0806,"agreement":0.8578,"both_correct":0.8306},{"perturbation":"form-inv100","label":"Restate 100%","family":"Share of premises restated","description":"Every premise restated with the inverse phrase (“B is behind A”); premise order unchanged.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8239,"acc_change":-0.1172,"agreement":0.8133,"both_correct":0.7894},{"perturbation":"form-converse","label":"Restate 100% + reorder","family":"Share of premises restated","description":"Every premise restated with the inverse phrase, and the premise order and names list reshuffled.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8294,"acc_change":-0.1117,"agreement":0.8111,"both_correct":0.7911},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.9139,"acc_change":-0.0272,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8039,"acc_change":-0.1372,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8922,"acc_change":-0.0489,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8167,"acc_change":-0.1244,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.8944,"acc_change":-0.0467,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"GPT-5","acc_reference":0.9411,"acc_perturbed":0.95,"acc_change":0.0089,"agreement":0.9356,"both_correct":0.9133},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.5328,"acc_change":-0.0233,"agreement":0.5717,"both_correct":0.3311},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6128,"acc_change":0.0567,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6422,"acc_change":0.0861,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6378,"acc_change":0.0817,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.5211,"acc_change":-0.035,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6322,"acc_change":0.0761,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6261,"acc_change":0.07,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ew","label":"east of / west of","family":"Relation wording","description":"Same problems with the facts stated as “east of / west of” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6094,"acc_change":0.0533,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_front","label":"closer to the front / back than","family":"Relation wording","description":"Same problems with the facts stated as “closer to the front / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6128,"acc_change":0.0567,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_forward","label":"further forward / back than","family":"Relation wording","description":"Same problems with the facts stated as “further forward / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6467,"acc_change":0.0906,"agreement":null,"both_correct":null},{"perturbation":"wording-above","label":"above / below","family":"Relation wording","description":"Same problems with the facts stated as “above / below” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6833,"acc_change":0.1272,"agreement":null,"both_correct":null},{"perturbation":"wording-higher","label":"higher up / lower down than","family":"Relation wording","description":"Same problems with the facts stated as “higher up / lower down than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7533,"acc_change":0.1972,"agreement":null,"both_correct":null},{"perturbation":"wording-arrival","label":"arrived earlier / later than","family":"Relation wording","description":"Same problems with the facts stated as “arrived earlier / later than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6361,"acc_change":0.08,"agreement":null,"both_correct":null},{"perturbation":"wording-born","label":"born before / after","family":"Relation wording","description":"Same problems with the facts stated as “born before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6706,"acc_change":0.1144,"agreement":null,"both_correct":null},{"perturbation":"wording-finish","label":"finished earlier / later than","family":"Relation wording","description":"Same problems with the facts stated as “finished earlier / later than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6144,"acc_change":0.0583,"agreement":null,"both_correct":null},{"perturbation":"wording-scheduled","label":"scheduled before / after","family":"Relation wording","description":"Same problems with the facts stated as “scheduled before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.6917,"acc_change":0.1356,"agreement":null,"both_correct":null},{"perturbation":"wording-older","label":"older / younger than","family":"Relation wording","description":"Same problems with the facts stated as “older / younger than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7528,"acc_change":0.1967,"agreement":null,"both_correct":null},{"perturbation":"wording-taller","label":"taller / shorter than","family":"Relation wording","description":"Same problems with the facts stated as “taller / shorter than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7672,"acc_change":0.2111,"agreement":null,"both_correct":null},{"perturbation":"wording-shorter","label":"shorter than (instead of taller)","family":"Relation wording","description":"Same problems with every fact stated using “shorter than”, the mirror image of the taller/shorter wording.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7067,"acc_change":0.1506,"agreement":null,"both_correct":null},{"perturbation":"wording-heavier","label":"heavier / lighter than","family":"Relation wording","description":"Same problems with the facts stated as “heavier / lighter than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7789,"acc_change":0.2228,"agreement":null,"both_correct":null},{"perturbation":"wording-faster","label":"faster / slower than","family":"Relation wording","description":"Same problems with the facts stated as “faster / slower than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7367,"acc_change":0.1806,"agreement":null,"both_correct":null},{"perturbation":"wording-stronger","label":"stronger / weaker than","family":"Relation wording","description":"Same problems with the facts stated as “stronger / weaker than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7878,"acc_change":0.2317,"agreement":null,"both_correct":null},{"perturbation":"wording-richer","label":"richer / poorer than","family":"Relation wording","description":"Same problems with the facts stated as “richer / poorer than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7706,"acc_change":0.2144,"agreement":null,"both_correct":null},{"perturbation":"wording-seniority","label":"more senior / more junior than","family":"Relation wording","description":"Same problems with the facts stated as “more senior / more junior than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7533,"acc_change":0.1972,"agreement":null,"both_correct":null},{"perturbation":"wording-ranked","label":"ranked higher / lower than","family":"Relation wording","description":"Same problems with the facts stated as “ranked higher / lower than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7672,"acc_change":0.2111,"agreement":null,"both_correct":null},{"perturbation":"wording-experienced","label":"more / less experienced than","family":"Relation wording","description":"Same problems with the facts stated as “more / less experienced than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7833,"acc_change":0.2272,"agreement":null,"both_correct":null},{"perturbation":"wording-popular","label":"more / less popular than","family":"Relation wording","description":"Same problems with the facts stated as “more / less popular than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7778,"acc_change":0.2217,"agreement":null,"both_correct":null},{"perturbation":"wording-paid","label":"paid more / less than","family":"Relation wording","description":"Same problems with the facts stated as “paid more / less than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7867,"acc_change":0.2306,"agreement":null,"both_correct":null},{"perturbation":"wording-proximity","label":"closer to / farther from the exit than","family":"Relation wording","description":"Same problems with the facts stated as “closer to / farther from the exit than” instead of “in front of / behind”.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.5733,"acc_change":0.0172,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_cmp","label":"more / less zorbic (made-up trait)","family":"Relation wording","description":"Same problems with the facts stated as “more / less zorbic than”, a made-up trait; the rule linking the two forms is stated in the prompt.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.7322,"acc_change":0.1761,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.5417,"acc_change":-0.0144,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Qwen3.5-9B","acc_reference":0.5561,"acc_perturbed":0.5589,"acc_change":0.0028,"agreement":0.6467,"both_correct":0.3811},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.7294,"acc_change":-0.0844,"agreement":0.7211,"both_correct":0.6322},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.7972,"acc_change":-0.0167,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8617,"acc_change":0.0478,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8289,"acc_change":0.015,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.6811,"acc_change":-0.1328,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.845,"acc_change":0.0311,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.7933,"acc_change":-0.0206,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ew","label":"east of / west of","family":"Relation wording","description":"Same problems with the facts stated as “east of / west of” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.765,"acc_change":-0.0489,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_front","label":"closer to the front / back than","family":"Relation wording","description":"Same problems with the facts stated as “closer to the front / back than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.7817,"acc_change":-0.0322,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_forward","label":"further forward / back than","family":"Relation wording","description":"Same problems with the facts stated as “further forward / back than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.7939,"acc_change":-0.02,"agreement":null,"both_correct":null},{"perturbation":"wording-above","label":"above / below","family":"Relation wording","description":"Same problems with the facts stated as “above / below” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8244,"acc_change":0.0106,"agreement":null,"both_correct":null},{"perturbation":"wording-higher","label":"higher up / lower down than","family":"Relation wording","description":"Same problems with the facts stated as “higher up / lower down than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8383,"acc_change":0.0244,"agreement":null,"both_correct":null},{"perturbation":"wording-arrival","label":"arrived earlier / later than","family":"Relation wording","description":"Same problems with the facts stated as “arrived earlier / later than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8544,"acc_change":0.0406,"agreement":null,"both_correct":null},{"perturbation":"wording-born","label":"born before / after","family":"Relation wording","description":"Same problems with the facts stated as “born before / after” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8483,"acc_change":0.0344,"agreement":null,"both_correct":null},{"perturbation":"wording-finish","label":"finished earlier / later than","family":"Relation wording","description":"Same problems with the facts stated as “finished earlier / later than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8444,"acc_change":0.0306,"agreement":null,"both_correct":null},{"perturbation":"wording-scheduled","label":"scheduled before / after","family":"Relation wording","description":"Same problems with the facts stated as “scheduled before / after” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8711,"acc_change":0.0572,"agreement":null,"both_correct":null},{"perturbation":"wording-older","label":"older / younger than","family":"Relation wording","description":"Same problems with the facts stated as “older / younger than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8594,"acc_change":0.0456,"agreement":null,"both_correct":null},{"perturbation":"wording-taller","label":"taller / shorter than","family":"Relation wording","description":"Same problems with the facts stated as “taller / shorter than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.87,"acc_change":0.0561,"agreement":null,"both_correct":null},{"perturbation":"wording-shorter","label":"shorter than (instead of taller)","family":"Relation wording","description":"Same problems with every fact stated using “shorter than”, the mirror image of the taller/shorter wording.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.87,"acc_change":0.0561,"agreement":null,"both_correct":null},{"perturbation":"wording-heavier","label":"heavier / lighter than","family":"Relation wording","description":"Same problems with the facts stated as “heavier / lighter than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8817,"acc_change":0.0678,"agreement":null,"both_correct":null},{"perturbation":"wording-faster","label":"faster / slower than","family":"Relation wording","description":"Same problems with the facts stated as “faster / slower than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8783,"acc_change":0.0644,"agreement":null,"both_correct":null},{"perturbation":"wording-stronger","label":"stronger / weaker than","family":"Relation wording","description":"Same problems with the facts stated as “stronger / weaker than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8878,"acc_change":0.0739,"agreement":null,"both_correct":null},{"perturbation":"wording-richer","label":"richer / poorer than","family":"Relation wording","description":"Same problems with the facts stated as “richer / poorer than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8578,"acc_change":0.0439,"agreement":null,"both_correct":null},{"perturbation":"wording-seniority","label":"more senior / more junior than","family":"Relation wording","description":"Same problems with the facts stated as “more senior / more junior than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.84,"acc_change":0.0261,"agreement":null,"both_correct":null},{"perturbation":"wording-ranked","label":"ranked higher / lower than","family":"Relation wording","description":"Same problems with the facts stated as “ranked higher / lower than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8828,"acc_change":0.0689,"agreement":null,"both_correct":null},{"perturbation":"wording-experienced","label":"more / less experienced than","family":"Relation wording","description":"Same problems with the facts stated as “more / less experienced than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.85,"acc_change":0.0361,"agreement":null,"both_correct":null},{"perturbation":"wording-popular","label":"more / less popular than","family":"Relation wording","description":"Same problems with the facts stated as “more / less popular than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8733,"acc_change":0.0594,"agreement":null,"both_correct":null},{"perturbation":"wording-paid","label":"paid more / less than","family":"Relation wording","description":"Same problems with the facts stated as “paid more / less than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8861,"acc_change":0.0722,"agreement":null,"both_correct":null},{"perturbation":"wording-proximity","label":"closer to / farther from the exit than","family":"Relation wording","description":"Same problems with the facts stated as “closer to / farther from the exit than” instead of “in front of / behind”.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.7206,"acc_change":-0.0933,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_cmp","label":"more / less zorbic (made-up trait)","family":"Relation wording","description":"Same problems with the facts stated as “more / less zorbic than”, a made-up trait; the rule linking the two forms is stated in the prompt.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.8706,"acc_change":0.0567,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.6967,"acc_change":-0.1172,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Qwen3.5-27B","acc_reference":0.8139,"acc_perturbed":0.7972,"acc_change":-0.0167,"agreement":0.78,"both_correct":0.6956},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + SFT (cross-relation)","acc_reference":0.5639,"acc_perturbed":0.55,"acc_change":-0.0139,"agreement":0.5633,"both_correct":0.3394},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"GPT-5.4 (effort none)","acc_reference":0.7711,"acc_perturbed":0.7489,"acc_change":-0.0222,"agreement":0.7811,"both_correct":0.6506},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + SFT (pairs + JS)","acc_reference":0.5617,"acc_perturbed":0.545,"acc_change":-0.0167,"agreement":0.5789,"both_correct":0.3444},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Qwen3.5-9B + SFT (pairs + JS)","acc_reference":0.5617,"acc_perturbed":0.5511,"acc_change":-0.0106,"agreement":0.6367,"both_correct":0.3772},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + SFT (pairs)","acc_reference":0.545,"acc_perturbed":0.5661,"acc_change":0.0211,"agreement":0.5772,"both_correct":0.345},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Qwen3.5-9B + SFT (pairs)","acc_reference":0.545,"acc_perturbed":0.5433,"acc_change":-0.0017,"agreement":0.6494,"both_correct":0.3689},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.7506,"acc_change":-0.0617,"agreement":0.7328,"both_correct":0.6478},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.9344,"acc_change":0.1222,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.9156,"acc_change":0.1033,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.9317,"acc_change":0.1194,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.6989,"acc_change":-0.1133,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.9028,"acc_change":0.0906,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.9194,"acc_change":0.1072,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ew","label":"east of / west of","family":"Relation wording","description":"Same problems with the facts stated as “east of / west of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.9206,"acc_change":0.1083,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_front","label":"closer to the front / back than","family":"Relation wording","description":"Same problems with the facts stated as “closer to the front / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.8672,"acc_change":0.055,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_forward","label":"further forward / back than","family":"Relation wording","description":"Same problems with the facts stated as “further forward / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.9517,"acc_change":0.1394,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_cmp","label":"more / less zorbic (made-up trait)","family":"Relation wording","description":"Same problems with the facts stated as “more / less zorbic than”, a made-up trait; the rule linking the two forms is stated in the prompt.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.965,"acc_change":0.1528,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.6933,"acc_change":-0.1189,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Qwen3.5-9B + GRPO (seed 42)","acc_reference":0.8122,"acc_perturbed":0.8183,"acc_change":0.0061,"agreement":0.9367,"both_correct":0.7839},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.7672,"acc_change":-0.0139,"agreement":0.6728,"both_correct":0.6106},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.95,"acc_change":0.1689,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.8739,"acc_change":0.0928,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.9283,"acc_change":0.1472,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.6711,"acc_change":-0.11,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.8967,"acc_change":0.1156,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.9544,"acc_change":0.1733,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ew","label":"east of / west of","family":"Relation wording","description":"Same problems with the facts stated as “east of / west of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.9372,"acc_change":0.1561,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_front","label":"closer to the front / back than","family":"Relation wording","description":"Same problems with the facts stated as “closer to the front / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.8706,"acc_change":0.0894,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_forward","label":"further forward / back than","family":"Relation wording","description":"Same problems with the facts stated as “further forward / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.9717,"acc_change":0.1906,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_cmp","label":"more / less zorbic (made-up trait)","family":"Relation wording","description":"Same problems with the facts stated as “more / less zorbic than”, a made-up trait; the rule linking the two forms is stated in the prompt.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.9806,"acc_change":0.1994,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.7322,"acc_change":-0.0489,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Qwen3.5-9B + GRPO (seed 43)","acc_reference":0.7811,"acc_perturbed":0.7956,"acc_change":0.0144,"agreement":0.8922,"both_correct":0.7344},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.7433,"acc_change":-0.0511,"agreement":0.6983,"both_correct":0.6183},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.9422,"acc_change":0.1478,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.915,"acc_change":0.1206,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.9356,"acc_change":0.1411,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.67,"acc_change":-0.1244,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.9161,"acc_change":0.1217,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.9322,"acc_change":0.1378,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ew","label":"east of / west of","family":"Relation wording","description":"Same problems with the facts stated as “east of / west of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.9294,"acc_change":0.135,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_front","label":"closer to the front / back than","family":"Relation wording","description":"Same problems with the facts stated as “closer to the front / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.8556,"acc_change":0.0611,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_forward","label":"further forward / back than","family":"Relation wording","description":"Same problems with the facts stated as “further forward / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.9533,"acc_change":0.1589,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_cmp","label":"more / less zorbic (made-up trait)","family":"Relation wording","description":"Same problems with the facts stated as “more / less zorbic than”, a made-up trait; the rule linking the two forms is stated in the prompt.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.9783,"acc_change":0.1839,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 42)","acc_reference":0.7944,"acc_perturbed":0.7011,"acc_change":-0.0933,"agreement":null,"both_correct":null},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.7622,"acc_change":-0.0722,"agreement":0.7511,"both_correct":0.6739},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.9461,"acc_change":0.1117,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.95,"acc_change":0.1156,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.9506,"acc_change":0.1161,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.7189,"acc_change":-0.1156,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.9283,"acc_change":0.0939,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.9444,"acc_change":0.11,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ew","label":"east of / west of","family":"Relation wording","description":"Same problems with the facts stated as “east of / west of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.9367,"acc_change":0.1022,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_front","label":"closer to the front / back than","family":"Relation wording","description":"Same problems with the facts stated as “closer to the front / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.8322,"acc_change":-0.0022,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_forward","label":"further forward / back than","family":"Relation wording","description":"Same problems with the facts stated as “further forward / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.9589,"acc_change":0.1244,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_cmp","label":"more / less zorbic (made-up trait)","family":"Relation wording","description":"Same problems with the facts stated as “more / less zorbic than”, a made-up trait; the rule linking the two forms is stated in the prompt.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.9744,"acc_change":0.14,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 42)","acc_reference":0.8344,"acc_perturbed":0.6839,"acc_change":-0.1506,"agreement":null,"both_correct":null},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.7456,"acc_change":-0.0394,"agreement":0.6561,"both_correct":0.5933},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.9567,"acc_change":0.1717,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.8933,"acc_change":0.1083,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.9339,"acc_change":0.1489,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.6556,"acc_change":-0.1294,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.8978,"acc_change":0.1128,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.9506,"acc_change":0.1656,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ew","label":"east of / west of","family":"Relation wording","description":"Same problems with the facts stated as “east of / west of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.95,"acc_change":0.165,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_front","label":"closer to the front / back than","family":"Relation wording","description":"Same problems with the facts stated as “closer to the front / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.8483,"acc_change":0.0633,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_forward","label":"further forward / back than","family":"Relation wording","description":"Same problems with the facts stated as “further forward / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.9694,"acc_change":0.1844,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_cmp","label":"more / less zorbic (made-up trait)","family":"Relation wording","description":"Same problems with the facts stated as “more / less zorbic than”, a made-up trait; the rule linking the two forms is stated in the prompt.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.9817,"acc_change":0.1967,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.5-9B + GRPO-IG λ=0 (seed 43)","acc_reference":0.785,"acc_perturbed":0.7072,"acc_change":-0.0778,"agreement":null,"both_correct":null},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.7611,"acc_change":-0.0483,"agreement":0.6689,"both_correct":0.62},{"perturbation":"wording-pos_ahead","label":"ahead of / behind","family":"Relation wording","description":"Same problems with the facts stated as “ahead of / behind” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.9494,"acc_change":0.14,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.8883,"acc_change":0.0789,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_rl","label":"right of (instead of left of)","family":"Relation wording","description":"Same problems with every fact stated using “right of”, the mirror image of the left/right wording.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.9311,"acc_change":0.1217,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_mirror","label":"behind (instead of in front of)","family":"Relation wording","description":"Same problems with every fact stated using “behind” instead of “in front of”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.6583,"acc_change":-0.1511,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ba","label":"before / after","family":"Relation wording","description":"Same problems with the facts stated as “before / after” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.9089,"acc_change":0.0994,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.9522,"acc_change":0.1428,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ew","label":"east of / west of","family":"Relation wording","description":"Same problems with the facts stated as “east of / west of” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.9411,"acc_change":0.1317,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_front","label":"closer to the front / back than","family":"Relation wording","description":"Same problems with the facts stated as “closer to the front / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.8689,"acc_change":0.0594,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_forward","label":"further forward / back than","family":"Relation wording","description":"Same problems with the facts stated as “further forward / back than” instead of “in front of / behind”.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.9733,"acc_change":0.1639,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_cmp","label":"more / less zorbic (made-up trait)","family":"Relation wording","description":"Same problems with the facts stated as “more / less zorbic than”, a made-up trait; the rule linking the two forms is stated in the prompt.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.98,"acc_change":0.1706,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.5-9B + GRPO-IG λ=0.5 (seed 43)","acc_reference":0.8094,"acc_perturbed":0.7172,"acc_change":-0.0922,"agreement":null,"both_correct":null},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.7506,"acc_change":-0.095,"agreement":0.6994,"both_correct":0.6967},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.7772,"acc_change":-0.0683,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.8711,"acc_change":0.0256,"agreement":null,"both_correct":null},{"perturbation":"wording-above","label":"above / below","family":"Relation wording","description":"Same problems with the facts stated as “above / below” instead of “in front of / behind”.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.9189,"acc_change":0.0733,"agreement":null,"both_correct":null},{"perturbation":"wording-born","label":"born before / after","family":"Relation wording","description":"Same problems with the facts stated as “born before / after” instead of “in front of / behind”.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.8194,"acc_change":-0.0261,"agreement":null,"both_correct":null},{"perturbation":"wording-taller","label":"taller / shorter than","family":"Relation wording","description":"Same problems with the facts stated as “taller / shorter than” instead of “in front of / behind”.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.9183,"acc_change":0.0728,"agreement":null,"both_correct":null},{"perturbation":"wording-shorter","label":"shorter than (instead of taller)","family":"Relation wording","description":"Same problems with every fact stated using “shorter than”, the mirror image of the taller/shorter wording.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.9061,"acc_change":0.0606,"agreement":null,"both_correct":null},{"perturbation":"wording-ranked","label":"ranked higher / lower than","family":"Relation wording","description":"Same problems with the facts stated as “ranked higher / lower than” instead of “in front of / behind”.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.9206,"acc_change":0.075,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.7683,"acc_change":-0.0772,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Gemma-4-31B","acc_reference":0.8456,"acc_perturbed":0.8433,"acc_change":-0.0022,"agreement":0.8461,"both_correct":0.8194},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.6094,"acc_change":-0.0344,"agreement":0.5978,"both_correct":0.4256},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.6883,"acc_change":0.0444,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.6294,"acc_change":-0.0144,"agreement":null,"both_correct":null},{"perturbation":"wording-above","label":"above / below","family":"Relation wording","description":"Same problems with the facts stated as “above / below” instead of “in front of / behind”.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.7033,"acc_change":0.0594,"agreement":null,"both_correct":null},{"perturbation":"wording-born","label":"born before / after","family":"Relation wording","description":"Same problems with the facts stated as “born before / after” instead of “in front of / behind”.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.6583,"acc_change":0.0144,"agreement":null,"both_correct":null},{"perturbation":"wording-taller","label":"taller / shorter than","family":"Relation wording","description":"Same problems with the facts stated as “taller / shorter than” instead of “in front of / behind”.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.7217,"acc_change":0.0778,"agreement":null,"both_correct":null},{"perturbation":"wording-shorter","label":"shorter than (instead of taller)","family":"Relation wording","description":"Same problems with every fact stated using “shorter than”, the mirror image of the taller/shorter wording.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.7289,"acc_change":0.085,"agreement":null,"both_correct":null},{"perturbation":"wording-ranked","label":"ranked higher / lower than","family":"Relation wording","description":"Same problems with the facts stated as “ranked higher / lower than” instead of “in front of / behind”.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.6944,"acc_change":0.0506,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.5628,"acc_change":-0.0811,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Qwen3.6-35B-A3B","acc_reference":0.6439,"acc_perturbed":0.65,"acc_change":0.0061,"agreement":0.6717,"both_correct":0.4828},{"perturbation":"rewording","label":"Premise rewording","family":"Rewording","description":"Between about 20% and all of the premises (60% on average) restated with the inverse phrase (“B is behind A”); premise order and names list reshuffled. This is the reworded version used throughout the page.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.745,"acc_change":-0.0817,"agreement":0.7994,"both_correct":0.6856},{"perturbation":"wording-pos_lr","label":"left of / right of","family":"Relation wording","description":"Same problems with the facts stated as “left of / right of” instead of “in front of / behind”.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.8639,"acc_change":0.0372,"agreement":null,"both_correct":null},{"perturbation":"wording-pos_ns","label":"north of / south of","family":"Relation wording","description":"Same problems with the facts stated as “north of / south of” instead of “in front of / behind”.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.9,"acc_change":0.0733,"agreement":null,"both_correct":null},{"perturbation":"wording-above","label":"above / below","family":"Relation wording","description":"Same problems with the facts stated as “above / below” instead of “in front of / behind”.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.8983,"acc_change":0.0717,"agreement":null,"both_correct":null},{"perturbation":"wording-born","label":"born before / after","family":"Relation wording","description":"Same problems with the facts stated as “born before / after” instead of “in front of / behind”.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.8239,"acc_change":-0.0028,"agreement":null,"both_correct":null},{"perturbation":"wording-taller","label":"taller / shorter than","family":"Relation wording","description":"Same problems with the facts stated as “taller / shorter than” instead of “in front of / behind”.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.9294,"acc_change":0.1028,"agreement":null,"both_correct":null},{"perturbation":"wording-shorter","label":"shorter than (instead of taller)","family":"Relation wording","description":"Same problems with every fact stated using “shorter than”, the mirror image of the taller/shorter wording.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.9006,"acc_change":0.0739,"agreement":null,"both_correct":null},{"perturbation":"wording-ranked","label":"ranked higher / lower than","family":"Relation wording","description":"Same problems with the facts stated as “ranked higher / lower than” instead of “in front of / behind”.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.9206,"acc_change":0.0939,"agreement":null,"both_correct":null},{"perturbation":"wording-nonce_dir","label":"vorpward / morpward of (made-up direction)","family":"Relation wording","description":"Same problems with the facts stated as “vorpward / morpward of”, two made-up direction words; the rule linking them is stated in the prompt.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.7222,"acc_change":-0.1044,"agreement":null,"both_correct":null},{"perturbation":"noise","label":"Identical prompt re-sent","family":"Repeat control","description":"The original prompt sent again with the same settings: this model's run-to-run agreement, a reference level rather than a lower bound.","model":"Qwen2.5-32B-Instruct","acc_reference":0.8267,"acc_perturbed":0.8267,"acc_change":-0.0,"agreement":0.9611,"both_correct":0.8072}]