Document failed explicit rejection experiment
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
|
||||
from src.meeting_lab.controlled_semantic_derivation.experiment_rejection import (
|
||||
DerivationValidationError,
|
||||
build_prompt,
|
||||
derive_rejection,
|
||||
evaluate_case,
|
||||
load_gold_cases,
|
||||
validate_recognition,
|
||||
)
|
||||
|
||||
|
||||
GOLD_PATH = Path("tests/gold/explicit_rejection_v0/cases.json")
|
||||
|
||||
|
||||
POSITIVE_TEXT = {
|
||||
"RJ-01": "reale Anlage für den Versuch nutzen",
|
||||
"RJ-02": "externe Lösung weiterverfolgen",
|
||||
"RJ-03": "Zusammenarbeit mit Dr. Schlummer fortsetzen",
|
||||
"RJ-11": "reale Anlage für den Druckversuch nutzen",
|
||||
"RJ-12": "Versuch in der realen Anlage durchführen",
|
||||
}
|
||||
|
||||
|
||||
def recognition_for(case):
|
||||
expected = case["expected_recognition"]
|
||||
positive = expected["rejection_form"] == "explicit_action_rejection"
|
||||
return {
|
||||
"rejection_observation_id": expected["rejection_observation_id"],
|
||||
"target_observation_id": expected["target_observation_id"] if positive else None,
|
||||
"rejection_form": expected["rejection_form"],
|
||||
"normalized_rejected_action_text": POSITIVE_TEXT.get(case["case_id"]) if positive else None,
|
||||
}
|
||||
|
||||
|
||||
class ExplicitRejectionGoldExperimentTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.cases = load_gold_cases(GOLD_PATH)
|
||||
cls.by_id = {case["case_id"]: case for case in cls.cases}
|
||||
|
||||
def test_fixture_contains_exactly_rj_01_through_rj_12(self):
|
||||
self.assertEqual(list(self.by_id), [f"RJ-{number:02d}" for number in range(1, 13)])
|
||||
|
||||
def test_cases_use_only_minimal_v3_style_observations(self):
|
||||
keys = {"observation_id", "evidence_id", "content", "speaker", "named_person", "addressee"}
|
||||
for case in self.cases:
|
||||
with self.subTest(case=case["case_id"]):
|
||||
self.assertIn(len(case["observations"]), (1, 2))
|
||||
self.assertTrue(all(set(item) == keys for item in case["observations"]))
|
||||
|
||||
def test_rj_01_derives_target_and_both_provenance_paths(self):
|
||||
case = self.by_id["RJ-01"]
|
||||
gates, result = derive_rejection(case["observations"], recognition_for(case))
|
||||
self.assertTrue(all(gates.values()))
|
||||
self.assertEqual(result["status"], "explicitly_rejected")
|
||||
self.assertEqual(result["support"]["target"], {"observation_id": "obs_1", "evidence_id": "e1"})
|
||||
self.assertEqual(result["support"]["rejection"], {"observation_id": "obs_2", "evidence_id": "e2"})
|
||||
|
||||
def test_rj_02_requires_paired_target_and_derives_abandonment(self):
|
||||
case = self.by_id["RJ-02"]
|
||||
_, result = derive_rejection(case["observations"], recognition_for(case))
|
||||
self.assertIn("externe Lösung", result["content"])
|
||||
with self.assertRaisesRegex(DerivationValidationError, "unknown target"):
|
||||
derive_rejection(case["observations"][1:], recognition_for(case))
|
||||
|
||||
def test_rj_03_supports_same_observation_target_and_rejection(self):
|
||||
case = self.by_id["RJ-03"]
|
||||
_, result = derive_rejection(case["observations"], recognition_for(case))
|
||||
self.assertEqual(result["support"]["target"], result["support"]["rejection"])
|
||||
self.assertIn("Dr. Schlummer", result["content"])
|
||||
|
||||
def test_all_required_negative_cases_remain_non_rejections(self):
|
||||
for case_id in ("RJ-04", "RJ-05", "RJ-06", "RJ-07", "RJ-08", "RJ-09", "RJ-10"):
|
||||
case = self.by_id[case_id]
|
||||
gates, result = derive_rejection(case["observations"], recognition_for(case))
|
||||
with self.subTest(case=case_id):
|
||||
self.assertFalse(gates["explicit_action_rejection"])
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_rj_11_derives_and_preserves_location_and_purpose_scope(self):
|
||||
case = self.by_id["RJ-11"]
|
||||
_, result = derive_rejection(case["observations"], recognition_for(case))
|
||||
self.assertIsNotNone(result)
|
||||
self.assertIn("reale Anlage", result["content"])
|
||||
self.assertIn("Druckversuch", result["content"])
|
||||
|
||||
def test_rj_12_rejects_only_real_plant_action_and_not_alternative(self):
|
||||
case = self.by_id["RJ-12"]
|
||||
_, result = derive_rejection(case["observations"], recognition_for(case))
|
||||
self.assertIn("realen Anlage", result["content"])
|
||||
self.assertNotIn("Technikum", result["content"])
|
||||
|
||||
def test_separate_target_cannot_follow_rejection(self):
|
||||
case = deepcopy(self.by_id["RJ-01"])
|
||||
case["observations"].reverse()
|
||||
gates, result = derive_rejection(case["observations"], recognition_for(case))
|
||||
self.assertFalse(gates["target_same_or_before_rejection"])
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_unknown_target_observation_id_is_rejected(self):
|
||||
case = self.by_id["RJ-01"]
|
||||
recognition = recognition_for(case)
|
||||
recognition["target_observation_id"] = "obs_99"
|
||||
with self.assertRaisesRegex(DerivationValidationError, "unknown target"):
|
||||
validate_recognition(recognition, case["observations"])
|
||||
|
||||
def test_unknown_rejection_observation_id_is_rejected(self):
|
||||
case = self.by_id["RJ-01"]
|
||||
recognition = recognition_for(case)
|
||||
recognition["rejection_observation_id"] = "obs_99"
|
||||
with self.assertRaisesRegex(DerivationValidationError, "unknown rejection"):
|
||||
validate_recognition(recognition, case["observations"])
|
||||
|
||||
def test_duplicate_observation_ids_are_rejected(self):
|
||||
fixture = json.loads(GOLD_PATH.read_text())
|
||||
fixture["cases"][0]["observations"][1]["observation_id"] = "obs_1"
|
||||
self._assert_bad_fixture(fixture, "observation IDs must be unique")
|
||||
|
||||
def test_inconsistent_evidence_provenance_is_rejected(self):
|
||||
fixture = json.loads(GOLD_PATH.read_text())
|
||||
fixture["cases"][0]["observations"][1]["evidence_id"] = "e1"
|
||||
self._assert_bad_fixture(fixture, "evidence provenance must be unique")
|
||||
|
||||
def test_none_rejects_populated_target_or_action(self):
|
||||
case = self.by_id["RJ-04"]
|
||||
for field, value, message in (
|
||||
("target_observation_id", "obs_1", "null target"),
|
||||
("normalized_rejected_action_text", "Anlage nutzen", "null normalized"),
|
||||
):
|
||||
recognition = recognition_for(case)
|
||||
recognition[field] = value
|
||||
with self.subTest(field=field), self.assertRaisesRegex(DerivationValidationError, message):
|
||||
validate_recognition(recognition, case["observations"])
|
||||
|
||||
def test_explicit_rejection_requires_normalized_target_text(self):
|
||||
case = self.by_id["RJ-01"]
|
||||
for value in (None, ""):
|
||||
recognition = recognition_for(case)
|
||||
recognition["normalized_rejected_action_text"] = value
|
||||
with self.subTest(value=value), self.assertRaises(DerivationValidationError):
|
||||
validate_recognition(recognition, case["observations"])
|
||||
|
||||
def test_unknown_schema_fields_are_rejected(self):
|
||||
case = self.by_id["RJ-01"]
|
||||
recognition = recognition_for(case)
|
||||
recognition["explanation"] = "extra"
|
||||
with self.assertRaisesRegex(DerivationValidationError, "unknown keys"):
|
||||
validate_recognition(recognition, case["observations"])
|
||||
|
||||
def test_forbidden_normative_fields_are_rejected_recursively(self):
|
||||
case = self.by_id["RJ-01"]
|
||||
fields = (
|
||||
"decision", "decision_status", "outcome", "topic_status", "closed",
|
||||
"agreement", "responsible_person", "responsibility", "responsibility_scope",
|
||||
"owner", "ownership", "assignee", "requested_actor", "status",
|
||||
"explicitly_rejected", "action_item", "protocol_category", "confidence",
|
||||
"relation", "relations", "graph", "unresolved_issue",
|
||||
)
|
||||
for field in fields:
|
||||
recognition = recognition_for(case)
|
||||
recognition["wrapper"] = {field: "forbidden"}
|
||||
with self.subTest(field=field), self.assertRaisesRegex(DerivationValidationError, "forbidden semantic keys"):
|
||||
validate_recognition(recognition, case["observations"])
|
||||
|
||||
def test_speaker_identity_creates_no_ownership_or_responsibility(self):
|
||||
case = deepcopy(self.by_id["RJ-03"])
|
||||
for speaker in ("Martin", "Clara", "Antonius"):
|
||||
case["observations"][0]["speaker"] = speaker
|
||||
_, result = derive_rejection(case["observations"], recognition_for(case))
|
||||
with self.subTest(speaker=speaker):
|
||||
self.assertNotIn("responsible_person", result)
|
||||
self.assertNotIn("owner", result)
|
||||
|
||||
def test_all_expected_recognitions_evaluate_as_pass(self):
|
||||
for case in self.cases:
|
||||
evaluation = evaluate_case(case, recognition_for(case))
|
||||
with self.subTest(case=case["case_id"]):
|
||||
self.assertEqual(evaluation["classification"], "PASS")
|
||||
|
||||
def test_rj_11_qualifier_loss_and_rj_12_alternative_absorption_fail(self):
|
||||
rj11 = self.by_id["RJ-11"]
|
||||
recognition = recognition_for(rj11)
|
||||
recognition["normalized_rejected_action_text"] = "reale Anlage nutzen"
|
||||
self.assertEqual(evaluate_case(rj11, recognition)["classification"], "FAIL")
|
||||
rj12 = self.by_id["RJ-12"]
|
||||
recognition = recognition_for(rj12)
|
||||
recognition["normalized_rejected_action_text"] += "; stattdessen im Technikum testen"
|
||||
self.assertEqual(evaluate_case(rj12, recognition)["classification"], "FAIL")
|
||||
|
||||
def test_prompt_is_fixed_narrow_and_does_not_expose_gold_expectation(self):
|
||||
prompt = build_prompt(self.by_id["RJ-01"])
|
||||
self.assertIn("candidate rejection observation is obs_2", prompt)
|
||||
self.assertNotIn("expected_result", prompt)
|
||||
self.assertNotIn("Who is responsible", prompt)
|
||||
|
||||
def _assert_bad_fixture(self, fixture, message):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
path = Path(temporary) / "cases.json"
|
||||
path.write_text(json.dumps(fixture), encoding="utf-8")
|
||||
with self.assertRaisesRegex(DerivationValidationError, message):
|
||||
load_gold_cases(path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user