Document controlled rejection V1 baseline
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"schema_version": "experimental-controlled-rejection-v1",
|
||||
"cases": [
|
||||
{"case_id":"CR-01","description":"self-contained non-pursuit","negative_act_source":"NA-01","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Mit Dr. Schlummer arbeiten wir nicht weiter.","speaker":"Martin","named_person":"Dr. Schlummer","addressee":null}],"expected":{"negative_act_form":"explicit_non_pursuit","candidate_observation_id":"obs_1","target_observation_id":"obs_1","action_concepts":[["Schlummer"],["Zusammenarbeit","arbeiten"],["fortsetzen","weiter"]],"material_concepts":[],"forbidden_concepts":[],"explicitly_rejected":true}},
|
||||
{"case_id":"CR-02","description":"paired non-pursuit","negative_act_source":"live","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Eine Möglichkeit wäre, die externe Lösung weiterzuverfolgen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Das verfolgen wir nicht weiter.","speaker":"Martin","named_person":null,"addressee":null}],"expected":{"negative_act_form":"explicit_non_pursuit","candidate_observation_id":"obs_2","target_observation_id":"obs_1","action_concepts":[["externe Lösung"],["weiterverfolgen","weiter verfolgen"]],"material_concepts":[],"forbidden_concepts":[],"explicitly_rejected":true}},
|
||||
{"case_id":"CR-03","description":"personal preference","negative_act_source":"NA-03","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Wir könnten die reale Anlage für den Versuch nutzen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Ich würde das nicht machen.","speaker":"Martin","named_person":null,"addressee":null}],"expected":{"negative_act_form":"personal_preference","candidate_observation_id":"obs_2","target_observation_id":"obs_1","action_concepts":[["reale Anlage"],["Versuch"],["nutzen"]],"material_concepts":[],"forbidden_concepts":[],"explicitly_rejected":false}},
|
||||
{"case_id":"CR-04","description":"recommendation","negative_act_source":"NA-04","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Wir könnten die reale Anlage verwenden.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Ich würde eher davon abraten.","speaker":"Martin","named_person":null,"addressee":null}],"expected":{"negative_act_form":"recommendation","candidate_observation_id":"obs_2","target_observation_id":"obs_1","action_concepts":[["reale Anlage"],["verwenden","nutzen"]],"material_concepts":[],"forbidden_concepts":[],"explicitly_rejected":false}},
|
||||
{"case_id":"CR-05","description":"temporary non-action","negative_act_source":"NA-05","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Wir könnten die Waschstufe einbauen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Das machen wir erstmal noch nicht.","speaker":"Martin","named_person":null,"addressee":null}],"expected":{"negative_act_form":"temporary_non_action","candidate_observation_id":"obs_2","target_observation_id":"obs_1","action_concepts":[["Waschstufe"],["einbauen"]],"material_concepts":[],"forbidden_concepts":[],"explicitly_rejected":false}},
|
||||
{"case_id":"CR-06","description":"concern","negative_act_source":"NA-06","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Wir könnten das neue Material einsetzen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Das wäre kritisch.","speaker":"Martin","named_person":null,"addressee":null}],"expected":{"negative_act_form":"none","candidate_observation_id":"obs_2","target_observation_id":"obs_1","action_concepts":[["neue Material","neues Material"],["einsetzen"]],"material_concepts":[],"forbidden_concepts":[],"explicitly_rejected":false}},
|
||||
{"case_id":"CR-07","description":"scoped explicit rejection","negative_act_source":"live","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Für den Druckversuch steht die reale Anlage zur Diskussion.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Die reale Anlage nutzen wir dafür nicht.","speaker":"Martin","named_person":null,"addressee":null}],"expected":{"negative_act_form":"explicit_non_pursuit","candidate_observation_id":"obs_2","target_observation_id":"obs_1","action_concepts":[["Anlage"],["nutzen"]],"material_concepts":[["real"],["Druckversuch"]],"forbidden_concepts":[],"explicitly_rejected":true}},
|
||||
{"case_id":"CR-08","description":"rejection plus alternative","negative_act_source":"live","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Wir könnten den Versuch in der realen Anlage durchführen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Das machen wir nicht; wir testen stattdessen im Technikum.","speaker":"Martin","named_person":null,"addressee":null}],"expected":{"negative_act_form":"explicit_non_pursuit","candidate_observation_id":"obs_2","target_observation_id":"obs_1","action_concepts":[["Versuch"],["durchführen"]],"material_concepts":[["real"],["Anlage"]],"forbidden_concepts":["Technikum"],"explicitly_rejected":true}}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import copy, json, unittest
|
||||
from pathlib import Path
|
||||
|
||||
from src.meeting_lab.controlled_semantic_derivation.experiment_h import DerivationValidationError
|
||||
from src.meeting_lab.controlled_semantic_derivation.experiment_rejection_v1 import (
|
||||
build_target_prompt, derive, load_cases, validate_target,
|
||||
)
|
||||
|
||||
CASES=load_cases(Path("tests/gold/controlled_rejection_v1/cases.json"))
|
||||
BY_ID={c["case_id"]:c for c in CASES}
|
||||
|
||||
def negative(case, form=None):
|
||||
return {"observation_id":case["expected"]["candidate_observation_id"],"negative_act_form":form or case["expected"]["negative_act_form"],"normalized_action_text":None if (form or case["expected"]["negative_act_form"])=="none" else "semantische Aktion"}
|
||||
def target(case, oid=None, text="konkrete Zielhandlung"):
|
||||
return {"candidate_observation_id":case["expected"]["candidate_observation_id"],"target_observation_id":oid or case["expected"]["target_observation_id"],"normalized_target_text":text}
|
||||
|
||||
class ControlledRejectionV1Tests(unittest.TestCase):
|
||||
def test_positive_forms_derive_and_provenance_survives(self):
|
||||
for cid in ("CR-01","CR-02","CR-07","CR-08"):
|
||||
c=BY_ID[cid]; out=derive(c["observations"],negative(c),target(c))
|
||||
self.assertEqual(out["derived_result"]["status"],"explicitly_rejected")
|
||||
self.assertEqual(out["derived_result"]["support"]["target"]["evidence_id"],"e1")
|
||||
self.assertNotIn("responsible_person",json.dumps(out["derived_result"]))
|
||||
|
||||
def test_noneligible_form_never_derives_even_with_target(self):
|
||||
for form in ["personal_preference","recommendation","temporary_non_action","none"]:
|
||||
c=BY_ID["CR-03"]; self.assertIsNone(derive(c["observations"],negative(c,form),target(c))["derived_result"])
|
||||
|
||||
def test_missing_target_prevents_derivation(self):
|
||||
c=BY_ID["CR-02"]; t=target(c); t.update(target_observation_id=None,normalized_target_text=None)
|
||||
self.assertIsNone(derive(c["observations"],negative(c),t)["derived_result"])
|
||||
|
||||
def test_unknown_ids_rejected(self):
|
||||
for field in ["candidate_observation_id","target_observation_id"]:
|
||||
c=BY_ID["CR-02"]; t=target(c); t[field]="obs_unknown"
|
||||
with self.assertRaises(DerivationValidationError): validate_target(t,c["observations"])
|
||||
|
||||
def test_target_after_candidate_cannot_derive(self):
|
||||
c=copy.deepcopy(BY_ID["CR-02"]); n={"observation_id":"obs_1","negative_act_form":"explicit_non_pursuit","normalized_action_text":"x"}; t={"candidate_observation_id":"obs_1","target_observation_id":"obs_2","normalized_target_text":"x"}
|
||||
self.assertIsNone(derive(c["observations"],n,t)["derived_result"])
|
||||
|
||||
def test_same_observation_target_allowed(self):
|
||||
c=BY_ID["CR-01"]; self.assertIsNotNone(derive(c["observations"],negative(c),target(c))["derived_result"])
|
||||
|
||||
def test_duplicate_provenance_rejected(self):
|
||||
for field in ["observation_id","evidence_id"]:
|
||||
c=copy.deepcopy(BY_ID["CR-02"]); c["observations"][1][field]=c["observations"][0][field]
|
||||
with self.assertRaises(DerivationValidationError): derive(c["observations"],negative(BY_ID["CR-02"]),target(BY_ID["CR-02"]))
|
||||
|
||||
def test_target_text_constraints(self):
|
||||
c=BY_ID["CR-02"]
|
||||
with self.assertRaises(DerivationValidationError): validate_target(target(c,text=""),c["observations"])
|
||||
t=target(c); t["target_observation_id"]=None
|
||||
with self.assertRaises(DerivationValidationError): validate_target(t,c["observations"])
|
||||
|
||||
def test_null_target_accepts_only_null_text(self):
|
||||
c=BY_ID["CR-02"]; t=target(c); t.update(target_observation_id=None,normalized_target_text=None)
|
||||
self.assertEqual(validate_target(t,c["observations"]),t)
|
||||
|
||||
def test_forbidden_and_unknown_fields_rejected(self):
|
||||
for extra in [{"status":"rejected"},{"nested":{"decision":True}},{"extra":1}]:
|
||||
c=BY_ID["CR-02"]; t=target(c); t.update(extra)
|
||||
with self.assertRaises(DerivationValidationError): validate_target(t,c["observations"])
|
||||
|
||||
def test_candidate_outputs_must_agree(self):
|
||||
c=BY_ID["CR-02"]; t=target(c); t["candidate_observation_id"]="obs_1"
|
||||
with self.assertRaises(DerivationValidationError): derive(c["observations"],negative(c),t)
|
||||
|
||||
def test_scope_and_alternative_fixture_contract(self):
|
||||
self.assertLessEqual({"real","Druckversuch"},{x for group in BY_ID["CR-07"]["expected"]["material_concepts"] for x in group})
|
||||
self.assertEqual(BY_ID["CR-08"]["expected"]["forbidden_concepts"],["Technikum"])
|
||||
|
||||
def test_target_prompt_is_semantic_only_and_fixed(self):
|
||||
prompt=build_target_prompt(BY_ID["CR-08"])
|
||||
self.assertIn("Ignore any separate positive alternative",prompt)
|
||||
self.assertIn("Do not classify the negative act",prompt)
|
||||
|
||||
def test_accepted_negative_act_inputs_are_exactly_reused(self):
|
||||
root=Path("artifacts/experiments/negative_act_form_v0/20260820_qwen35_9b_single_run")
|
||||
for cid,nid in (("CR-01","NA-01"),("CR-03","NA-03"),("CR-04","NA-04"),("CR-05","NA-05"),("CR-06","NA-06")):
|
||||
accepted=json.loads((root/nid.lower()/"v3_style_input_observations.json").read_text())
|
||||
self.assertEqual(BY_ID[cid]["observations"],accepted)
|
||||
Reference in New Issue
Block a user