Add negative act form experiment
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"schema_version": "experimental-negative-act-form-gold-v0",
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "NA-01", "description": "Explicit non-pursuit",
|
||||
"observations": [
|
||||
{"observation_id": "obs_1", "evidence_id": "e1", "content": "Martin: Mit Dr. Schlummer arbeiten wir nicht weiter.", "speaker": "Martin", "named_person": "Dr. Schlummer", "addressee": null}
|
||||
],
|
||||
"expected": {"observation_id": "obs_1", "negative_act_form": "explicit_non_pursuit", "action_concepts": [["schlummer"], ["arbeit", "collabor"], ["weiter", "fortsetz", "continu"]]}
|
||||
},
|
||||
{
|
||||
"case_id": "NA-02", "description": "Explicit non-pursuit paraphrase",
|
||||
"observations": [
|
||||
{"observation_id": "obs_1", "evidence_id": "e1", "content": "Martin: Die externe Lösung verfolgen wir nicht weiter.", "speaker": "Martin", "named_person": null, "addressee": null}
|
||||
],
|
||||
"expected": {"observation_id": "obs_1", "negative_act_form": "explicit_non_pursuit", "action_concepts": [["extern"], ["lösung", "solution"], ["weiter", "pursu", "continu"]]}
|
||||
},
|
||||
{
|
||||
"case_id": "NA-03", "description": "Personal preference with local context",
|
||||
"observations": [
|
||||
{"observation_id": "obs_1", "evidence_id": "e1", "content": "Martin: Wir könnten die reale Anlage für den Versuch nutzen.", "speaker": "Martin", "named_person": null, "addressee": null},
|
||||
{"observation_id": "obs_2", "evidence_id": "e2", "content": "Martin: Ich würde das nicht machen.", "speaker": "Martin", "named_person": null, "addressee": null}
|
||||
],
|
||||
"expected": {"observation_id": "obs_2", "negative_act_form": "personal_preference", "action_concepts": [["real"], ["anlage", "plant"], ["versuch", "trial", "test"]]}
|
||||
},
|
||||
{
|
||||
"case_id": "NA-04", "description": "Negative recommendation",
|
||||
"observations": [
|
||||
{"observation_id": "obs_1", "evidence_id": "e1", "content": "Martin: Wir könnten die reale Anlage verwenden.", "speaker": "Martin", "named_person": null, "addressee": null},
|
||||
{"observation_id": "obs_2", "evidence_id": "e2", "content": "Martin: Ich würde eher davon abraten.", "speaker": "Martin", "named_person": null, "addressee": null}
|
||||
],
|
||||
"expected": {"observation_id": "obs_2", "negative_act_form": "recommendation", "action_concepts": [["real"], ["anlage", "plant"], ["verwend", "use"]]}
|
||||
},
|
||||
{
|
||||
"case_id": "NA-05", "description": "Temporary non-action",
|
||||
"observations": [
|
||||
{"observation_id": "obs_1", "evidence_id": "e1", "content": "Martin: Wir könnten die Waschstufe einbauen.", "speaker": "Martin", "named_person": null, "addressee": null},
|
||||
{"observation_id": "obs_2", "evidence_id": "e2", "content": "Martin: Das machen wir erstmal noch nicht.", "speaker": "Martin", "named_person": null, "addressee": null}
|
||||
],
|
||||
"expected": {"observation_id": "obs_2", "negative_act_form": "temporary_non_action", "action_concepts": [["waschstufe", "washing stage"], ["einbau", "install"]]}
|
||||
},
|
||||
{
|
||||
"case_id": "NA-06", "description": "Concern only",
|
||||
"observations": [
|
||||
{"observation_id": "obs_1", "evidence_id": "e1", "content": "Martin: Wir könnten das neue Material einsetzen.", "speaker": "Martin", "named_person": null, "addressee": null},
|
||||
{"observation_id": "obs_2", "evidence_id": "e2", "content": "Martin: Das wäre kritisch.", "speaker": "Martin", "named_person": null, "addressee": null}
|
||||
],
|
||||
"expected": {"observation_id": "obs_2", "negative_act_form": "none", "action_concepts": []}
|
||||
},
|
||||
{
|
||||
"case_id": "NA-07", "description": "Uncertainty",
|
||||
"observations": [
|
||||
{"observation_id": "obs_1", "evidence_id": "e1", "content": "Martin: Eine Möglichkeit wäre, die Waschstufe einzubauen.", "speaker": "Martin", "named_person": null, "addressee": null},
|
||||
{"observation_id": "obs_2", "evidence_id": "e2", "content": "Martin: Ich weiß nicht, ob das sinnvoll ist.", "speaker": "Martin", "named_person": null, "addressee": null}
|
||||
],
|
||||
"expected": {"observation_id": "obs_2", "negative_act_form": "none", "action_concepts": []}
|
||||
},
|
||||
{
|
||||
"case_id": "NA-08", "description": "Factual negation",
|
||||
"observations": [
|
||||
{"observation_id": "obs_1", "evidence_id": "e1", "content": "Martin: Das Material ist nicht verfügbar.", "speaker": "Martin", "named_person": null, "addressee": null}
|
||||
],
|
||||
"expected": {"observation_id": "obs_1", "negative_act_form": "none", "action_concepts": []}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import argparse
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import src.meeting_lab.controlled_semantic_derivation.experiment_negative_act as module
|
||||
from src.meeting_lab.controlled_semantic_derivation.experiment_negative_act import (
|
||||
DerivationValidationError,
|
||||
build_ollama_payload,
|
||||
build_prompt,
|
||||
evaluate_case,
|
||||
load_gold_cases,
|
||||
parse_model_json,
|
||||
run_experiment,
|
||||
validate_classification,
|
||||
)
|
||||
|
||||
|
||||
GOLD_PATH = Path("tests/gold/negative_act_form_v0/cases.json")
|
||||
|
||||
|
||||
FORM_TEXT = {
|
||||
"NA-01": "Zusammenarbeit mit Dr. Schlummer fortsetzen",
|
||||
"NA-02": "externe Lösung weiterverfolgen",
|
||||
"NA-03": "reale Anlage für den Versuch nutzen",
|
||||
"NA-04": "reale Anlage verwenden",
|
||||
"NA-05": "Waschstufe einbauen",
|
||||
}
|
||||
|
||||
|
||||
def classification_for(case):
|
||||
expected = case["expected"]
|
||||
return {
|
||||
"observation_id": expected["observation_id"],
|
||||
"negative_act_form": expected["negative_act_form"],
|
||||
"normalized_action_text": FORM_TEXT.get(case["case_id"]),
|
||||
}
|
||||
|
||||
|
||||
class NegativeActFormExperimentTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.cases = load_gold_cases(GOLD_PATH)
|
||||
cls.by_id = {case["case_id"]: case for case in cls.cases}
|
||||
|
||||
def test_fixture_contains_exactly_na_01_through_na_08(self):
|
||||
self.assertEqual(list(self.by_id), [f"NA-{number:02d}" for number in range(1, 9)])
|
||||
|
||||
def test_exact_schema_is_accepted(self):
|
||||
case = self.by_id["NA-01"]
|
||||
self.assertEqual(validate_classification(classification_for(case), case["observations"]), classification_for(case))
|
||||
|
||||
def test_unknown_field_is_rejected(self):
|
||||
case = self.by_id["NA-01"]
|
||||
classification = classification_for(case)
|
||||
classification["explanation"] = "extra"
|
||||
with self.assertRaisesRegex(DerivationValidationError, "unknown keys"):
|
||||
validate_classification(classification, case["observations"])
|
||||
|
||||
def test_invalid_enum_is_rejected(self):
|
||||
case = self.by_id["NA-01"]
|
||||
classification = classification_for(case)
|
||||
classification["negative_act_form"] = "rejection"
|
||||
with self.assertRaisesRegex(DerivationValidationError, "unsupported value"):
|
||||
validate_classification(classification, case["observations"])
|
||||
|
||||
def test_non_none_requires_normalized_action_text(self):
|
||||
case = self.by_id["NA-01"]
|
||||
for value in (None, ""):
|
||||
classification = classification_for(case)
|
||||
classification["normalized_action_text"] = value
|
||||
with self.subTest(value=value), self.assertRaises(DerivationValidationError):
|
||||
validate_classification(classification, case["observations"])
|
||||
|
||||
def test_none_requires_null_normalized_action_text(self):
|
||||
case = self.by_id["NA-06"]
|
||||
classification = classification_for(case)
|
||||
self.assertIsNone(classification["normalized_action_text"])
|
||||
classification["normalized_action_text"] = "Material einsetzen"
|
||||
with self.assertRaisesRegex(DerivationValidationError, "requires null"):
|
||||
validate_classification(classification, case["observations"])
|
||||
|
||||
def test_forbidden_normative_fields_are_rejected_recursively(self):
|
||||
case = self.by_id["NA-01"]
|
||||
fields = (
|
||||
"rejection_form", "explicitly_rejected", "status", "decision", "outcome",
|
||||
"topic_status", "responsible_person", "responsibility", "owner",
|
||||
"requested_actor", "action_item", "protocol_category", "confidence",
|
||||
"relation", "relations", "graph", "unresolved_issue",
|
||||
)
|
||||
for field in fields:
|
||||
classification = classification_for(case)
|
||||
classification["wrapper"] = {field: "forbidden"}
|
||||
with self.subTest(field=field), self.assertRaisesRegex(DerivationValidationError, "forbidden semantic keys"):
|
||||
validate_classification(classification, case["observations"])
|
||||
|
||||
def test_unknown_observation_id_is_rejected(self):
|
||||
case = self.by_id["NA-01"]
|
||||
classification = classification_for(case)
|
||||
classification["observation_id"] = "obs_99"
|
||||
with self.assertRaisesRegex(DerivationValidationError, "unknown observation"):
|
||||
validate_classification(classification, case["observations"])
|
||||
|
||||
def test_malformed_json_is_rejected(self):
|
||||
with self.assertRaises(json.JSONDecodeError):
|
||||
parse_model_json("{bad json")
|
||||
|
||||
def test_all_expected_classifications_evaluate_as_pass(self):
|
||||
for case in self.cases:
|
||||
evaluation = evaluate_case(case, classification_for(case))
|
||||
with self.subTest(case=case["case_id"]):
|
||||
self.assertEqual(evaluation["classification"], "PASS")
|
||||
|
||||
def test_fixed_prompt_contains_candidate_and_no_gold_expectation(self):
|
||||
prompt = build_prompt(self.by_id["NA-03"])
|
||||
self.assertIn("candidate observation is obs_2", prompt)
|
||||
self.assertNotIn("expected", prompt)
|
||||
self.assertNotIn("Who is responsible", prompt)
|
||||
|
||||
def test_fixed_model_configuration(self):
|
||||
payload = build_ollama_payload("qwen3.5:9B", "prompt", 16384, 1024)
|
||||
self.assertFalse(payload["think"])
|
||||
self.assertFalse(payload["stream"])
|
||||
self.assertEqual(payload["options"]["temperature"], 0)
|
||||
|
||||
def test_no_rejection_or_status_derivation_function_exists(self):
|
||||
public_names = {name for name in dir(module) if not name.startswith("_")}
|
||||
self.assertNotIn("derive_rejection", public_names)
|
||||
self.assertFalse(any(name.startswith("derive_") for name in public_names))
|
||||
|
||||
def test_artifacts_preserve_semantic_classification_only(self):
|
||||
case = self.by_id["NA-01"]
|
||||
raw = json.dumps(classification_for(case), ensure_ascii=False)
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
output = Path(temporary) / "run"
|
||||
args = argparse.Namespace(
|
||||
cases=GOLD_PATH, output=output, model="qwen3.5:9B",
|
||||
endpoint="http://unused", timeout=1, num_ctx=16384, num_predict=1024,
|
||||
)
|
||||
with patch.object(module, "load_gold_cases", return_value=[deepcopy(case)]), patch.object(
|
||||
module, "call_ollama", return_value=(raw, {"model": "qwen3.5:9B"})
|
||||
):
|
||||
summary = run_experiment(args)
|
||||
self.assertEqual(summary["successful_llm_call_count"], 1)
|
||||
case_dir = output / "na-01"
|
||||
for filename in (
|
||||
"v3_style_input_observations.json", "prompt.txt", "raw_model_response.txt",
|
||||
"parsed_semantic_classification.json", "structural_validation.json",
|
||||
"evaluation.json", "ollama_metadata.json",
|
||||
):
|
||||
self.assertTrue((case_dir / filename).is_file(), filename)
|
||||
self.assertFalse((case_dir / "final_derived_result.json").exists())
|
||||
self.assertFalse((case_dir / "deterministic_gate_results.json").exists())
|
||||
parsed = json.loads((case_dir / "parsed_semantic_classification.json").read_text())
|
||||
self.assertEqual(set(parsed), {"observation_id", "negative_act_form", "normalized_action_text"})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user