Document target normalization V0 experiment

docs/experiments.md

@@ -2138,6 +2138,55 @@ normalization, but not deriving rejection or integrating a new pipeline.
 Artifacts are preserved under
 `artifacts/experiments/target_resolution_v1_diagnostic/20260820_qwen35_9b_single_run/`.
 
+## EXP-0039 — Target Normalization V0
+
+Status: Experimental; normalization improved but incomplete
+
+Date: 2026-08-20
+
+Target Resolution V1 established correct linkage for all four narrow cases and
+eliminated structural ID failures with deterministic self-linkage, closed ID
+lists, and true JSON Schemas. Its remaining failures were normalization-only.
+This isolated follow-up therefore accepted candidate and target IDs as fixed
+input and tested only reconstruction of the positive German action meaning. It
+contains no target selection, Negative Act classification, eligibility logic,
+rejection derivation, or production integration.
+
+The strict output schema contained exactly `candidate_observation_id`,
+`target_observation_id`, and `normalized_target_text`. Both IDs were constrained
+to their supplied values with JSON Schema `const`; normalized text was a
+non-empty string and null was disallowed. The one fixed prompt required removal
+of negative polarity, preservation of action, continuation, material scope and
+source language, and exclusion of separate alternatives.
+
+| Case | Actual normalized target | Verdict |
+| --- | --- | --- |
+| TN-01 | `Mit Dr. Schlummer zusammenarbeiten` | FAIL: positive polarity and collaboration survived, but continuation was lost |
+| TN-02 | `externe Lösung weiterverfolgen` | PASS |
+| TN-03 | `reale Anlage für den Druckversuch nutzen` | PASS |
+| TN-04 | `Versuch in der realen Anlage durchführen` | PASS; Technikum alternative excluded |
+
+Result: three PASS, zero PARTIAL, one FAIL. All four outputs passed strict
+schema validation and copied both fixed IDs exactly, so changed-ID count was
+zero. Polarity-error count was zero: even TN-01 removed rejection and negation.
+Action/continuation-loss count was one (TN-01); material purpose/location
+scope-loss count was zero; alternative-absorption count was zero. There was no
+unsupported strengthening or normative leakage.
+
+Configuration: exactly four `qwen3.5:9B` calls, temperature 0,
+`think=false`, true JSON Schema, `num_ctx=16384`, `num_predict=1024`, no
+retries, voting, or prompt tuning. Aggregate runner time was 5.066 seconds.
+
+Conclusion: isolating normalization solved the polarity and scoped-action
+failures seen in Target Resolution V1 for three of four cases, including exact
+pressure-test scope and alternative isolation. Continuation semantics remain
+unreliable in the self-contained collaboration case, so Target Normalization
+V0 does not meet its full acceptance criterion. The result does not justify
+rejection derivation or production integration.
+
+Artifacts are preserved under
+`artifacts/experiments/target_normalization_v0/20260820_qwen35_9b_single_run/`.
+
 ## EXP-0026 — Topic-oriented Discussion Subject reconstruction V2 prototype
 
 Date: 2026-08-11

scripts/run_target_normalization_experiment.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+import sys
+from pathlib import Path
+ROOT=Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path: sys.path.insert(0,str(ROOT))
+from src.meeting_lab.controlled_semantic_derivation.experiment_target_normalization import main
+if __name__=="__main__": raise SystemExit(main())

src/meeting_lab/controlled_semantic_derivation/experiment_target_normalization.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""Target Normalization V0: reconstruct target text with fixed linkage."""
+from __future__ import annotations
+import argparse,json,time
+from pathlib import Path
+from typing import Any,Callable
+import requests
+from .experiment_h import DEFAULT_ENDPOINT,DEFAULT_MODEL,DerivationValidationError,OBSERVATION_KEYS
+
+SCHEMA_VERSION="experimental-target-normalization-v0"
+OUTPUT_KEYS={"candidate_observation_id","target_observation_id","normalized_target_text"}
+LINK_KEYS={"candidate_observation_id","target_observation_id"}
+FORBIDDEN={"negative_act_form","rejection_form","explicitly_rejected","status","decision","outcome","topic_status","responsible_person","responsibility","owner","requested_actor","action_item","protocol_category","confidence","relation","relations","graph","unresolved_issue"}
+PROMPT="""The candidate and target observation IDs below are already resolved. Copy both IDs exactly; do not perform target selection. Reconstruct only the concrete POSITIVE action or option meaning targeted by the negative act. Remove rejection and negation polarity while preserving the underlying positive action. Preserve German source language, material qualifiers, purpose, location, named people, and continuation. Exclude separate positive alternatives. Do not summarize the discussion or infer rejection, decision, outcome, status, responsibility, ownership, protocol relevance, confidence, relations, graphs, or topic state. Return only the JSON-Schema-conforming object; null is not permitted.\n\nExample A observations: [{{"observation_id":"obs_a","content":"Mit Frau Beispiel arbeiten wir nicht weiter."}}]\nFixed IDs: candidate=obs_a, target=obs_a\nOutput: {{"candidate_observation_id":"obs_a","target_observation_id":"obs_a","normalized_target_text":"Zusammenarbeit mit Frau Beispiel fortsetzen"}}\n\nExample B observations: [{{"observation_id":"obs_a","content":"Für den Druckversuch steht die reale Anlage zur Diskussion."}},{{"observation_id":"obs_b","content":"Die reale Anlage nutzen wir dafür nicht."}}]\nFixed IDs: candidate=obs_b, target=obs_a\nOutput: {{"candidate_observation_id":"obs_b","target_observation_id":"obs_a","normalized_target_text":"reale Anlage für den Druckversuch nutzen"}}\n\nFixed candidate_observation_id: {candidate}\nFixed target_observation_id: {target}\nV3-style observations:\n{observations}"""
+
+def _keys(value,required,where):
+    if not isinstance(value,dict): raise DerivationValidationError(f"{where} must be an object")
+    if set(value)!=required: raise DerivationValidationError(f"{where} keys invalid: missing={sorted(required-set(value))}, unknown={sorted(set(value)-required)}")
+def _text(value,where):
+    if not isinstance(value,str) or not value.strip(): raise DerivationValidationError(f"{where} must be non-empty")
+    return value.strip()
+def _reject_forbidden(value,where="output"):
+    if isinstance(value,dict):
+        bad=FORBIDDEN & set(value)
+        if bad: raise DerivationValidationError(f"{where} contains forbidden fields: {sorted(bad)}")
+        for key,item in value.items(): _reject_forbidden(item,f"{where}.{key}")
+    elif isinstance(value,list):
+        for index,item in enumerate(value): _reject_forbidden(item,f"{where}[{index}]")
+def validate_observations(observations):
+    if not isinstance(observations,list) or not observations: raise DerivationValidationError("observations must be non-empty")
+    ids=set(); evidence=set()
+    for index,item in enumerate(observations):
+        _keys(item,OBSERVATION_KEYS,f"observations[{index}]"); oid=_text(item["observation_id"],"observation_id"); eid=_text(item["evidence_id"],"evidence_id")
+        if oid in ids or eid in evidence: raise DerivationValidationError("observation/evidence provenance must be unique")
+        ids.add(oid); evidence.add(eid); _text(item["content"],"content"); _text(item["speaker"],"speaker")
+    return ids
+def validate_linkage(case):
+    ids=validate_observations(case["observations"]); linkage=case["fixed_linkage"]; _keys(linkage,LINK_KEYS,"fixed_linkage")
+    for field in LINK_KEYS:
+        if _text(linkage[field],field) not in ids: raise DerivationValidationError(f"{field} is unknown")
+    return linkage
+def output_schema(case):
+    link=validate_linkage(case)
+    return {"type":"object","additionalProperties":False,"required":["candidate_observation_id","target_observation_id","normalized_target_text"],"properties":{"candidate_observation_id":{"const":link["candidate_observation_id"]},"target_observation_id":{"const":link["target_observation_id"]},"normalized_target_text":{"type":"string","minLength":1}}}
+def build_prompt(case):
+    link=validate_linkage(case)
+    return PROMPT.format(candidate=link["candidate_observation_id"],target=link["target_observation_id"],observations=json.dumps(case["observations"],ensure_ascii=False,indent=2))
+def validate_output(data,case):
+    _reject_forbidden(data); _keys(data,OUTPUT_KEYS,"output"); link=validate_linkage(case)
+    if data["candidate_observation_id"]!=link["candidate_observation_id"]: raise DerivationValidationError("candidate ID changed")
+    if data["target_observation_id"]!=link["target_observation_id"]: raise DerivationValidationError("target ID changed")
+    _text(data["normalized_target_text"],"normalized_target_text"); return data
+def build_payload(model,prompt,schema,num_ctx,num_predict): return {"model":model,"prompt":prompt,"think":False,"stream":False,"format":schema,"options":{"temperature":0,"num_ctx":num_ctx,"num_predict":num_predict}}
+def call_schema(endpoint,model,prompt,schema,timeout,num_ctx,num_predict):
+    started=time.perf_counter(); response=requests.post(endpoint,json=build_payload(model,prompt,schema,num_ctx,num_predict),timeout=timeout); elapsed=time.perf_counter()-started; response.raise_for_status(); body=response.json(); raw=body.get("response")
+    if not isinstance(raw,str) or not raw.strip(): raise ValueError("Ollama returned no usable response")
+    return raw.strip(),{"model":body.get("model",model),"elapsed_seconds":round(elapsed,3),"total_duration_ns":body.get("total_duration"),"prompt_eval_count":body.get("prompt_eval_count"),"eval_count":body.get("eval_count"),"configuration":{"temperature":0,"think":False,"format":"json_schema_object","num_ctx":num_ctx,"num_predict":num_predict,"retries":0}}
+def _concepts(text,groups):
+    folded=text.casefold(); return all(any(alias.casefold() in folded for alias in group) for group in groups)
+def evaluate(case,output):
+    validate_output(output,case); expected=case["expected"]; text=output["normalized_target_text"]; folded=text.casefold(); action=_concepts(text,expected["action_concepts"]); scope=_concepts(text,expected["material_concepts"]); forbidden=[x for x in expected["forbidden_concepts"] if x.casefold() in folded]; positive=not any(x in forbidden for x in ("nicht","beenden")); german=any(x.casefold() in folded for x in expected["german_markers"]); alternative=not any(x.casefold() in folded for x in ("technikum","stattdessen")); strengthening=False
+    label="PASS" if positive and action and scope and german and alternative and not forbidden and not strengthening else "FAIL"
+    return {"case_id":case["case_id"],"classification":label,"expected_normalized_target_text":expected["normalized_target_text"],"actual_normalized_target_text":text,"positive_polarity_correct":positive,"action_semantics_preserved":action,"material_scope_preserved":scope,"source_language_preserved":german,"separate_alternative_excluded":alternative,"forbidden_semantics_present":forbidden,"unsupported_strengthening":strengthening,"normative_leakage":False,"candidate_id_unchanged":True,"target_id_unchanged":True}
+def load_cases(path):
+    data=json.loads(path.read_text(encoding="utf-8")); _keys(data,{"schema_version","cases"},"fixture")
+    if data["schema_version"]!=SCHEMA_VERSION: raise DerivationValidationError("wrong schema version")
+    for case in data["cases"]: validate_linkage(case)
+    return data["cases"]
+def _write(path,value): path.write_text(json.dumps(value,ensure_ascii=False,indent=2)+"\n",encoding="utf-8")
+def run(args,caller:Callable=call_schema):
+    cases=load_cases(args.cases); args.output.mkdir(parents=True,exist_ok=False); _write(args.output/"gold_cases.json",{"schema_version":SCHEMA_VERSION,"cases":cases}); evaluations=[]; calls=failures=0; started=time.perf_counter()
+    for case in cases:
+        folder=args.output/case["case_id"].lower(); folder.mkdir(); _write(folder/"v3_style_input_observations.json",case["observations"]); _write(folder/"fixed_linkage.json",case["fixed_linkage"]); schema=output_schema(case); _write(folder/"ollama_json_schema.json",schema); prompt=build_prompt(case); (folder/"prompt.txt").write_text(prompt,encoding="utf-8")
+        try:
+            raw,metadata=caller(args.endpoint,args.model,prompt,schema,args.timeout,args.num_ctx,args.num_predict); calls+=1; (folder/"raw_model_response.txt").write_text(raw+"\n",encoding="utf-8"); _write(folder/"ollama_metadata.json",metadata); parsed=json.loads(raw); _write(folder/"parsed_response.json",parsed); validate_output(parsed,case); _write(folder/"structural_validation.json",{"valid":True}); _write(folder/"normalized_target_result.json",{"normalized_target_text":parsed["normalized_target_text"]}); evaluation=evaluate(case,parsed)
+        except Exception as exc:
+            failures+=1; _write(folder/"structural_validation.json",{"valid":False,"error":str(exc)}); evaluation={"case_id":case["case_id"],"classification":"FAIL","error":str(exc),"normative_leakage":False}
+        _write(folder/"evaluation.json",evaluation); evaluations.append(evaluation)
+    summary={"experiment":"target_normalization_v0","model":args.model,"llm_call_count":calls,"structural_validation_failure_count":failures,"runtime_seconds":round(time.perf_counter()-started,3),"counts":{x:sum(e["classification"]==x for e in evaluations) for x in ["PASS","PARTIAL","FAIL"]},"evaluations":evaluations}; _write(args.output/"summary.json",summary); return summary
+def main():
+    p=argparse.ArgumentParser(); p.add_argument("cases",type=Path); p.add_argument("-o","--output",type=Path,required=True); p.add_argument("--model",default=DEFAULT_MODEL); p.add_argument("--endpoint",default=DEFAULT_ENDPOINT); p.add_argument("--timeout",type=int,default=300); p.add_argument("--num-ctx",type=int,default=16384); p.add_argument("--num-predict",type=int,default=1024); print(json.dumps(run(p.parse_args()),ensure_ascii=False,indent=2)); return 0

tests/gold/target_normalization_v0/cases.json

@@ -0,0 +1,9 @@
+{
+  "schema_version":"experimental-target-normalization-v0",
+  "cases":[
+    {"case_id":"TN-01","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Mit Dr. Schlummer arbeiten wir nicht weiter.","speaker":"Martin","named_person":"Dr. Schlummer","addressee":null}],"fixed_linkage":{"candidate_observation_id":"obs_1","target_observation_id":"obs_1"},"expected":{"normalized_target_text":"Zusammenarbeit mit Dr. Schlummer fortsetzen","action_concepts":[["Zusammenarbeit","arbeiten"],["Schlummer"],["fortsetzen","weiter"]],"material_concepts":[],"forbidden_concepts":["nicht","beenden"],"german_markers":["Zusammenarbeit","arbeiten","fortsetzen"]}},
+    {"case_id":"TN-02","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Eine Möglichkeit wäre, die externe Lösung weiterzuverfolgen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Das verfolgen wir nicht weiter.","speaker":"Martin","named_person":null,"addressee":null}],"fixed_linkage":{"candidate_observation_id":"obs_2","target_observation_id":"obs_1"},"expected":{"normalized_target_text":"externe Lösung weiterverfolgen","action_concepts":[["externe Lösung"],["weiterverfolgen","weiter verfolgen"]],"material_concepts":[],"forbidden_concepts":["nicht"],"german_markers":["Lösung","verfolgen"]}},
+    {"case_id":"TN-03","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Für den Druckversuch steht die reale Anlage zur Diskussion.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Die reale Anlage nutzen wir dafür nicht.","speaker":"Martin","named_person":null,"addressee":null}],"fixed_linkage":{"candidate_observation_id":"obs_2","target_observation_id":"obs_1"},"expected":{"normalized_target_text":"reale Anlage für den Druckversuch nutzen","action_concepts":[["Anlage"],["nutzen"]],"material_concepts":[["real"],["Druckversuch"]],"forbidden_concepts":["nicht","zur Diskussion"],"german_markers":["Anlage","Druckversuch","nutzen"]}},
+    {"case_id":"TN-04","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Wir könnten den Versuch in der realen Anlage durchführen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Das machen wir nicht; wir testen stattdessen im Technikum.","speaker":"Martin","named_person":null,"addressee":null}],"fixed_linkage":{"candidate_observation_id":"obs_2","target_observation_id":"obs_1"},"expected":{"normalized_target_text":"Versuch in der realen Anlage durchführen","action_concepts":[["Versuch"],["durchführen"]],"material_concepts":[["real"],["Anlage"]],"forbidden_concepts":["nicht","Technikum","stattdessen"],"german_markers":["Versuch","Anlage","durchführen"]}}
+  ]
+}

tests/test_target_normalization_experiment.py

@@ -0,0 +1,51 @@
+import argparse,copy,json,tempfile,unittest
+from pathlib import Path
+from unittest.mock import Mock
+import src.meeting_lab.controlled_semantic_derivation.experiment_target_normalization as module
+from src.meeting_lab.controlled_semantic_derivation.experiment_h import DerivationValidationError
+
+CASES=module.load_cases(Path("tests/gold/target_normalization_v0/cases.json")); BY={c["case_id"]:c for c in CASES}
+def output(case,text=None):
+ link=case["fixed_linkage"]; return {"candidate_observation_id":link["candidate_observation_id"],"target_observation_id":link["target_observation_id"],"normalized_target_text":text or case["expected"]["normalized_target_text"]}
+
+class TargetNormalizationTests(unittest.TestCase):
+ def test_exact_id_copying_accepted(self):
+  for case in CASES: self.assertEqual(module.validate_output(output(case),case),output(case))
+ def test_changed_candidate_rejected(self):
+  case=BY["TN-02"]; data=output(case); data["candidate_observation_id"]="obs_1"
+  with self.assertRaises(DerivationValidationError): module.validate_output(data,case)
+ def test_changed_target_rejected(self):
+  case=BY["TN-02"]; data=output(case); data["target_observation_id"]="obs_2"
+  with self.assertRaises(DerivationValidationError): module.validate_output(data,case)
+ def test_empty_and_null_text_rejected(self):
+  case=BY["TN-01"]
+  for value in ("",None):
+   data=output(case); data["normalized_target_text"]=value
+   with self.assertRaises(DerivationValidationError): module.validate_output(data,case)
+ def test_unknown_and_forbidden_fields_rejected(self):
+  case=BY["TN-01"]
+  for extra in ({"extra":1},{"status":"x"},{"nested":{"decision":True}}):
+   data=output(case); data.update(extra)
+   with self.assertRaises(DerivationValidationError): module.validate_output(data,case)
+ def test_true_schema_fixes_both_ids_and_disallows_null(self):
+  case=BY["TN-02"]; schema=module.output_schema(case); self.assertEqual(schema["properties"]["candidate_observation_id"]["const"],"obs_2"); self.assertEqual(schema["properties"]["target_observation_id"]["const"],"obs_1"); self.assertEqual(schema["properties"]["normalized_target_text"]["type"],"string"); self.assertFalse(schema["additionalProperties"])
+ def test_payload_uses_schema_object(self):
+  schema=module.output_schema(BY["TN-01"]); payload=module.build_payload("qwen3.5:9B","p",schema,16384,1024); self.assertIs(payload["format"],schema); self.assertIsInstance(payload["format"],dict)
+ def test_duplicate_ids_and_evidence_rejected(self):
+  for field in ("observation_id","evidence_id"):
+   case=copy.deepcopy(BY["TN-02"]); case["observations"][1][field]=case["observations"][0][field]
+   with self.assertRaises(DerivationValidationError): module.validate_linkage(case)
+ def test_prompt_is_fixed_normalization_only(self):
+  first=module.build_prompt(BY["TN-01"]); second=module.build_prompt(BY["TN-02"]); self.assertIn("do not perform target selection",first); self.assertIn("concrete POSITIVE action",first); self.assertEqual(first.split("Fixed candidate_observation_id:")[0],second.split("Fixed candidate_observation_id:")[0])
+ def test_no_target_selection_or_rejection_derivation_exists(self):
+  self.assertFalse(hasattr(module,"select_target")); self.assertFalse(hasattr(module,"derive")); self.assertNotIn("explicitly_rejected",module.OUTPUT_KEYS); self.assertNotIn("status",module.OUTPUT_KEYS)
+ def test_artifacts_preserve_fixed_linkage(self):
+  case=BY["TN-01"]; fixture={"schema_version":module.SCHEMA_VERSION,"cases":[case]}; caller=Mock(return_value=(json.dumps(output(case)),{"model":"qwen3.5:9B"}))
+  with tempfile.TemporaryDirectory() as tmp:
+   root=Path(tmp); path=root/"cases.json"; path.write_text(json.dumps(fixture)); out=root/"out"; args=argparse.Namespace(cases=path,output=out,endpoint="x",model="qwen3.5:9B",timeout=1,num_ctx=16384,num_predict=1024); summary=module.run(args,caller); self.assertEqual(summary["llm_call_count"],1); self.assertEqual(json.loads((out/"tn-01"/"fixed_linkage.json").read_text()),case["fixed_linkage"]); self.assertTrue((out/"tn-01"/"normalized_target_result.json").exists())
+ def test_negative_polarity_fails_evaluation(self):
+  case=BY["TN-01"]; self.assertEqual(module.evaluate(case,output(case,"Mit Dr. Schlummer arbeiten wir nicht weiter"))["classification"],"FAIL")
+ def test_material_scope_and_alternative_contract(self):
+  self.assertEqual(module.evaluate(BY["TN-03"],output(BY["TN-03"]))["classification"],"PASS"); self.assertEqual(module.evaluate(BY["TN-04"],output(BY["TN-04"]))["classification"],"PASS")
+
+if __name__=="__main__": unittest.main()