Document target resolution V1 diagnostic
This commit is contained in:
@@ -2088,6 +2088,56 @@ derivation is justified by this result.
|
||||
Artifacts are preserved under
|
||||
`artifacts/experiments/target_resolution_v0/20260820_qwen35_9b_single_run/`.
|
||||
|
||||
## EXP-0038 — Target Resolution V1 Diagnostic
|
||||
|
||||
Status: Experimental; linkage boundary successful, normalization incomplete
|
||||
|
||||
Date: 2026-08-20
|
||||
|
||||
Forensics on failed Target Resolution V0 found a definite prompt defect: its
|
||||
illustrative value `"observation ID or null"` placed both alternatives inside
|
||||
a JSON string. V0 also sent only `format: "json"`, which enforced JSON syntax
|
||||
but not field types. This isolated diagnostic changed only the linkage/output
|
||||
boundary. It contains no rejection derivation or normative semantics.
|
||||
|
||||
Ollama 0.32.6 accepted a true JSON Schema object in `format`. TR1-V1 removed
|
||||
target selection from the model output entirely and deterministically linked
|
||||
the self-contained candidate to itself. TR2-V1 through TR4-V1 used a closed
|
||||
allowed-ID list, an enum of those IDs plus JSON null, typed positive and null
|
||||
examples, recursive strict validation, and one fixed paired prompt. Linkage and
|
||||
normalization were persisted separately.
|
||||
|
||||
| Case | Strategy / ID source | Target | Normalized target | Verdict |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| TR1-V1 | self-contained / deterministic | `obs_1` | `Mit Dr. Schlummer arbeiten wir nicht weiter.` retained negation instead of a positive action meaning | FAIL |
|
||||
| TR2-V1 | paired / LLM | `obs_1` | `externe Lösung weiterverfolgen` | PASS |
|
||||
| TR3-V1 | paired / LLM | `obs_1` | `reale Anlage zur Diskussion` lost `Druckversuch` purpose and the `nutzen` action | FAIL |
|
||||
| TR4-V1 | paired / LLM | `obs_1` | `Versuch in der realen Anlage durchführen`; Technikum excluded | PASS |
|
||||
|
||||
Result: two PASS, zero PARTIAL, two FAIL. All four responses passed their true
|
||||
JSON Schemas. Every resulting target was `obs_1`; wrong-target and
|
||||
unresolved-target counts were zero. The string `"null"` recurrence count was
|
||||
zero, and there were zero structural validation failures. TR1 preserved the
|
||||
collaboration, person, and continuation wording but failed positive-action
|
||||
normalization by retaining negation. TR3 had one material scope loss. TR4
|
||||
preserved real-plant scope and isolated the Technikum alternative. No
|
||||
normative leakage occurred.
|
||||
|
||||
Configuration: exactly four `qwen3.5:9B` calls, temperature 0,
|
||||
`think=false`, `num_ctx=16384`, `num_predict=1024`, no retries, voting, or
|
||||
prompt tuning. Aggregate runner time was 4.557 seconds.
|
||||
|
||||
Conclusion: the V0 string-null failure was primarily a linkage/output-boundary
|
||||
failure rather than evidence that observation-ID linkage is semantically
|
||||
impossible. True typed schemas, closed ID lists, and deterministic self-linkage
|
||||
eliminated every structural and target-ID failure. The experiment still fails
|
||||
its complete acceptance criterion because target normalization is not reliably
|
||||
positive or scope-preserving. These results justify separating linkage from
|
||||
normalization, but not deriving rejection or integrating a new pipeline.
|
||||
|
||||
Artifacts are preserved under
|
||||
`artifacts/experiments/target_resolution_v1_diagnostic/20260820_qwen35_9b_single_run/`.
|
||||
|
||||
## EXP-0026 — Topic-oriented Discussion Subject reconstruction V2 prototype
|
||||
|
||||
Date: 2026-08-11
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
from pathlib import Path
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path: sys.path.insert(0,str(ROOT))
|
||||
from src.meeting_lab.controlled_semantic_derivation.experiment_target_resolution_v1 import main
|
||||
if __name__=="__main__": raise SystemExit(main())
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Target Resolution V1 diagnostic: linkage and normalization only."""
|
||||
from __future__ import annotations
|
||||
import argparse,json,time
|
||||
from pathlib import Path
|
||||
from typing import Any,Callable
|
||||
import requests
|
||||
from .experiment_h import DEFAULT_ENDPOINT,DEFAULT_MODEL,DerivationValidationError,OBSERVATION_KEYS
|
||||
from .experiment_negative_act import validate_classification
|
||||
|
||||
SCHEMA_VERSION="experimental-target-resolution-v1-diagnostic"
|
||||
SELF_KEYS={"candidate_observation_id","normalized_target_text"}; PAIRED_KEYS={"candidate_observation_id","target_observation_id","normalized_target_text"}
|
||||
FORBIDDEN={"negative_act_form","rejection_form","explicitly_rejected","status","decision","outcome","responsible_person","responsibility","owner","requested_actor","action_item","protocol_category","confidence","relation","relations","graph","topic_status","closed","unresolved_issue"}
|
||||
SELF_PROMPT="""Normalize only the concrete positive action meaning in the self-contained candidate observation. The target linkage is already deterministic and is not your task. Preserve German, collaboration, named people, and continuation meaning. Do not output a target ID, rejection, status, decision, outcome, responsibility, ownership, protocol concepts, confidence, relations, graphs, or topic closure. Return only the schema-conforming object.\nCandidate observation ID: {candidate}\nObservation:\n{observations}"""
|
||||
PAIRED_PROMPT="""Resolve and normalize only the concrete local action or option referred to by the candidate negative act. Choose exactly one listed allowed target observation ID, or use JSON null only when no unique local target exists. Never return the string \"null\". Preserve German source language and all material purpose/location scope. Do not absorb a separate positive alternative. Do not output negative-act form, rejection, status, decision, outcome, responsibility, ownership, protocol concepts, confidence, relations, graphs, or topic closure.\nAllowed target observation IDs:\n{allowed}\nConcrete positive typed example:\n{{"candidate_observation_id":"obs_2","target_observation_id":"obs_1","normalized_target_text":"externe Lösung weiterverfolgen"}}\nActual JSON-null example:\n{{"candidate_observation_id":"obs_2","target_observation_id":null,"normalized_target_text":null}}\nReturn only the schema-conforming object.\nCandidate observation ID: {candidate}\nObservations:\n{observations}"""
|
||||
|
||||
def _keys(value,required,where):
|
||||
if not isinstance(value,dict): raise DerivationValidationError(f"{where} must be an object")
|
||||
if set(value)!=required: raise DerivationValidationError(f"{where} keys invalid: missing={sorted(required-set(value))}, unknown={sorted(set(value)-required)}")
|
||||
def _text(value,where):
|
||||
if not isinstance(value,str) or not value.strip(): raise DerivationValidationError(f"{where} must be non-empty")
|
||||
return value.strip()
|
||||
def _forbidden(value,where="output"):
|
||||
if isinstance(value,dict):
|
||||
bad=FORBIDDEN & set(value)
|
||||
if bad: raise DerivationValidationError(f"{where} contains forbidden fields: {sorted(bad)}")
|
||||
for key,item in value.items(): _forbidden(item,f"{where}.{key}")
|
||||
elif isinstance(value,list):
|
||||
for index,item in enumerate(value): _forbidden(item,f"{where}[{index}]")
|
||||
def validate_observations(obs):
|
||||
if not isinstance(obs,list) or not obs: raise DerivationValidationError("observations must be non-empty")
|
||||
ids=[]; evidence=set()
|
||||
for index,item in enumerate(obs):
|
||||
_keys(item,OBSERVATION_KEYS,f"observations[{index}]"); oid=_text(item["observation_id"],"observation_id"); eid=_text(item["evidence_id"],"evidence_id")
|
||||
if oid in ids or eid in evidence: raise DerivationValidationError("observation/evidence provenance must be unique")
|
||||
ids.append(oid); evidence.add(eid); _text(item["content"],"content"); _text(item["speaker"],"speaker")
|
||||
return ids
|
||||
def allowed_ids(case): return validate_observations(case["observations"])
|
||||
def deterministic_self_link(case):
|
||||
if case["strategy"]!="self_contained": raise DerivationValidationError("self-linkage requires self-contained strategy")
|
||||
ids=validate_observations(case["observations"]); candidate=case["negative_act"]["observation_id"]
|
||||
if candidate not in ids: raise DerivationValidationError("unknown candidate")
|
||||
return {"linkage_source":"deterministic","candidate_observation_id":candidate,"target_observation_id":candidate}
|
||||
def output_schema(case):
|
||||
candidate=case["negative_act"]["observation_id"]
|
||||
if case["strategy"]=="self_contained":
|
||||
return {"type":"object","additionalProperties":False,"required":["candidate_observation_id","normalized_target_text"],"properties":{"candidate_observation_id":{"const":candidate},"normalized_target_text":{"type":"string","minLength":1}}}
|
||||
ids=allowed_ids(case)
|
||||
return {"type":"object","additionalProperties":False,"required":["candidate_observation_id","target_observation_id","normalized_target_text"],"properties":{"candidate_observation_id":{"const":candidate},"target_observation_id":{"enum":ids+[None]},"normalized_target_text":{"type":["string","null"]}},"allOf":[{"if":{"properties":{"target_observation_id":{"type":"null"}}},"then":{"properties":{"normalized_target_text":{"type":"null"}}},"else":{"properties":{"normalized_target_text":{"type":"string","minLength":1}}}}]}
|
||||
def build_prompt(case):
|
||||
validate_classification(case["negative_act"],case["observations"]); candidate=case["negative_act"]["observation_id"]
|
||||
if case["strategy"]=="self_contained": return SELF_PROMPT.format(candidate=candidate,observations=json.dumps(case["observations"],ensure_ascii=False,indent=2))
|
||||
return PAIRED_PROMPT.format(candidate=candidate,allowed=json.dumps(allowed_ids(case),ensure_ascii=False),observations=json.dumps(case["observations"],ensure_ascii=False,indent=2))
|
||||
def validate_semantic_output(data,case):
|
||||
_forbidden(data); candidate=case["negative_act"]["observation_id"]
|
||||
if case["strategy"]=="self_contained":
|
||||
_keys(data,SELF_KEYS,"self output")
|
||||
if data["candidate_observation_id"]!=candidate: raise DerivationValidationError("candidate mismatch")
|
||||
_text(data["normalized_target_text"],"normalized_target_text")
|
||||
else:
|
||||
_keys(data,PAIRED_KEYS,"paired output")
|
||||
if data["candidate_observation_id"]!=candidate: raise DerivationValidationError("candidate mismatch")
|
||||
target=data["target_observation_id"]
|
||||
if target=="null": raise DerivationValidationError('string "null" is forbidden')
|
||||
if target is None:
|
||||
if data["normalized_target_text"] is not None: raise DerivationValidationError("null target requires null text")
|
||||
else:
|
||||
if target not in allowed_ids(case): raise DerivationValidationError("target is not an allowed ID")
|
||||
ids=allowed_ids(case)
|
||||
if ids.index(target)>ids.index(candidate): raise DerivationValidationError("target must not occur after candidate")
|
||||
_text(data["normalized_target_text"],"normalized_target_text")
|
||||
return data
|
||||
def combine(case,semantic):
|
||||
validate_semantic_output(semantic,case)
|
||||
if case["strategy"]=="self_contained":
|
||||
link=deterministic_self_link(case); return {**link,"normalized_target_text":semantic["normalized_target_text"]}
|
||||
return {"linkage_source":"llm","candidate_observation_id":semantic["candidate_observation_id"],"target_observation_id":semantic["target_observation_id"],"normalized_target_text":semantic["normalized_target_text"]}
|
||||
def build_payload(model,prompt,schema,num_ctx,num_predict):
|
||||
return {"model":model,"prompt":prompt,"think":False,"stream":False,"format":schema,"options":{"temperature":0,"num_ctx":num_ctx,"num_predict":num_predict}}
|
||||
def call_schema(endpoint,model,prompt,schema,timeout,num_ctx,num_predict):
|
||||
started=time.perf_counter(); response=requests.post(endpoint,json=build_payload(model,prompt,schema,num_ctx,num_predict),timeout=timeout); elapsed=time.perf_counter()-started; response.raise_for_status(); body=response.json(); raw=body.get("response")
|
||||
if not isinstance(raw,str) or not raw.strip(): raise ValueError("Ollama returned no usable response")
|
||||
meta={"model":body.get("model",model),"elapsed_seconds":round(elapsed,3),"total_duration_ns":body.get("total_duration"),"prompt_eval_count":body.get("prompt_eval_count"),"eval_count":body.get("eval_count"),"configuration":{"temperature":0,"think":False,"format":"json_schema_object","num_ctx":num_ctx,"num_predict":num_predict,"retries":0}}
|
||||
return raw.strip(),meta
|
||||
def _concepts(text,groups):
|
||||
folded=(text or "").casefold(); return all(any(x.casefold() in folded for x in group) for group in groups)
|
||||
def evaluate(case,semantic,combined):
|
||||
expected=case["expected"]; text=combined["normalized_target_text"]; target=combined["target_observation_id"]; concepts=_concepts(text,expected["concepts"]); material=_concepts(text,expected["material_concepts"]); isolated=not any(x.casefold() in (text or "").casefold() for x in expected["forbidden_concepts"]); recurrence=semantic.get("target_observation_id")=="null"; correct=target==expected["target_observation_id"]
|
||||
label="PASS" if correct and concepts and material and isolated and not recurrence else ("PARTIAL" if correct and material and isolated and not recurrence else "FAIL")
|
||||
return {"case_id":case["case_id"],"classification":label,"strategy":case["strategy"],"target_id_decision_source":combined["linkage_source"],"expected_target_observation_id":expected["target_observation_id"],"actual_target_observation_id":target,"normalized_target_text":text,"continuation_or_action_preserved":concepts,"material_scope_preserved":material,"alternative_isolated":isolated,"schema_valid":True,"string_null_recurrence":recurrence,"normative_leakage":False}
|
||||
def load_cases(path):
|
||||
data=json.loads(path.read_text(encoding="utf-8")); _keys(data,{"schema_version","cases"},"fixture")
|
||||
if data["schema_version"]!=SCHEMA_VERSION: raise DerivationValidationError("wrong schema version")
|
||||
return data["cases"]
|
||||
def _write(path,value): path.write_text(json.dumps(value,ensure_ascii=False,indent=2)+"\n",encoding="utf-8")
|
||||
def run(args,caller:Callable=call_schema):
|
||||
cases=load_cases(args.cases); args.output.mkdir(parents=True,exist_ok=False); _write(args.output/"gold_cases.json",{"schema_version":SCHEMA_VERSION,"cases":cases}); evaluations=[]; calls=failures=0; started=time.perf_counter()
|
||||
for case in cases:
|
||||
folder=args.output/case["case_id"].lower(); folder.mkdir(); _write(folder/"v3_style_input_observations.json",case["observations"]); _write(folder/"negative_act_form.json",case["negative_act"]); _write(folder/"eligibility.json",{"eligible_for_target_resolution":True,"reason":None}); _write(folder/"deterministic_strategy.json",{"strategy":case["strategy"],"target_id_decision_source":"deterministic" if case["strategy"]=="self_contained" else "llm"}); _write(folder/"allowed_target_ids.json",allowed_ids(case)); schema=output_schema(case); _write(folder/"ollama_json_schema.json",schema); prompt=build_prompt(case); (folder/"prompt.txt").write_text(prompt,encoding="utf-8")
|
||||
try:
|
||||
raw,meta=caller(args.endpoint,args.model,prompt,schema,args.timeout,args.num_ctx,args.num_predict); calls+=1; (folder/"raw_model_response.txt").write_text(raw+"\n",encoding="utf-8"); _write(folder/"ollama_metadata.json",meta); semantic=json.loads(raw); _write(folder/"parsed_semantic_output.json",semantic); validate_semantic_output(semantic,case); combined=combine(case,semantic); _write(folder/"structural_validation.json",{"valid":True}); _write(folder/"deterministic_linkage_result.json",{k:combined[k] for k in ("linkage_source","candidate_observation_id","target_observation_id")}); _write(folder/"normalized_target_result.json",{"normalized_target_text":combined["normalized_target_text"]}); evaluation=evaluate(case,semantic,combined)
|
||||
except Exception as exc:
|
||||
failures+=1; _write(folder/"structural_validation.json",{"valid":False,"error":str(exc)}); evaluation={"case_id":case["case_id"],"classification":"FAIL","strategy":case["strategy"],"schema_valid":False,"error":str(exc)}
|
||||
_write(folder/"evaluation.json",evaluation); evaluations.append(evaluation)
|
||||
summary={"experiment":"target_resolution_v1_diagnostic","model":args.model,"llm_call_count":calls,"structural_validation_failure_count":failures,"runtime_seconds":round(time.perf_counter()-started,3),"counts":{x:sum(e["classification"]==x for e in evaluations) for x in ["PASS","PARTIAL","FAIL"]},"evaluations":evaluations}; _write(args.output/"summary.json",summary); return summary
|
||||
def main():
|
||||
p=argparse.ArgumentParser(); p.add_argument("cases",type=Path); p.add_argument("-o","--output",type=Path,required=True); p.add_argument("--model",default=DEFAULT_MODEL); p.add_argument("--endpoint",default=DEFAULT_ENDPOINT); p.add_argument("--timeout",type=int,default=300); p.add_argument("--num-ctx",type=int,default=16384); p.add_argument("--num-predict",type=int,default=1024); print(json.dumps(run(p.parse_args()),ensure_ascii=False,indent=2)); return 0
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"schema_version":"experimental-target-resolution-v1-diagnostic",
|
||||
"cases":[
|
||||
{"case_id":"TR1-V1","strategy":"self_contained","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Mit Dr. Schlummer arbeiten wir nicht weiter.","speaker":"Martin","named_person":"Dr. Schlummer","addressee":null}],"negative_act":{"observation_id":"obs_1","negative_act_form":"explicit_non_pursuit","normalized_action_text":"working with Dr. Schlummer"},"expected":{"target_observation_id":"obs_1","concepts":[["Zusammenarbeit","arbeiten"],["Schlummer"],["fortsetzen","weiter"]],"material_concepts":[],"forbidden_concepts":["nicht"]}},
|
||||
{"case_id":"TR2-V1","strategy":"paired","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Eine Möglichkeit wäre, die externe Lösung weiterzuverfolgen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Das verfolgen wir nicht weiter.","speaker":"Martin","named_person":null,"addressee":null}],"negative_act":{"observation_id":"obs_2","negative_act_form":"explicit_non_pursuit","normalized_action_text":"verfolgen wir nicht weiter"},"expected":{"target_observation_id":"obs_1","concepts":[["externe Lösung"],["weiterverfolgen","weiter verfolgen"]],"material_concepts":[],"forbidden_concepts":[]}},
|
||||
{"case_id":"TR3-V1","strategy":"paired","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Für den Druckversuch steht die reale Anlage zur Diskussion.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Die reale Anlage nutzen wir dafür nicht.","speaker":"Martin","named_person":null,"addressee":null}],"negative_act":{"observation_id":"obs_2","negative_act_form":"explicit_non_pursuit","normalized_action_text":"reale Anlage dafür nicht nutzen"},"expected":{"target_observation_id":"obs_1","concepts":[["Anlage"],["nutzen"]],"material_concepts":[["real"],["Druckversuch"]],"forbidden_concepts":[]}},
|
||||
{"case_id":"TR4-V1","strategy":"paired","observations":[{"observation_id":"obs_1","evidence_id":"e1","content":"Martin: Wir könnten den Versuch in der realen Anlage durchführen.","speaker":"Martin","named_person":null,"addressee":null},{"observation_id":"obs_2","evidence_id":"e2","content":"Martin: Das machen wir nicht; wir testen stattdessen im Technikum.","speaker":"Martin","named_person":null,"addressee":null}],"negative_act":{"observation_id":"obs_2","negative_act_form":"explicit_non_pursuit","normalized_action_text":"Versuch in der realen Anlage nicht durchführen"},"expected":{"target_observation_id":"obs_1","concepts":[["Versuch"],["durchführen"]],"material_concepts":[["real"],["Anlage"]],"forbidden_concepts":["Technikum"]}}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import argparse,copy,json,tempfile,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
import src.meeting_lab.controlled_semantic_derivation.experiment_target_resolution_v1 as module
|
||||
from src.meeting_lab.controlled_semantic_derivation.experiment_h import DerivationValidationError
|
||||
|
||||
CASES=module.load_cases(Path("tests/gold/target_resolution_v1/cases.json")); BY={c["case_id"]:c for c in CASES}
|
||||
def self_output(text="Zusammenarbeit mit Dr. Schlummer fortsetzen"): return {"candidate_observation_id":"obs_1","normalized_target_text":text}
|
||||
def paired(case,target="obs_1",text="externe Lösung weiterverfolgen"): return {"candidate_observation_id":case["negative_act"]["observation_id"],"target_observation_id":target,"normalized_target_text":text}
|
||||
|
||||
class TargetResolutionV1Tests(unittest.TestCase):
|
||||
def test_self_linkage_is_deterministic_and_equals_candidate(self):
|
||||
result=module.deterministic_self_link(BY["TR1-V1"]); self.assertEqual(result["linkage_source"],"deterministic"); self.assertEqual(result["target_observation_id"],result["candidate_observation_id"])
|
||||
def test_self_llm_output_has_no_target_id(self):
|
||||
self.assertEqual(module.SELF_KEYS,{"candidate_observation_id","normalized_target_text"}); self.assertNotIn("target_observation_id",module.output_schema(BY["TR1-V1"])["properties"])
|
||||
def test_self_combination_records_link_and_normalization_separately(self):
|
||||
combined=module.combine(BY["TR1-V1"],self_output()); self.assertEqual(combined["target_observation_id"],"obs_1"); self.assertIn("fortsetzen",combined["normalized_target_text"])
|
||||
def test_self_link_rejects_paired_strategy(self):
|
||||
with self.assertRaises(DerivationValidationError): module.deterministic_self_link(BY["TR2-V1"])
|
||||
def test_paired_schema_enumerates_allowed_ids_and_null(self):
|
||||
schema=module.output_schema(BY["TR2-V1"]); self.assertEqual(schema["properties"]["target_observation_id"]["enum"],["obs_1","obs_2",None]); self.assertFalse(schema["additionalProperties"])
|
||||
def test_allowed_obs1_and_obs2_are_structurally_accepted(self):
|
||||
case=BY["TR2-V1"]
|
||||
module.validate_semantic_output(paired(case,"obs_1"),case); module.validate_semantic_output(paired(case,"obs_2"),case)
|
||||
def test_unknown_and_string_null_targets_rejected(self):
|
||||
case=BY["TR2-V1"]
|
||||
for value in ("obs_9","null"):
|
||||
with self.assertRaises(DerivationValidationError): module.validate_semantic_output(paired(case,value),case)
|
||||
def test_json_null_accepted_and_requires_null_text(self):
|
||||
case=BY["TR2-V1"]; valid=paired(case,None,None); self.assertEqual(module.validate_semantic_output(valid,case),valid)
|
||||
with self.assertRaises(DerivationValidationError): module.validate_semantic_output(paired(case,None,"x"),case)
|
||||
def test_non_null_requires_nonempty_text(self):
|
||||
with self.assertRaises(DerivationValidationError): module.validate_semantic_output(paired(BY["TR2-V1"],"obs_1",""),BY["TR2-V1"])
|
||||
def test_target_after_candidate_rejected(self):
|
||||
case=copy.deepcopy(BY["TR2-V1"]); case["negative_act"]["observation_id"]="obs_1"
|
||||
with self.assertRaises(DerivationValidationError): module.validate_semantic_output({"candidate_observation_id":"obs_1","target_observation_id":"obs_2","normalized_target_text":"x"},case)
|
||||
def test_duplicate_ids_and_evidence_rejected(self):
|
||||
for field in ("observation_id","evidence_id"):
|
||||
obs=copy.deepcopy(BY["TR2-V1"]["observations"]); obs[1][field]=obs[0][field]
|
||||
with self.assertRaises(DerivationValidationError): module.validate_observations(obs)
|
||||
def test_forbidden_and_unknown_fields_rejected(self):
|
||||
case=BY["TR2-V1"]
|
||||
for extra in ({"status":"x"},{"nested":{"decision":True}},{"extra":1}):
|
||||
data=paired(case); data.update(extra)
|
||||
with self.assertRaises(DerivationValidationError): module.validate_semantic_output(data,case)
|
||||
def test_payload_uses_true_schema_object(self):
|
||||
schema=module.output_schema(BY["TR2-V1"]); payload=module.build_payload("qwen3.5:9B","p",schema,16384,1024); self.assertIs(payload["format"],schema); self.assertIsInstance(payload["format"],dict); self.assertEqual(payload["options"]["temperature"],0)
|
||||
def test_prompt_has_typed_examples_and_allowed_ids(self):
|
||||
prompt=module.build_prompt(BY["TR2-V1"]); self.assertIn('["obs_1", "obs_2"]',prompt); self.assertIn('"target_observation_id":null',prompt); self.assertIn('Never return the string "null"',prompt); self.assertNotIn('observation ID or null',prompt)
|
||||
def test_self_normalization_must_be_positive(self):
|
||||
case=BY["TR1-V1"]; semantic=self_output("Mit Dr. Schlummer arbeiten wir nicht weiter."); combined=module.combine(case,semantic); self.assertEqual(module.evaluate(case,semantic,combined)["classification"],"FAIL")
|
||||
def test_no_rejection_derivation_exists(self):
|
||||
self.assertFalse(hasattr(module,"derive")); self.assertNotIn("status",module.PAIRED_KEYS); self.assertNotIn("explicitly_rejected",module.PAIRED_KEYS)
|
||||
def test_runner_artifacts_distinguish_linkage_and_normalization(self):
|
||||
case=BY["TR1-V1"]; fixture={"schema_version":module.SCHEMA_VERSION,"cases":[case]}; caller=Mock(return_value=(json.dumps(self_output()),{"model":"qwen3.5:9B"}))
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp); path=root/"cases.json"; path.write_text(json.dumps(fixture)); out=root/"out"; args=argparse.Namespace(cases=path,output=out,endpoint="x",model="qwen3.5:9B",timeout=1,num_ctx=16384,num_predict=1024); summary=module.run(args,caller); self.assertEqual(summary["llm_call_count"],1); self.assertTrue((out/"tr1-v1"/"deterministic_linkage_result.json").exists()); self.assertTrue((out/"tr1-v1"/"normalized_target_result.json").exists())
|
||||
|
||||
if __name__=="__main__": unittest.main()
|
||||
Reference in New Issue
Block a user