Document target resolution V1 diagnostic
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Target Resolution V1 diagnostic: linkage and normalization only."""
|
||||
from __future__ import annotations
|
||||
import argparse,json,time
|
||||
from pathlib import Path
|
||||
from typing import Any,Callable
|
||||
import requests
|
||||
from .experiment_h import DEFAULT_ENDPOINT,DEFAULT_MODEL,DerivationValidationError,OBSERVATION_KEYS
|
||||
from .experiment_negative_act import validate_classification
|
||||
|
||||
SCHEMA_VERSION="experimental-target-resolution-v1-diagnostic"
|
||||
SELF_KEYS={"candidate_observation_id","normalized_target_text"}; PAIRED_KEYS={"candidate_observation_id","target_observation_id","normalized_target_text"}
|
||||
FORBIDDEN={"negative_act_form","rejection_form","explicitly_rejected","status","decision","outcome","responsible_person","responsibility","owner","requested_actor","action_item","protocol_category","confidence","relation","relations","graph","topic_status","closed","unresolved_issue"}
|
||||
SELF_PROMPT="""Normalize only the concrete positive action meaning in the self-contained candidate observation. The target linkage is already deterministic and is not your task. Preserve German, collaboration, named people, and continuation meaning. Do not output a target ID, rejection, status, decision, outcome, responsibility, ownership, protocol concepts, confidence, relations, graphs, or topic closure. Return only the schema-conforming object.\nCandidate observation ID: {candidate}\nObservation:\n{observations}"""
|
||||
PAIRED_PROMPT="""Resolve and normalize only the concrete local action or option referred to by the candidate negative act. Choose exactly one listed allowed target observation ID, or use JSON null only when no unique local target exists. Never return the string \"null\". Preserve German source language and all material purpose/location scope. Do not absorb a separate positive alternative. Do not output negative-act form, rejection, status, decision, outcome, responsibility, ownership, protocol concepts, confidence, relations, graphs, or topic closure.\nAllowed target observation IDs:\n{allowed}\nConcrete positive typed example:\n{{"candidate_observation_id":"obs_2","target_observation_id":"obs_1","normalized_target_text":"externe Lösung weiterverfolgen"}}\nActual JSON-null example:\n{{"candidate_observation_id":"obs_2","target_observation_id":null,"normalized_target_text":null}}\nReturn only the schema-conforming object.\nCandidate observation ID: {candidate}\nObservations:\n{observations}"""
|
||||
|
||||
def _keys(value,required,where):
|
||||
if not isinstance(value,dict): raise DerivationValidationError(f"{where} must be an object")
|
||||
if set(value)!=required: raise DerivationValidationError(f"{where} keys invalid: missing={sorted(required-set(value))}, unknown={sorted(set(value)-required)}")
|
||||
def _text(value,where):
|
||||
if not isinstance(value,str) or not value.strip(): raise DerivationValidationError(f"{where} must be non-empty")
|
||||
return value.strip()
|
||||
def _forbidden(value,where="output"):
|
||||
if isinstance(value,dict):
|
||||
bad=FORBIDDEN & set(value)
|
||||
if bad: raise DerivationValidationError(f"{where} contains forbidden fields: {sorted(bad)}")
|
||||
for key,item in value.items(): _forbidden(item,f"{where}.{key}")
|
||||
elif isinstance(value,list):
|
||||
for index,item in enumerate(value): _forbidden(item,f"{where}[{index}]")
|
||||
def validate_observations(obs):
|
||||
if not isinstance(obs,list) or not obs: raise DerivationValidationError("observations must be non-empty")
|
||||
ids=[]; evidence=set()
|
||||
for index,item in enumerate(obs):
|
||||
_keys(item,OBSERVATION_KEYS,f"observations[{index}]"); oid=_text(item["observation_id"],"observation_id"); eid=_text(item["evidence_id"],"evidence_id")
|
||||
if oid in ids or eid in evidence: raise DerivationValidationError("observation/evidence provenance must be unique")
|
||||
ids.append(oid); evidence.add(eid); _text(item["content"],"content"); _text(item["speaker"],"speaker")
|
||||
return ids
|
||||
def allowed_ids(case): return validate_observations(case["observations"])
|
||||
def deterministic_self_link(case):
|
||||
if case["strategy"]!="self_contained": raise DerivationValidationError("self-linkage requires self-contained strategy")
|
||||
ids=validate_observations(case["observations"]); candidate=case["negative_act"]["observation_id"]
|
||||
if candidate not in ids: raise DerivationValidationError("unknown candidate")
|
||||
return {"linkage_source":"deterministic","candidate_observation_id":candidate,"target_observation_id":candidate}
|
||||
def output_schema(case):
|
||||
candidate=case["negative_act"]["observation_id"]
|
||||
if case["strategy"]=="self_contained":
|
||||
return {"type":"object","additionalProperties":False,"required":["candidate_observation_id","normalized_target_text"],"properties":{"candidate_observation_id":{"const":candidate},"normalized_target_text":{"type":"string","minLength":1}}}
|
||||
ids=allowed_ids(case)
|
||||
return {"type":"object","additionalProperties":False,"required":["candidate_observation_id","target_observation_id","normalized_target_text"],"properties":{"candidate_observation_id":{"const":candidate},"target_observation_id":{"enum":ids+[None]},"normalized_target_text":{"type":["string","null"]}},"allOf":[{"if":{"properties":{"target_observation_id":{"type":"null"}}},"then":{"properties":{"normalized_target_text":{"type":"null"}}},"else":{"properties":{"normalized_target_text":{"type":"string","minLength":1}}}}]}
|
||||
def build_prompt(case):
|
||||
validate_classification(case["negative_act"],case["observations"]); candidate=case["negative_act"]["observation_id"]
|
||||
if case["strategy"]=="self_contained": return SELF_PROMPT.format(candidate=candidate,observations=json.dumps(case["observations"],ensure_ascii=False,indent=2))
|
||||
return PAIRED_PROMPT.format(candidate=candidate,allowed=json.dumps(allowed_ids(case),ensure_ascii=False),observations=json.dumps(case["observations"],ensure_ascii=False,indent=2))
|
||||
def validate_semantic_output(data,case):
|
||||
_forbidden(data); candidate=case["negative_act"]["observation_id"]
|
||||
if case["strategy"]=="self_contained":
|
||||
_keys(data,SELF_KEYS,"self output")
|
||||
if data["candidate_observation_id"]!=candidate: raise DerivationValidationError("candidate mismatch")
|
||||
_text(data["normalized_target_text"],"normalized_target_text")
|
||||
else:
|
||||
_keys(data,PAIRED_KEYS,"paired output")
|
||||
if data["candidate_observation_id"]!=candidate: raise DerivationValidationError("candidate mismatch")
|
||||
target=data["target_observation_id"]
|
||||
if target=="null": raise DerivationValidationError('string "null" is forbidden')
|
||||
if target is None:
|
||||
if data["normalized_target_text"] is not None: raise DerivationValidationError("null target requires null text")
|
||||
else:
|
||||
if target not in allowed_ids(case): raise DerivationValidationError("target is not an allowed ID")
|
||||
ids=allowed_ids(case)
|
||||
if ids.index(target)>ids.index(candidate): raise DerivationValidationError("target must not occur after candidate")
|
||||
_text(data["normalized_target_text"],"normalized_target_text")
|
||||
return data
|
||||
def combine(case,semantic):
|
||||
validate_semantic_output(semantic,case)
|
||||
if case["strategy"]=="self_contained":
|
||||
link=deterministic_self_link(case); return {**link,"normalized_target_text":semantic["normalized_target_text"]}
|
||||
return {"linkage_source":"llm","candidate_observation_id":semantic["candidate_observation_id"],"target_observation_id":semantic["target_observation_id"],"normalized_target_text":semantic["normalized_target_text"]}
|
||||
def build_payload(model,prompt,schema,num_ctx,num_predict):
|
||||
return {"model":model,"prompt":prompt,"think":False,"stream":False,"format":schema,"options":{"temperature":0,"num_ctx":num_ctx,"num_predict":num_predict}}
|
||||
def call_schema(endpoint,model,prompt,schema,timeout,num_ctx,num_predict):
|
||||
started=time.perf_counter(); response=requests.post(endpoint,json=build_payload(model,prompt,schema,num_ctx,num_predict),timeout=timeout); elapsed=time.perf_counter()-started; response.raise_for_status(); body=response.json(); raw=body.get("response")
|
||||
if not isinstance(raw,str) or not raw.strip(): raise ValueError("Ollama returned no usable response")
|
||||
meta={"model":body.get("model",model),"elapsed_seconds":round(elapsed,3),"total_duration_ns":body.get("total_duration"),"prompt_eval_count":body.get("prompt_eval_count"),"eval_count":body.get("eval_count"),"configuration":{"temperature":0,"think":False,"format":"json_schema_object","num_ctx":num_ctx,"num_predict":num_predict,"retries":0}}
|
||||
return raw.strip(),meta
|
||||
def _concepts(text,groups):
|
||||
folded=(text or "").casefold(); return all(any(x.casefold() in folded for x in group) for group in groups)
|
||||
def evaluate(case,semantic,combined):
|
||||
expected=case["expected"]; text=combined["normalized_target_text"]; target=combined["target_observation_id"]; concepts=_concepts(text,expected["concepts"]); material=_concepts(text,expected["material_concepts"]); isolated=not any(x.casefold() in (text or "").casefold() for x in expected["forbidden_concepts"]); recurrence=semantic.get("target_observation_id")=="null"; correct=target==expected["target_observation_id"]
|
||||
label="PASS" if correct and concepts and material and isolated and not recurrence else ("PARTIAL" if correct and material and isolated and not recurrence else "FAIL")
|
||||
return {"case_id":case["case_id"],"classification":label,"strategy":case["strategy"],"target_id_decision_source":combined["linkage_source"],"expected_target_observation_id":expected["target_observation_id"],"actual_target_observation_id":target,"normalized_target_text":text,"continuation_or_action_preserved":concepts,"material_scope_preserved":material,"alternative_isolated":isolated,"schema_valid":True,"string_null_recurrence":recurrence,"normative_leakage":False}
|
||||
def load_cases(path):
|
||||
data=json.loads(path.read_text(encoding="utf-8")); _keys(data,{"schema_version","cases"},"fixture")
|
||||
if data["schema_version"]!=SCHEMA_VERSION: raise DerivationValidationError("wrong schema version")
|
||||
return data["cases"]
|
||||
def _write(path,value): path.write_text(json.dumps(value,ensure_ascii=False,indent=2)+"\n",encoding="utf-8")
|
||||
def run(args,caller:Callable=call_schema):
|
||||
cases=load_cases(args.cases); args.output.mkdir(parents=True,exist_ok=False); _write(args.output/"gold_cases.json",{"schema_version":SCHEMA_VERSION,"cases":cases}); evaluations=[]; calls=failures=0; started=time.perf_counter()
|
||||
for case in cases:
|
||||
folder=args.output/case["case_id"].lower(); folder.mkdir(); _write(folder/"v3_style_input_observations.json",case["observations"]); _write(folder/"negative_act_form.json",case["negative_act"]); _write(folder/"eligibility.json",{"eligible_for_target_resolution":True,"reason":None}); _write(folder/"deterministic_strategy.json",{"strategy":case["strategy"],"target_id_decision_source":"deterministic" if case["strategy"]=="self_contained" else "llm"}); _write(folder/"allowed_target_ids.json",allowed_ids(case)); schema=output_schema(case); _write(folder/"ollama_json_schema.json",schema); prompt=build_prompt(case); (folder/"prompt.txt").write_text(prompt,encoding="utf-8")
|
||||
try:
|
||||
raw,meta=caller(args.endpoint,args.model,prompt,schema,args.timeout,args.num_ctx,args.num_predict); calls+=1; (folder/"raw_model_response.txt").write_text(raw+"\n",encoding="utf-8"); _write(folder/"ollama_metadata.json",meta); semantic=json.loads(raw); _write(folder/"parsed_semantic_output.json",semantic); validate_semantic_output(semantic,case); combined=combine(case,semantic); _write(folder/"structural_validation.json",{"valid":True}); _write(folder/"deterministic_linkage_result.json",{k:combined[k] for k in ("linkage_source","candidate_observation_id","target_observation_id")}); _write(folder/"normalized_target_result.json",{"normalized_target_text":combined["normalized_target_text"]}); evaluation=evaluate(case,semantic,combined)
|
||||
except Exception as exc:
|
||||
failures+=1; _write(folder/"structural_validation.json",{"valid":False,"error":str(exc)}); evaluation={"case_id":case["case_id"],"classification":"FAIL","strategy":case["strategy"],"schema_valid":False,"error":str(exc)}
|
||||
_write(folder/"evaluation.json",evaluation); evaluations.append(evaluation)
|
||||
summary={"experiment":"target_resolution_v1_diagnostic","model":args.model,"llm_call_count":calls,"structural_validation_failure_count":failures,"runtime_seconds":round(time.perf_counter()-started,3),"counts":{x:sum(e["classification"]==x for e in evaluations) for x in ["PASS","PARTIAL","FAIL"]},"evaluations":evaluations}; _write(args.output/"summary.json",summary); return summary
|
||||
def main():
|
||||
p=argparse.ArgumentParser(); p.add_argument("cases",type=Path); p.add_argument("-o","--output",type=Path,required=True); p.add_argument("--model",default=DEFAULT_MODEL); p.add_argument("--endpoint",default=DEFAULT_ENDPOINT); p.add_argument("--timeout",type=int,default=300); p.add_argument("--num-ctx",type=int,default=16384); p.add_argument("--num-predict",type=int,default=1024); print(json.dumps(run(p.parse_args()),ensure_ascii=False,indent=2)); return 0
|
||||
Reference in New Issue
Block a user