93 lines
10 KiB
Python
93 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Isolated local target-resolution experiment; performs no rejection derivation."""
|
|
from __future__ import annotations
|
|
import argparse, json, time
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
from .experiment_h import DEFAULT_ENDPOINT, DEFAULT_MODEL, DerivationValidationError, OBSERVATION_KEYS, call_ollama
|
|
from .experiment_negative_act import validate_classification
|
|
|
|
SCHEMA_VERSION="experimental-target-resolution-v0"
|
|
TARGET_KEYS={"candidate_observation_id","target_observation_id","normalized_target_text"}
|
|
FORBIDDEN={"negative_act_form","rejection_form","explicitly_rejected","status","decision","outcome","topic_status","closed","responsible_person","responsibility","owner","requested_actor","action_item","protocol_category","confidence","relation","relations","graph","unresolved_issue"}
|
|
PROMPT="""Resolve and normalize only the concrete action or option referred to by the candidate negative act. Candidate: {candidate}. Strategy: {instruction} Choose only a supplied observation ID. If no unique local target exists, use JSON null for both target fields. Preserve German source meaning, continuation, purpose, location, and other material scope. Do not translate Anlage as asset. Ignore separate positive alternatives. Do not output negative-act form, rejection, status, decision, outcome, responsibility, topic closure, protocol concepts, confidence, relations, or graphs. Return exactly this JSON object with no additional fields: {{"candidate_observation_id":"{candidate}","target_observation_id":"observation ID or null","normalized_target_text":"concise positive action in German or null"}}\nObservations:\n{observations}"""
|
|
|
|
def _keys(v:Any, required:set[str], where:str):
|
|
if not isinstance(v,dict): raise DerivationValidationError(f"{where} must be an object")
|
|
if set(v)!=required: raise DerivationValidationError(f"{where} keys invalid: missing={sorted(required-set(v))}, unknown={sorted(set(v)-required)}")
|
|
def _text(v:Any, where:str):
|
|
if not isinstance(v,str) or not v.strip(): raise DerivationValidationError(f"{where} must be non-empty")
|
|
return v.strip()
|
|
def _reject_forbidden(v:Any, where="output"):
|
|
if isinstance(v,dict):
|
|
bad=FORBIDDEN & set(v)
|
|
if bad: raise DerivationValidationError(f"{where} contains forbidden fields: {sorted(bad)}")
|
|
for k,x in v.items(): _reject_forbidden(x,f"{where}.{k}")
|
|
elif isinstance(v,list):
|
|
for i,x in enumerate(v): _reject_forbidden(x,f"{where}[{i}]")
|
|
def validate_observations(obs):
|
|
if not isinstance(obs,list) or not obs: raise DerivationValidationError("observations must be non-empty")
|
|
ids=set(); evidence=set()
|
|
for i,o in enumerate(obs):
|
|
_keys(o,OBSERVATION_KEYS,f"observations[{i}]"); oid=_text(o["observation_id"],"observation_id"); eid=_text(o["evidence_id"],"evidence_id")
|
|
if oid in ids or eid in evidence: raise DerivationValidationError("observation/evidence provenance must be unique")
|
|
ids.add(oid); evidence.add(eid); _text(o["content"],"content"); _text(o["speaker"],"speaker")
|
|
return ids
|
|
def eligibility(negative,obs):
|
|
validate_classification(negative,obs)
|
|
eligible=negative["negative_act_form"]=="explicit_non_pursuit"
|
|
return {"eligible_for_target_resolution":eligible,"reason":None if eligible else "negative_act_form_not_explicit_non_pursuit"}
|
|
def validate_target(data,obs,candidate):
|
|
ids=validate_observations(obs); _reject_forbidden(data); _keys(data,TARGET_KEYS,"target output")
|
|
if _text(data["candidate_observation_id"],"candidate_observation_id")!=candidate: raise DerivationValidationError("candidate observation mismatch")
|
|
if candidate not in ids: raise DerivationValidationError("unknown candidate observation")
|
|
target=data["target_observation_id"]; normalized=data["normalized_target_text"]
|
|
if target is None:
|
|
if normalized is not None: raise DerivationValidationError("null target requires null text")
|
|
else:
|
|
target=_text(target,"target_observation_id")
|
|
if target not in ids: raise DerivationValidationError("unknown target observation")
|
|
if [o["observation_id"] for o in obs].index(target)>[o["observation_id"] for o in obs].index(candidate): raise DerivationValidationError("target must not occur after candidate")
|
|
_text(normalized,"normalized_target_text")
|
|
return data
|
|
def build_prompt(case):
|
|
gate=eligibility(case["negative_act"],case["observations"])
|
|
if not gate["eligible_for_target_resolution"]: raise DerivationValidationError("ineligible case must not build a target prompt")
|
|
candidate=case["negative_act"]["observation_id"]
|
|
instruction=(f"The target linkage is deterministically fixed to {candidate}; output that exact target ID and only normalize its positive action meaning." if case["strategy"]=="self_contained" else "Resolve the unique preceding local observation that supplies the referenced action.")
|
|
return PROMPT.format(candidate=candidate,instruction=instruction,observations=json.dumps(case["observations"],ensure_ascii=False,indent=2))
|
|
def _concepts(text,groups):
|
|
folded=(text or "").casefold(); return all(any(alias.casefold() in folded for alias in group) for group in groups)
|
|
def evaluate(case,gate,called,target):
|
|
e=case["expected"]; eligible=gate["eligible_for_target_resolution"]==e["eligible"]; call_ok=called==e["eligible"]
|
|
if not e["eligible"]:
|
|
label="PASS" if eligible and call_ok and target is None else "FAIL"
|
|
return {"case_id":case["case_id"],"classification":label,"negative_act_form":case["negative_act"]["negative_act_form"],"eligible":gate["eligible_for_target_resolution"],"target_resolution_call_made":called,"expected_target_observation_id":None,"actual_target_observation_id":None,"normalized_target_text":None,"material_scope_preserved":True,"alternative_isolation":True,"normative_leakage":False}
|
|
text=target["normalized_target_text"]; target_ok=target["target_observation_id"]==e["target_observation_id"]; concepts=_concepts(text,e["concepts"]); material=_concepts(text,e["material_concepts"]); isolated=not any(x.casefold() in (text or "").casefold() for x in e["forbidden_concepts"])
|
|
label="PASS" if eligible and call_ok and target_ok and concepts and material and isolated else ("PARTIAL" if eligible and call_ok and target_ok and material and isolated else "FAIL")
|
|
return {"case_id":case["case_id"],"classification":label,"negative_act_form":case["negative_act"]["negative_act_form"],"eligible":gate["eligible_for_target_resolution"],"target_resolution_call_made":called,"expected_target_observation_id":e["target_observation_id"],"actual_target_observation_id":target["target_observation_id"],"normalized_target_text":text,"normalized_action_correct":concepts,"material_scope_preserved":material,"alternative_isolation":isolated,"normative_leakage":False}
|
|
def load_cases(path):
|
|
data=json.loads(path.read_text(encoding="utf-8")); _keys(data,{"schema_version","cases"},"fixture")
|
|
if data["schema_version"]!=SCHEMA_VERSION: raise DerivationValidationError("unexpected schema version")
|
|
for case in data["cases"]: validate_observations(case["observations"]); validate_classification(case["negative_act"],case["observations"])
|
|
return data["cases"]
|
|
def _write(path,value): path.write_text(json.dumps(value,ensure_ascii=False,indent=2)+"\n",encoding="utf-8")
|
|
def run(args, resolver:Callable=call_ollama):
|
|
cases=load_cases(args.cases); args.output.mkdir(parents=True,exist_ok=False); _write(args.output/"gold_cases.json",{"schema_version":SCHEMA_VERSION,"cases":cases})
|
|
evaluations=[]; calls=technical_failures=structural_failures=0; started=time.perf_counter()
|
|
for case in cases:
|
|
folder=args.output/case["case_id"].lower(); folder.mkdir(); _write(folder/"v3_style_input_observations.json",case["observations"]); _write(folder/"negative_act_form.json",case["negative_act"])
|
|
gate=eligibility(case["negative_act"],case["observations"]); _write(folder/"eligibility.json",gate)
|
|
if not gate["eligible_for_target_resolution"]:
|
|
skipped={"call_made":False,"reason":gate["reason"]}; _write(folder/"target_resolution_skipped.json",skipped); ev=evaluate(case,gate,False,None)
|
|
else:
|
|
prompt=build_prompt(case); (folder/"prompt.txt").write_text(prompt,encoding="utf-8")
|
|
try:
|
|
raw,meta=resolver(args.endpoint,args.model,prompt,args.timeout,args.num_ctx,args.num_predict); calls+=1; (folder/"raw_model_response.txt").write_text(raw+"\n",encoding="utf-8"); _write(folder/"ollama_metadata.json",meta); parsed=json.loads(raw); _write(folder/"parsed_target_resolution.json",parsed); validate_target(parsed,case["observations"],case["negative_act"]["observation_id"]); _write(folder/"structural_validation.json",{"valid":True}); ev=evaluate(case,gate,True,parsed)
|
|
except Exception as exc:
|
|
structural_failures+=1; _write(folder/"structural_validation.json",{"valid":False,"error":str(exc)}); ev={"case_id":case["case_id"],"classification":"FAIL","negative_act_form":case["negative_act"]["negative_act_form"],"eligible":True,"target_resolution_call_made":True,"error":str(exc)}
|
|
_write(folder/"evaluation.json",ev); evaluations.append(ev)
|
|
summary={"experiment":"target_resolution_v0","model":args.model,"target_resolution_llm_call_count":calls,"technical_failed_call_count":technical_failures,"structural_validation_failure_count":structural_failures,"runtime_seconds":round(time.perf_counter()-started,3),"counts":{x:sum(e["classification"]==x for e in evaluations) for x in ["PASS","PARTIAL","FAIL"]},"evaluations":evaluations}; _write(args.output/"summary.json",summary); return summary
|
|
def main():
|
|
p=argparse.ArgumentParser(); p.add_argument("cases",type=Path); p.add_argument("-o","--output",type=Path,required=True); p.add_argument("--model",default=DEFAULT_MODEL); p.add_argument("--endpoint",default=DEFAULT_ENDPOINT); p.add_argument("--timeout",type=int,default=300); p.add_argument("--num-ctx",type=int,default=16384); p.add_argument("--num-predict",type=int,default=1024); print(json.dumps(run(p.parse_args()),ensure_ascii=False,indent=2)); return 0
|