Document target normalization V0 experiment
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Target Normalization V0: reconstruct target text with fixed linkage."""
|
||||
from __future__ import annotations
|
||||
import argparse,json,time
|
||||
from pathlib import Path
|
||||
from typing import Any,Callable
|
||||
import requests
|
||||
from .experiment_h import DEFAULT_ENDPOINT,DEFAULT_MODEL,DerivationValidationError,OBSERVATION_KEYS
|
||||
|
||||
SCHEMA_VERSION="experimental-target-normalization-v0"
|
||||
OUTPUT_KEYS={"candidate_observation_id","target_observation_id","normalized_target_text"}
|
||||
LINK_KEYS={"candidate_observation_id","target_observation_id"}
|
||||
FORBIDDEN={"negative_act_form","rejection_form","explicitly_rejected","status","decision","outcome","topic_status","responsible_person","responsibility","owner","requested_actor","action_item","protocol_category","confidence","relation","relations","graph","unresolved_issue"}
|
||||
PROMPT="""The candidate and target observation IDs below are already resolved. Copy both IDs exactly; do not perform target selection. Reconstruct only the concrete POSITIVE action or option meaning targeted by the negative act. Remove rejection and negation polarity while preserving the underlying positive action. Preserve German source language, material qualifiers, purpose, location, named people, and continuation. Exclude separate positive alternatives. Do not summarize the discussion or infer rejection, decision, outcome, status, responsibility, ownership, protocol relevance, confidence, relations, graphs, or topic state. Return only the JSON-Schema-conforming object; null is not permitted.\n\nExample A observations: [{{"observation_id":"obs_a","content":"Mit Frau Beispiel arbeiten wir nicht weiter."}}]\nFixed IDs: candidate=obs_a, target=obs_a\nOutput: {{"candidate_observation_id":"obs_a","target_observation_id":"obs_a","normalized_target_text":"Zusammenarbeit mit Frau Beispiel fortsetzen"}}\n\nExample B observations: [{{"observation_id":"obs_a","content":"Für den Druckversuch steht die reale Anlage zur Diskussion."}},{{"observation_id":"obs_b","content":"Die reale Anlage nutzen wir dafür nicht."}}]\nFixed IDs: candidate=obs_b, target=obs_a\nOutput: {{"candidate_observation_id":"obs_b","target_observation_id":"obs_a","normalized_target_text":"reale Anlage für den Druckversuch nutzen"}}\n\nFixed candidate_observation_id: {candidate}\nFixed target_observation_id: {target}\nV3-style observations:\n{observations}"""
|
||||
|
||||
def _keys(value,required,where):
|
||||
if not isinstance(value,dict): raise DerivationValidationError(f"{where} must be an object")
|
||||
if set(value)!=required: raise DerivationValidationError(f"{where} keys invalid: missing={sorted(required-set(value))}, unknown={sorted(set(value)-required)}")
|
||||
def _text(value,where):
|
||||
if not isinstance(value,str) or not value.strip(): raise DerivationValidationError(f"{where} must be non-empty")
|
||||
return value.strip()
|
||||
def _reject_forbidden(value,where="output"):
|
||||
if isinstance(value,dict):
|
||||
bad=FORBIDDEN & set(value)
|
||||
if bad: raise DerivationValidationError(f"{where} contains forbidden fields: {sorted(bad)}")
|
||||
for key,item in value.items(): _reject_forbidden(item,f"{where}.{key}")
|
||||
elif isinstance(value,list):
|
||||
for index,item in enumerate(value): _reject_forbidden(item,f"{where}[{index}]")
|
||||
def validate_observations(observations):
|
||||
if not isinstance(observations,list) or not observations: raise DerivationValidationError("observations must be non-empty")
|
||||
ids=set(); evidence=set()
|
||||
for index,item in enumerate(observations):
|
||||
_keys(item,OBSERVATION_KEYS,f"observations[{index}]"); oid=_text(item["observation_id"],"observation_id"); eid=_text(item["evidence_id"],"evidence_id")
|
||||
if oid in ids or eid in evidence: raise DerivationValidationError("observation/evidence provenance must be unique")
|
||||
ids.add(oid); evidence.add(eid); _text(item["content"],"content"); _text(item["speaker"],"speaker")
|
||||
return ids
|
||||
def validate_linkage(case):
|
||||
ids=validate_observations(case["observations"]); linkage=case["fixed_linkage"]; _keys(linkage,LINK_KEYS,"fixed_linkage")
|
||||
for field in LINK_KEYS:
|
||||
if _text(linkage[field],field) not in ids: raise DerivationValidationError(f"{field} is unknown")
|
||||
return linkage
|
||||
def output_schema(case):
|
||||
link=validate_linkage(case)
|
||||
return {"type":"object","additionalProperties":False,"required":["candidate_observation_id","target_observation_id","normalized_target_text"],"properties":{"candidate_observation_id":{"const":link["candidate_observation_id"]},"target_observation_id":{"const":link["target_observation_id"]},"normalized_target_text":{"type":"string","minLength":1}}}
|
||||
def build_prompt(case):
|
||||
link=validate_linkage(case)
|
||||
return PROMPT.format(candidate=link["candidate_observation_id"],target=link["target_observation_id"],observations=json.dumps(case["observations"],ensure_ascii=False,indent=2))
|
||||
def validate_output(data,case):
|
||||
_reject_forbidden(data); _keys(data,OUTPUT_KEYS,"output"); link=validate_linkage(case)
|
||||
if data["candidate_observation_id"]!=link["candidate_observation_id"]: raise DerivationValidationError("candidate ID changed")
|
||||
if data["target_observation_id"]!=link["target_observation_id"]: raise DerivationValidationError("target ID changed")
|
||||
_text(data["normalized_target_text"],"normalized_target_text"); return data
|
||||
def build_payload(model,prompt,schema,num_ctx,num_predict): return {"model":model,"prompt":prompt,"think":False,"stream":False,"format":schema,"options":{"temperature":0,"num_ctx":num_ctx,"num_predict":num_predict}}
|
||||
def call_schema(endpoint,model,prompt,schema,timeout,num_ctx,num_predict):
|
||||
started=time.perf_counter(); response=requests.post(endpoint,json=build_payload(model,prompt,schema,num_ctx,num_predict),timeout=timeout); elapsed=time.perf_counter()-started; response.raise_for_status(); body=response.json(); raw=body.get("response")
|
||||
if not isinstance(raw,str) or not raw.strip(): raise ValueError("Ollama returned no usable response")
|
||||
return raw.strip(),{"model":body.get("model",model),"elapsed_seconds":round(elapsed,3),"total_duration_ns":body.get("total_duration"),"prompt_eval_count":body.get("prompt_eval_count"),"eval_count":body.get("eval_count"),"configuration":{"temperature":0,"think":False,"format":"json_schema_object","num_ctx":num_ctx,"num_predict":num_predict,"retries":0}}
|
||||
def _concepts(text,groups):
|
||||
folded=text.casefold(); return all(any(alias.casefold() in folded for alias in group) for group in groups)
|
||||
def evaluate(case,output):
|
||||
validate_output(output,case); expected=case["expected"]; text=output["normalized_target_text"]; folded=text.casefold(); action=_concepts(text,expected["action_concepts"]); scope=_concepts(text,expected["material_concepts"]); forbidden=[x for x in expected["forbidden_concepts"] if x.casefold() in folded]; positive=not any(x in forbidden for x in ("nicht","beenden")); german=any(x.casefold() in folded for x in expected["german_markers"]); alternative=not any(x.casefold() in folded for x in ("technikum","stattdessen")); strengthening=False
|
||||
label="PASS" if positive and action and scope and german and alternative and not forbidden and not strengthening else "FAIL"
|
||||
return {"case_id":case["case_id"],"classification":label,"expected_normalized_target_text":expected["normalized_target_text"],"actual_normalized_target_text":text,"positive_polarity_correct":positive,"action_semantics_preserved":action,"material_scope_preserved":scope,"source_language_preserved":german,"separate_alternative_excluded":alternative,"forbidden_semantics_present":forbidden,"unsupported_strengthening":strengthening,"normative_leakage":False,"candidate_id_unchanged":True,"target_id_unchanged":True}
|
||||
def load_cases(path):
|
||||
data=json.loads(path.read_text(encoding="utf-8")); _keys(data,{"schema_version","cases"},"fixture")
|
||||
if data["schema_version"]!=SCHEMA_VERSION: raise DerivationValidationError("wrong schema version")
|
||||
for case in data["cases"]: validate_linkage(case)
|
||||
return data["cases"]
|
||||
def _write(path,value): path.write_text(json.dumps(value,ensure_ascii=False,indent=2)+"\n",encoding="utf-8")
|
||||
def run(args,caller:Callable=call_schema):
|
||||
cases=load_cases(args.cases); args.output.mkdir(parents=True,exist_ok=False); _write(args.output/"gold_cases.json",{"schema_version":SCHEMA_VERSION,"cases":cases}); evaluations=[]; calls=failures=0; started=time.perf_counter()
|
||||
for case in cases:
|
||||
folder=args.output/case["case_id"].lower(); folder.mkdir(); _write(folder/"v3_style_input_observations.json",case["observations"]); _write(folder/"fixed_linkage.json",case["fixed_linkage"]); schema=output_schema(case); _write(folder/"ollama_json_schema.json",schema); prompt=build_prompt(case); (folder/"prompt.txt").write_text(prompt,encoding="utf-8")
|
||||
try:
|
||||
raw,metadata=caller(args.endpoint,args.model,prompt,schema,args.timeout,args.num_ctx,args.num_predict); calls+=1; (folder/"raw_model_response.txt").write_text(raw+"\n",encoding="utf-8"); _write(folder/"ollama_metadata.json",metadata); parsed=json.loads(raw); _write(folder/"parsed_response.json",parsed); validate_output(parsed,case); _write(folder/"structural_validation.json",{"valid":True}); _write(folder/"normalized_target_result.json",{"normalized_target_text":parsed["normalized_target_text"]}); evaluation=evaluate(case,parsed)
|
||||
except Exception as exc:
|
||||
failures+=1; _write(folder/"structural_validation.json",{"valid":False,"error":str(exc)}); evaluation={"case_id":case["case_id"],"classification":"FAIL","error":str(exc),"normative_leakage":False}
|
||||
_write(folder/"evaluation.json",evaluation); evaluations.append(evaluation)
|
||||
summary={"experiment":"target_normalization_v0","model":args.model,"llm_call_count":calls,"structural_validation_failure_count":failures,"runtime_seconds":round(time.perf_counter()-started,3),"counts":{x:sum(e["classification"]==x for e in evaluations) for x in ["PASS","PARTIAL","FAIL"]},"evaluations":evaluations}; _write(args.output/"summary.json",summary); return summary
|
||||
def main():
|
||||
p=argparse.ArgumentParser(); p.add_argument("cases",type=Path); p.add_argument("-o","--output",type=Path,required=True); p.add_argument("--model",default=DEFAULT_MODEL); p.add_argument("--endpoint",default=DEFAULT_ENDPOINT); p.add_argument("--timeout",type=int,default=300); p.add_argument("--num-ctx",type=int,default=16384); p.add_argument("--num-predict",type=int,default=1024); print(json.dumps(run(p.parse_args()),ensure_ascii=False,indent=2)); return 0
|
||||
Reference in New Issue
Block a user