368 lines
18 KiB
Python
368 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Isolated request/acceptance Gold reliability experiment."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .experiment_h import (
|
|
DEFAULT_ENDPOINT,
|
|
DEFAULT_MODEL,
|
|
DerivationValidationError,
|
|
FORBIDDEN_LLM_KEYS,
|
|
OBSERVATION_KEYS,
|
|
call_ollama,
|
|
)
|
|
|
|
|
|
GOLD_SCHEMA_VERSION = "experimental-request-acceptance-gold-v0"
|
|
RECOGNITION_SCHEMA_VERSION = "experimental-request-acceptance-recognition-v0"
|
|
REQUEST_KEYS = {"observation_id", "is_concrete_request", "normalized_action_text"}
|
|
ACCEPTANCE_KEYS = {
|
|
"observation_id", "is_explicit_commitment", "same_requested_work",
|
|
"normalized_action_text",
|
|
}
|
|
WEEKDAYS = {
|
|
"monday": "Montag", "montag": "Montag", "tuesday": "Dienstag",
|
|
"dienstag": "Dienstag", "wednesday": "Mittwoch", "mittwoch": "Mittwoch",
|
|
"thursday": "Donnerstag", "donnerstag": "Donnerstag", "friday": "Freitag",
|
|
"freitag": "Freitag", "saturday": "Samstag", "samstag": "Samstag",
|
|
"sunday": "Sonntag", "sonntag": "Sonntag",
|
|
}
|
|
|
|
PROMPT_TEMPLATE = """Recognize only a concrete directed request and a later explicit personal commitment in the supplied V3-style observations.
|
|
|
|
The input is observations only, not a transcript. Identify:
|
|
1. A concrete request directed to the observation's explicit addressee, if one exists.
|
|
2. A later response that explicitly commits its speaker to work, if one exists.
|
|
3. Whether that explicit commitment concerns substantially the same requested work.
|
|
|
|
Lexical identity is not required: a contextual paraphrase may denote the same work. Mere acknowledgement, tentative or conditional language, collective "we" statements, impersonal necessity, suggestions, and statements that work should be done are not explicit personal commitments. A commitment to different work is an explicit commitment but not the same requested work.
|
|
|
|
Do not decide or output responsibility, requested actor, established status, Action Item status, protocol eligibility, confidence, semantic relations, or graphs. Do not answer who is responsible. Deterministic code applies those gates later.
|
|
|
|
Return exactly this JSON shape and no other fields. Use null for request or acceptance when no qualifying observation exists:
|
|
{{
|
|
"schema_version": "experimental-request-acceptance-recognition-v0",
|
|
"request": null | {{
|
|
"observation_id": "observation ID",
|
|
"is_concrete_request": true,
|
|
"normalized_action_text": "concise requested work"
|
|
}},
|
|
"acceptance": null | {{
|
|
"observation_id": "observation ID",
|
|
"is_explicit_commitment": true,
|
|
"same_requested_work": true,
|
|
"normalized_action_text": "concise committed work"
|
|
}}
|
|
}}
|
|
|
|
V3-style observations:
|
|
{observations_json}
|
|
"""
|
|
|
|
|
|
def _exact_keys(value: dict[str, Any], required: set[str], location: str) -> None:
|
|
missing = required - value.keys()
|
|
unknown = value.keys() - required
|
|
if missing:
|
|
raise DerivationValidationError(f"{location} missing required keys: {sorted(missing)}")
|
|
if unknown:
|
|
raise DerivationValidationError(f"{location} has unknown keys: {sorted(unknown)}")
|
|
|
|
|
|
def _nonempty_text(value: Any, location: str) -> str:
|
|
if not isinstance(value, str) or not value.strip():
|
|
raise DerivationValidationError(f"{location} must be a non-empty string")
|
|
return value.strip()
|
|
|
|
|
|
def load_gold_cases(path: Path) -> list[dict[str, Any]]:
|
|
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
if not isinstance(data, dict):
|
|
raise DerivationValidationError("Gold fixture must be an object")
|
|
_exact_keys(data, {"schema_version", "cases"}, "Gold fixture")
|
|
if data["schema_version"] != GOLD_SCHEMA_VERSION:
|
|
raise DerivationValidationError("unexpected Gold fixture schema_version")
|
|
cases = data["cases"]
|
|
if not isinstance(cases, list) or not cases:
|
|
raise DerivationValidationError("Gold fixture cases must be a non-empty list")
|
|
seen_cases: set[str] = set()
|
|
for case in cases:
|
|
_exact_keys(case, {"case_id", "description", "observations", "expected_recognition", "expected_result"}, "Gold case")
|
|
case_id = _nonempty_text(case["case_id"], "case_id")
|
|
if case_id in seen_cases:
|
|
raise DerivationValidationError(f"duplicate case ID: {case_id}")
|
|
seen_cases.add(case_id)
|
|
_validate_observations(case["observations"])
|
|
return cases
|
|
|
|
|
|
def _validate_observations(observations: Any) -> None:
|
|
if not isinstance(observations, list) or not observations:
|
|
raise DerivationValidationError("observations must be a non-empty list")
|
|
seen_ids: set[str] = set()
|
|
seen_evidence: set[str] = set()
|
|
for index, observation in enumerate(observations):
|
|
location = f"observations[{index}]"
|
|
if not isinstance(observation, dict):
|
|
raise DerivationValidationError(f"{location} must be an object")
|
|
_exact_keys(observation, OBSERVATION_KEYS, location)
|
|
observation_id = _nonempty_text(observation["observation_id"], f"{location}.observation_id")
|
|
evidence_id = _nonempty_text(observation["evidence_id"], f"{location}.evidence_id")
|
|
if observation_id in seen_ids or evidence_id in seen_evidence:
|
|
raise DerivationValidationError("observation and evidence IDs must be unique")
|
|
seen_ids.add(observation_id)
|
|
seen_evidence.add(evidence_id)
|
|
_nonempty_text(observation["content"], f"{location}.content")
|
|
_nonempty_text(observation["speaker"], f"{location}.speaker")
|
|
for field in ("named_person", "addressee"):
|
|
if observation[field] is not None:
|
|
_nonempty_text(observation[field], f"{location}.{field}")
|
|
|
|
|
|
def build_prompt(observations: list[dict[str, Any]]) -> str:
|
|
_validate_observations(observations)
|
|
return PROMPT_TEMPLATE.format(
|
|
observations_json=json.dumps(observations, ensure_ascii=False, indent=2)
|
|
)
|
|
|
|
|
|
def parse_model_json(raw_text: str) -> dict[str, Any]:
|
|
data = json.loads(raw_text)
|
|
if not isinstance(data, dict):
|
|
raise DerivationValidationError("semantic recognition must be an object")
|
|
return data
|
|
|
|
|
|
def _reject_forbidden_keys(value: Any, location: str = "output") -> None:
|
|
if isinstance(value, dict):
|
|
forbidden = FORBIDDEN_LLM_KEYS.intersection(value)
|
|
if forbidden:
|
|
raise DerivationValidationError(
|
|
f"{location} contains forbidden semantic keys: {sorted(forbidden)}"
|
|
)
|
|
for key, item in value.items():
|
|
_reject_forbidden_keys(item, f"{location}.{key}")
|
|
elif isinstance(value, list):
|
|
for index, item in enumerate(value):
|
|
_reject_forbidden_keys(item, f"{location}[{index}]")
|
|
|
|
|
|
def validate_recognition(data: Any, observations: list[dict[str, Any]]) -> dict[str, Any]:
|
|
if not isinstance(data, dict):
|
|
raise DerivationValidationError("semantic recognition must be an object")
|
|
_reject_forbidden_keys(data)
|
|
_exact_keys(data, {"schema_version", "request", "acceptance"}, "output")
|
|
if data["schema_version"] != RECOGNITION_SCHEMA_VERSION:
|
|
raise DerivationValidationError("unexpected recognition schema_version")
|
|
known_ids = {item["observation_id"] for item in observations}
|
|
request = data["request"]
|
|
acceptance = data["acceptance"]
|
|
if request is not None:
|
|
if not isinstance(request, dict):
|
|
raise DerivationValidationError("output.request must be an object or null")
|
|
_exact_keys(request, REQUEST_KEYS, "output.request")
|
|
if request["observation_id"] not in known_ids:
|
|
raise DerivationValidationError("request references unknown observation")
|
|
if not isinstance(request["is_concrete_request"], bool):
|
|
raise DerivationValidationError("is_concrete_request must be boolean")
|
|
_nonempty_text(request["normalized_action_text"], "request.normalized_action_text")
|
|
if acceptance is not None:
|
|
if not isinstance(acceptance, dict):
|
|
raise DerivationValidationError("output.acceptance must be an object or null")
|
|
_exact_keys(acceptance, ACCEPTANCE_KEYS, "output.acceptance")
|
|
if acceptance["observation_id"] not in known_ids:
|
|
raise DerivationValidationError("acceptance references unknown observation")
|
|
for field in ("is_explicit_commitment", "same_requested_work"):
|
|
if not isinstance(acceptance[field], bool):
|
|
raise DerivationValidationError(f"{field} must be boolean")
|
|
_nonempty_text(acceptance["normalized_action_text"], "acceptance.normalized_action_text")
|
|
if request is not None and acceptance is not None and request["observation_id"] == acceptance["observation_id"]:
|
|
raise DerivationValidationError("request and acceptance must reference different observations")
|
|
return data
|
|
|
|
|
|
def _bounded_due(observations: list[dict[str, Any]]) -> tuple[str | None, bool]:
|
|
forms: set[str] = set()
|
|
for observation in observations:
|
|
for token in re.findall(r"\b[A-Za-zÄÖÜäöü]+\b", observation["content"].casefold()):
|
|
if token in WEEKDAYS:
|
|
forms.add(WEEKDAYS[token])
|
|
return (next(iter(forms)) if len(forms) == 1 else None, len(forms) <= 1)
|
|
|
|
|
|
def _strip_due(action_text: str) -> str:
|
|
weekday = "|".join(re.escape(value) for value in WEEKDAYS)
|
|
result = re.sub(rf"\s+(?:bis|by)\s+(?:{weekday})\b", "", action_text, flags=re.IGNORECASE)
|
|
return result.strip(" .,:;-") or action_text.strip()
|
|
|
|
|
|
def derive_action(
|
|
observations: list[dict[str, Any]], recognition: dict[str, Any]
|
|
) -> tuple[dict[str, bool], dict[str, Any] | None]:
|
|
_validate_observations(observations)
|
|
validate_recognition(recognition, observations)
|
|
by_id = {item["observation_id"]: item for item in observations}
|
|
positions = {item["observation_id"]: index for index, item in enumerate(observations)}
|
|
request_semantic = recognition["request"]
|
|
acceptance_semantic = recognition["acceptance"]
|
|
request = by_id.get(request_semantic["observation_id"]) if request_semantic else None
|
|
acceptance = by_id.get(acceptance_semantic["observation_id"]) if acceptance_semantic else None
|
|
due, deadline_consistent = _bounded_due(observations)
|
|
gates = {
|
|
"request_semantic_positive": request_semantic is not None and request_semantic["is_concrete_request"] is True,
|
|
"request_observation_exists": request is not None,
|
|
"request_has_addressee": request is not None and isinstance(request["addressee"], str) and bool(request["addressee"].strip()),
|
|
"acceptance_semantic_positive": acceptance_semantic is not None and acceptance_semantic["is_explicit_commitment"] is True,
|
|
"same_requested_work": acceptance_semantic is not None and acceptance_semantic["same_requested_work"] is True,
|
|
"acceptance_observation_exists": acceptance is not None,
|
|
"acceptance_after_request": request is not None and acceptance is not None and positions[acceptance["observation_id"]] > positions[request["observation_id"]],
|
|
"acceptance_speaker_matches_addressee": request is not None and acceptance is not None and acceptance["speaker"] == request["addressee"],
|
|
"provenance_valid_and_consistent": request is not None and acceptance is not None and request["evidence_id"] in {item["evidence_id"] for item in observations} and acceptance["evidence_id"] in {item["evidence_id"] for item in observations},
|
|
"deadline_consistent": deadline_consistent,
|
|
}
|
|
if not all(gates.values()):
|
|
return gates, None
|
|
return gates, {
|
|
"action_id": "action_1",
|
|
"content": _strip_due(request_semantic["normalized_action_text"]),
|
|
"status": "established",
|
|
"requested_actor": request["addressee"],
|
|
"responsible_person": acceptance["speaker"],
|
|
"due": due,
|
|
"support": {
|
|
"request": {"observation_id": request["observation_id"], "evidence_id": request["evidence_id"]},
|
|
"acceptance": {"observation_id": acceptance["observation_id"], "evidence_id": acceptance["evidence_id"]},
|
|
},
|
|
}
|
|
|
|
|
|
def evaluate_case(case: dict[str, Any], recognition: dict[str, Any]) -> dict[str, Any]:
|
|
observations = case["observations"]
|
|
validate_recognition(recognition, observations)
|
|
gates, result = derive_action(observations, recognition)
|
|
expected_recognition = case["expected_recognition"]
|
|
request_correct = (recognition["request"] is not None and recognition["request"]["is_concrete_request"]) == expected_recognition["request"]
|
|
commitment_correct = (recognition["acceptance"] is not None and recognition["acceptance"]["is_explicit_commitment"]) == expected_recognition["commitment"]
|
|
same_work_correct = (recognition["acceptance"] is not None and recognition["acceptance"]["same_requested_work"]) == expected_recognition["same_work"]
|
|
expected = case["expected_result"]
|
|
actual_established = result is not None
|
|
final_correct = actual_established == expected["established"]
|
|
if result is not None:
|
|
final_correct = final_correct and all(
|
|
result[key] == expected[key]
|
|
for key in ("requested_actor", "responsible_person", "due")
|
|
)
|
|
else:
|
|
final_correct = final_correct and expected["responsible_person"] is None
|
|
semantic_correct = request_correct and commitment_correct and same_work_correct
|
|
classification = "PASS" if final_correct and semantic_correct else ("PARTIAL" if final_correct else "FAIL")
|
|
return {
|
|
"case_id": case["case_id"], "classification": classification,
|
|
"request_correct": request_correct, "commitment_correct": commitment_correct,
|
|
"same_work_correct": same_work_correct, "deterministic_gates_correct": final_correct,
|
|
"final_result_correct": final_correct, "result": result, "gates": gates,
|
|
"unsupported_semantic_strengthening": not semantic_correct and actual_established,
|
|
"responsibility_status_leakage": False,
|
|
}
|
|
|
|
|
|
def reevaluate_existing(args: argparse.Namespace) -> dict[str, Any]:
|
|
cases = load_gold_cases(args.cases)
|
|
evaluations = []
|
|
for case in cases:
|
|
case_dir = args.output / case["case_id"].lower()
|
|
parsed = json.loads(
|
|
(case_dir / "parsed_semantic_recognition.json").read_text(encoding="utf-8")
|
|
)
|
|
evaluation = evaluate_case(case, parsed)
|
|
_write_json(case_dir / "evaluation.json", evaluation)
|
|
evaluations.append(evaluation)
|
|
metadata = [
|
|
json.loads(
|
|
(args.output / case["case_id"].lower() / "ollama_metadata.json").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
for case in cases
|
|
]
|
|
summary = {
|
|
"experiment": "request_acceptance_gold_v0", "model": args.model,
|
|
"llm_call_count": len(cases),
|
|
"runtime_seconds": round(sum(item["elapsed_seconds"] for item in metadata), 3),
|
|
"counts": {label: sum(item["classification"] == label for item in evaluations) for label in ("PASS", "PARTIAL", "FAIL")},
|
|
"evaluations": evaluations,
|
|
}
|
|
_write_json(args.output / "summary.json", summary)
|
|
return summary
|
|
|
|
|
|
def _write_json(path: Path, value: Any) -> None:
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def run_gold(args: argparse.Namespace) -> dict[str, Any]:
|
|
cases = load_gold_cases(args.cases)
|
|
args.output.mkdir(parents=True, exist_ok=False)
|
|
_write_json(args.output / "gold_cases.json", {"schema_version": GOLD_SCHEMA_VERSION, "cases": cases})
|
|
evaluations = []
|
|
total_started = time.perf_counter()
|
|
for case in cases:
|
|
case_dir = args.output / case["case_id"].lower()
|
|
case_dir.mkdir()
|
|
_write_json(case_dir / "v3_style_input_observations.json", case["observations"])
|
|
prompt = build_prompt(case["observations"])
|
|
(case_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
|
|
raw, metadata = call_ollama(args.endpoint, args.model, prompt, args.timeout, args.num_ctx, args.num_predict)
|
|
(case_dir / "raw_model_response.txt").write_text(raw + "\n", encoding="utf-8")
|
|
_write_json(case_dir / "ollama_metadata.json", metadata)
|
|
parsed = parse_model_json(raw)
|
|
_write_json(case_dir / "parsed_semantic_recognition.json", parsed)
|
|
try:
|
|
evaluation = evaluate_case(case, parsed)
|
|
validation = {"valid": True, "error": None}
|
|
except (DerivationValidationError, json.JSONDecodeError) as exc:
|
|
validation = {"valid": False, "error_type": type(exc).__name__, "error": str(exc)}
|
|
evaluation = {"case_id": case["case_id"], "classification": "FAIL", "error": str(exc), "responsibility_status_leakage": "forbidden" in str(exc)}
|
|
_write_json(case_dir / "structural_validation.json", validation)
|
|
_write_json(case_dir / "evaluation.json", evaluation)
|
|
evaluations.append(evaluation)
|
|
summary = {
|
|
"experiment": "request_acceptance_gold_v0", "model": args.model,
|
|
"llm_call_count": len(cases), "runtime_seconds": round(time.perf_counter() - total_started, 3),
|
|
"counts": {label: sum(item["classification"] == label for item in evaluations) for label in ("PASS", "PARTIAL", "FAIL")},
|
|
"evaluations": evaluations,
|
|
}
|
|
_write_json(args.output / "summary.json", summary)
|
|
return summary
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Run isolated request/acceptance Gold experiment")
|
|
parser.add_argument("cases", type=Path)
|
|
parser.add_argument("-o", "--output", type=Path, required=True)
|
|
parser.add_argument("--model", default=DEFAULT_MODEL)
|
|
parser.add_argument("--endpoint", default=DEFAULT_ENDPOINT)
|
|
parser.add_argument("--timeout", type=int, default=300)
|
|
parser.add_argument("--num-ctx", type=int, default=16384)
|
|
parser.add_argument("--num-predict", type=int, default=1024)
|
|
parser.add_argument("--reevaluate-existing", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
summary = reevaluate_existing(args) if args.reevaluate_existing else run_gold(args)
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
return 0 if summary["counts"]["FAIL"] == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|