350 lines
17 KiB
Python
350 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Isolated collective-commitment Gold reliability experiment."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .experiment_h import (
|
|
DEFAULT_ENDPOINT,
|
|
DEFAULT_MODEL,
|
|
DerivationValidationError,
|
|
OBSERVATION_KEYS,
|
|
call_ollama,
|
|
)
|
|
|
|
|
|
GOLD_SCHEMA_VERSION = "experimental-collective-commitment-gold-v0"
|
|
RECOGNITION_KEYS = {"observation_id", "commitment_form", "normalized_action_text"}
|
|
COMMITMENT_FORMS = {"individual_first_person", "collective_first_person", "none"}
|
|
FORBIDDEN_LLM_KEYS = {
|
|
"responsible_person", "responsibility", "responsibility_scope",
|
|
"requested_actor", "owner", "ownership", "assignee", "status",
|
|
"established", "action_item", "protocol", "protocol_category", "decision",
|
|
"unresolved_issue", "confidence", "relation", "relations", "graph",
|
|
}
|
|
WEEKDAYS = {
|
|
"monday": "Montag", "montag": "Montag", "tuesday": "Dienstag",
|
|
"dienstag": "Dienstag", "wednesday": "Mittwoch", "mittwoch": "Mittwoch",
|
|
"thursday": "Donnerstag", "donnerstag": "Donnerstag", "friday": "Freitag",
|
|
"freitag": "Freitag", "saturday": "Samstag", "samstag": "Samstag",
|
|
"sunday": "Sonntag", "sonntag": "Sonntag",
|
|
}
|
|
|
|
PROMPT_TEMPLATE = """Recognize only the explicit first-person commitment form and concise action meaning in the supplied single V3-style observation.
|
|
|
|
Answer only:
|
|
1. What explicit first-person commitment form is present?
|
|
- individual_first_person: the speaker explicitly commits themself personally.
|
|
- collective_first_person: the speaker explicitly commits a "we" group.
|
|
- none: there is no explicit first-person commitment.
|
|
2. What is the concise normalized action meaning?
|
|
|
|
Tentative possibility is not commitment. Suggestion or recommendation is not commitment. Impersonal necessity is not commitment. Passive future wording is not commitment. Rejection or negation is not positive commitment. Speaker identity does not convert collective "we" into individual commitment.
|
|
|
|
Preserve material limitations such as "nur im Technikum", "nur als Versuch", or "nur 20 Meter" in normalized_action_text. Keep normalized action text in the observation language. When commitment_form is not "none", normalized_action_text must be a non-empty string. When commitment_form is "none", normalized_action_text may be a non-empty action meaning or null.
|
|
|
|
Do not infer who is responsible. Do not decide whether an action is established. Do not output responsibility, responsibility scope, requested actor, owner, assignee, status, established, Action Item, protocol, confidence, semantic relations, graphs, decisions, or unresolved issues.
|
|
|
|
Return exactly this JSON shape and no additional fields:
|
|
{{
|
|
"observation_id": "observation ID",
|
|
"commitment_form": "individual_first_person | collective_first_person | none",
|
|
"normalized_action_text": "concise action meaning" | null
|
|
}}
|
|
|
|
V3-style observation:
|
|
{observation_json}
|
|
"""
|
|
|
|
|
|
def _exact_keys(value: dict[str, Any], required: set[str], location: str) -> None:
|
|
missing = required - value.keys()
|
|
unknown = value.keys() - required
|
|
if missing:
|
|
raise DerivationValidationError(f"{location} missing required keys: {sorted(missing)}")
|
|
if unknown:
|
|
raise DerivationValidationError(f"{location} has unknown keys: {sorted(unknown)}")
|
|
|
|
|
|
def _nonempty_text(value: Any, location: str) -> str:
|
|
if not isinstance(value, str) or not value.strip():
|
|
raise DerivationValidationError(f"{location} must be a non-empty string")
|
|
return value.strip()
|
|
|
|
|
|
def _validate_observations(observations: Any) -> None:
|
|
if not isinstance(observations, list) or not observations:
|
|
raise DerivationValidationError("observations must be a non-empty list")
|
|
seen_observations: set[str] = set()
|
|
seen_evidence: set[str] = set()
|
|
for index, observation in enumerate(observations):
|
|
location = f"observations[{index}]"
|
|
if not isinstance(observation, dict):
|
|
raise DerivationValidationError(f"{location} must be an object")
|
|
_exact_keys(observation, OBSERVATION_KEYS, location)
|
|
observation_id = _nonempty_text(observation["observation_id"], f"{location}.observation_id")
|
|
evidence_id = _nonempty_text(observation["evidence_id"], f"{location}.evidence_id")
|
|
if observation_id in seen_observations or evidence_id in seen_evidence:
|
|
raise DerivationValidationError("observation and evidence provenance must be unique")
|
|
seen_observations.add(observation_id)
|
|
seen_evidence.add(evidence_id)
|
|
_nonempty_text(observation["content"], f"{location}.content")
|
|
_nonempty_text(observation["speaker"], f"{location}.speaker")
|
|
for field in ("named_person", "addressee"):
|
|
if observation[field] is not None:
|
|
_nonempty_text(observation[field], f"{location}.{field}")
|
|
|
|
|
|
def load_gold_cases(path: Path) -> list[dict[str, Any]]:
|
|
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
if not isinstance(data, dict):
|
|
raise DerivationValidationError("Gold fixture must be an object")
|
|
_exact_keys(data, {"schema_version", "cases"}, "Gold fixture")
|
|
if data["schema_version"] != GOLD_SCHEMA_VERSION:
|
|
raise DerivationValidationError("unexpected Gold fixture schema_version")
|
|
cases = data["cases"]
|
|
if not isinstance(cases, list) or not cases:
|
|
raise DerivationValidationError("Gold fixture cases must be a non-empty list")
|
|
seen: set[str] = set()
|
|
for case in cases:
|
|
_exact_keys(case, {"case_id", "description", "observations", "expected_recognition", "expected_result"}, "Gold case")
|
|
case_id = _nonempty_text(case["case_id"], "Gold case.case_id")
|
|
if case_id in seen:
|
|
raise DerivationValidationError(f"duplicate case ID: {case_id}")
|
|
seen.add(case_id)
|
|
_validate_observations(case["observations"])
|
|
if len(case["observations"]) != 1:
|
|
raise DerivationValidationError("collective Gold cases require exactly one observation")
|
|
return cases
|
|
|
|
|
|
def build_prompt(observations: list[dict[str, Any]]) -> str:
|
|
_validate_observations(observations)
|
|
if len(observations) != 1:
|
|
raise DerivationValidationError("collective recognition requires exactly one observation")
|
|
return PROMPT_TEMPLATE.format(
|
|
observation_json=json.dumps(observations[0], ensure_ascii=False, indent=2)
|
|
)
|
|
|
|
|
|
def parse_model_json(raw_text: str) -> dict[str, Any]:
|
|
data = json.loads(raw_text)
|
|
if not isinstance(data, dict):
|
|
raise DerivationValidationError("semantic recognition must be an object")
|
|
return data
|
|
|
|
|
|
def _reject_forbidden_keys(value: Any, location: str = "output") -> None:
|
|
if isinstance(value, dict):
|
|
forbidden = FORBIDDEN_LLM_KEYS.intersection(value)
|
|
if forbidden:
|
|
raise DerivationValidationError(
|
|
f"{location} contains forbidden semantic keys: {sorted(forbidden)}"
|
|
)
|
|
for key, item in value.items():
|
|
_reject_forbidden_keys(item, f"{location}.{key}")
|
|
elif isinstance(value, list):
|
|
for index, item in enumerate(value):
|
|
_reject_forbidden_keys(item, f"{location}[{index}]")
|
|
|
|
|
|
def validate_recognition(data: Any, observations: list[dict[str, Any]]) -> dict[str, Any]:
|
|
_validate_observations(observations)
|
|
if not isinstance(data, dict):
|
|
raise DerivationValidationError("semantic recognition must be an object")
|
|
_reject_forbidden_keys(data)
|
|
_exact_keys(data, RECOGNITION_KEYS, "output")
|
|
observation_id = _nonempty_text(data["observation_id"], "output.observation_id")
|
|
if observation_id not in {item["observation_id"] for item in observations}:
|
|
raise DerivationValidationError("recognition references unknown observation")
|
|
if data["commitment_form"] not in COMMITMENT_FORMS:
|
|
raise DerivationValidationError("commitment_form has an unsupported value")
|
|
action_text = data["normalized_action_text"]
|
|
if action_text is not None:
|
|
_nonempty_text(action_text, "output.normalized_action_text")
|
|
if data["commitment_form"] != "none" and action_text is None:
|
|
raise DerivationValidationError("non-none commitment requires normalized_action_text")
|
|
return data
|
|
|
|
|
|
def _bounded_due(observations: list[dict[str, Any]]) -> tuple[str | None, bool]:
|
|
due_forms: set[str] = set()
|
|
for observation in observations:
|
|
content = observation["content"].casefold()
|
|
for token in re.findall(r"\b[A-Za-zÄÖÜäöü]+\b", content):
|
|
if token in WEEKDAYS:
|
|
due_forms.add(WEEKDAYS[token])
|
|
if re.search(r"\bnächste\s+woche\b", content):
|
|
due_forms.add("nächste Woche")
|
|
return (next(iter(due_forms)) if len(due_forms) == 1 else None, len(due_forms) <= 1)
|
|
|
|
|
|
def _has_explicit_negation(observations: list[dict[str, Any]]) -> bool:
|
|
return any(
|
|
re.search(r"\b(?:nicht|kein(?:e|en|er|es)?|nein|not|no)\b", item["content"], re.IGNORECASE)
|
|
for item in observations
|
|
)
|
|
|
|
|
|
def derive_collective_action(
|
|
observations: list[dict[str, Any]], recognition: dict[str, Any]
|
|
) -> tuple[dict[str, bool], dict[str, Any] | None]:
|
|
validate_recognition(recognition, observations)
|
|
by_id = {item["observation_id"]: item for item in observations}
|
|
observation = by_id.get(recognition["observation_id"])
|
|
due, deadline_consistent = _bounded_due(observations)
|
|
gates = {
|
|
"recognition_schema_valid": True,
|
|
"observation_exists": observation is not None,
|
|
"provenance_valid_and_unique": observation is not None and len({item["evidence_id"] for item in observations}) == len(observations),
|
|
"collective_commitment_form": recognition["commitment_form"] == "collective_first_person",
|
|
"normalized_action_present": isinstance(recognition["normalized_action_text"], str) and bool(recognition["normalized_action_text"].strip()),
|
|
"deadline_supported_and_consistent": deadline_consistent,
|
|
"no_explicit_negation": not _has_explicit_negation(observations),
|
|
}
|
|
if not all(gates.values()):
|
|
return gates, None
|
|
return gates, {
|
|
"action_id": "action_1",
|
|
"content": recognition["normalized_action_text"].strip(),
|
|
"status": "established",
|
|
"commitment_scope": "collective",
|
|
"responsible_person": None,
|
|
"due": due,
|
|
"support": {
|
|
"commitment": {
|
|
"observation_id": observation["observation_id"],
|
|
"evidence_id": observation["evidence_id"],
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def _concepts_present(text: str | None, concepts: list[list[str]]) -> bool:
|
|
if not concepts:
|
|
return True
|
|
if not isinstance(text, str):
|
|
return False
|
|
folded = text.casefold()
|
|
return all(any(alias.casefold() in folded for alias in alternatives) for alternatives in concepts)
|
|
|
|
|
|
def evaluate_case(case: dict[str, Any], recognition: dict[str, Any]) -> dict[str, Any]:
|
|
validate_recognition(recognition, case["observations"])
|
|
gates, result = derive_collective_action(case["observations"], recognition)
|
|
expected_recognition = case["expected_recognition"]
|
|
expected_result = case["expected_result"]
|
|
form_correct = recognition["commitment_form"] == expected_recognition["commitment_form"]
|
|
action_correct = _concepts_present(recognition["normalized_action_text"], expected_recognition["action_concepts"])
|
|
qualifier_preserved = _concepts_present(recognition["normalized_action_text"], expected_recognition["qualifier_concepts"])
|
|
established = result is not None
|
|
final_correct = established == expected_result["established"]
|
|
if result is not None:
|
|
final_correct = final_correct and result["due"] == expected_result["due"] and result["responsible_person"] is None and result["commitment_scope"] == "collective"
|
|
owner_correct = result is None or result["responsible_person"] is None
|
|
automatic_failure = (established and not expected_result["established"]) or not owner_correct or (established and not qualifier_preserved)
|
|
semantic_correct = form_correct and action_correct and qualifier_preserved
|
|
classification = "FAIL" if automatic_failure or not final_correct else ("PASS" if semantic_correct else "PARTIAL")
|
|
return {
|
|
"case_id": case["case_id"], "classification": classification,
|
|
"commitment_form_correct": form_correct,
|
|
"normalized_action_meaning_correct": action_correct,
|
|
"material_qualifier_preserved": qualifier_preserved,
|
|
"deterministic_gates_correct": final_correct,
|
|
"final_result_correct": final_correct,
|
|
"responsible_person_correctly_null": owner_correct,
|
|
"due_correct": result is None or result["due"] == expected_result["due"],
|
|
"unsupported_semantic_strengthening": recognition["commitment_form"] == "collective_first_person" and expected_recognition["commitment_form"] != "collective_first_person",
|
|
"responsibility_status_leakage": False,
|
|
"gates": gates, "result": result,
|
|
}
|
|
|
|
|
|
def _write_json(path: Path, value: Any) -> None:
|
|
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
|
|
|
|
def run_gold(args: argparse.Namespace) -> dict[str, Any]:
|
|
cases = load_gold_cases(args.cases)
|
|
args.output.mkdir(parents=True, exist_ok=False)
|
|
_write_json(args.output / "gold_cases.json", {"schema_version": GOLD_SCHEMA_VERSION, "cases": cases})
|
|
evaluations: list[dict[str, Any]] = []
|
|
successful_calls = 0
|
|
technical_failures = 0
|
|
started = time.perf_counter()
|
|
for case in cases:
|
|
case_dir = args.output / case["case_id"].lower()
|
|
case_dir.mkdir()
|
|
observations = case["observations"]
|
|
_write_json(case_dir / "v3_style_input_observations.json", observations)
|
|
prompt = build_prompt(observations)
|
|
(case_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
|
|
try:
|
|
raw, metadata = call_ollama(args.endpoint, args.model, prompt, args.timeout, args.num_ctx, args.num_predict)
|
|
successful_calls += 1
|
|
except Exception as exc: # one recorded attempt; never retry
|
|
technical_failures += 1
|
|
failure = {"case_id": case["case_id"], "classification": "FAIL", "technical_failure": True, "error_type": type(exc).__name__, "error": str(exc)}
|
|
_write_json(case_dir / "ollama_metadata.json", {"model": args.model, "configuration": {"temperature": 0, "think": False, "num_ctx": args.num_ctx, "num_predict": args.num_predict, "retries": 0}, "technical_failure": failure})
|
|
_write_json(case_dir / "structural_validation.json", {"valid": False, "error": str(exc)})
|
|
_write_json(case_dir / "deterministic_gate_results.json", {})
|
|
_write_json(case_dir / "final_derived_result.json", None)
|
|
_write_json(case_dir / "evaluation.json", failure)
|
|
evaluations.append(failure)
|
|
continue
|
|
(case_dir / "raw_model_response.txt").write_text(raw + "\n", encoding="utf-8")
|
|
_write_json(case_dir / "ollama_metadata.json", metadata)
|
|
try:
|
|
parsed = parse_model_json(raw)
|
|
_write_json(case_dir / "parsed_semantic_recognition.json", parsed)
|
|
evaluation = evaluate_case(case, parsed)
|
|
validation = {"valid": True, "error": None}
|
|
gates, result = derive_collective_action(observations, parsed)
|
|
except (DerivationValidationError, json.JSONDecodeError) as exc:
|
|
validation = {"valid": False, "error_type": type(exc).__name__, "error": str(exc)}
|
|
evaluation = {"case_id": case["case_id"], "classification": "FAIL", "error": str(exc), "responsibility_status_leakage": "forbidden" in str(exc)}
|
|
gates, result = {}, None
|
|
_write_json(case_dir / "structural_validation.json", validation)
|
|
_write_json(case_dir / "deterministic_gate_results.json", gates)
|
|
_write_json(case_dir / "final_derived_result.json", result)
|
|
_write_json(case_dir / "evaluation.json", evaluation)
|
|
evaluations.append(evaluation)
|
|
summary = {
|
|
"experiment": "collective_commitment_gold_v0", "model": args.model,
|
|
"successful_llm_call_count": successful_calls,
|
|
"technical_failed_call_count": technical_failures,
|
|
"runtime_seconds": round(time.perf_counter() - started, 3),
|
|
"counts": {label: sum(item["classification"] == label for item in evaluations) for label in ("PASS", "PARTIAL", "FAIL")},
|
|
"evaluations": evaluations,
|
|
}
|
|
_write_json(args.output / "summary.json", summary)
|
|
return summary
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Run isolated collective-commitment Gold experiment")
|
|
parser.add_argument("cases", type=Path)
|
|
parser.add_argument("-o", "--output", type=Path, required=True)
|
|
parser.add_argument("--model", default=DEFAULT_MODEL)
|
|
parser.add_argument("--endpoint", default=DEFAULT_ENDPOINT)
|
|
parser.add_argument("--timeout", type=int, default=300)
|
|
parser.add_argument("--num-ctx", type=int, default=16384)
|
|
parser.add_argument("--num-predict", type=int, default=1024)
|
|
return parser.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
summary = run_gold(parse_args())
|
|
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
|
return 0 if summary["counts"]["FAIL"] == 0 and summary["technical_failed_call_count"] == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|