Implement Semantic Consolidator V0
- add deterministic canonicalization support for extraction items - add facts-only semantic consolidation using local Ollama - preserve source evidence and validate complete fact coverage - add conservative merge rules and non-LLM tests - record the first validated real-life consolidation benchmark - document current scope, limitations and next evaluation step
This commit is contained in:
@@ -0,0 +1,456 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Canonicalize chunk extraction JSON without semantic merging."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCHEMA_VERSION = "1"
|
||||
|
||||
REQUIRED_CATEGORIES = (
|
||||
"facts",
|
||||
"decisions",
|
||||
"todos",
|
||||
"questions",
|
||||
"positions",
|
||||
"technical",
|
||||
)
|
||||
|
||||
CATEGORY_NAMES = {
|
||||
"facts": "fact",
|
||||
"decisions": "decision",
|
||||
"todos": "action_item",
|
||||
"questions": "open_question",
|
||||
"positions": "position",
|
||||
"technical": "technical_detail",
|
||||
}
|
||||
|
||||
CANONICAL_CATEGORIES = tuple(CATEGORY_NAMES.values())
|
||||
|
||||
CHUNK_EXTRACTION_RE = re.compile(r"^chunk_(\d+)_extraction\.json$")
|
||||
WHITESPACE_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Canonicalize chunk extraction JSON files deterministically."
|
||||
)
|
||||
parser.add_argument(
|
||||
"input_dir",
|
||||
type=Path,
|
||||
help="Directory containing chunk_XX_extraction.json files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path("canonicalized_extractions.json"),
|
||||
help="Output JSON path (default: canonicalized_extractions.json).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-merge-exact-duplicates",
|
||||
action="store_true",
|
||||
help="Preserve exact duplicate items instead of merging them.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def chunk_sort_key(path: Path) -> tuple[int, str]:
|
||||
match = CHUNK_EXTRACTION_RE.match(path.name)
|
||||
if not match:
|
||||
return (sys.maxsize, path.name)
|
||||
return (int(match.group(1)), path.name)
|
||||
|
||||
|
||||
def find_extraction_files(input_dir: Path) -> list[Path]:
|
||||
if not input_dir.is_dir():
|
||||
raise FileNotFoundError(f"Input directory not found: {input_dir}")
|
||||
return sorted(input_dir.glob("chunk_*_extraction.json"), key=chunk_sort_key)
|
||||
|
||||
|
||||
def load_json_object(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid JSON in {path}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"Extraction file must contain a JSON object: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def validate_required_categories(data: dict[str, Any], path: Path) -> None:
|
||||
missing = [category for category in REQUIRED_CATEGORIES if category not in data]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Missing required categories in {path}: {', '.join(missing)}"
|
||||
)
|
||||
invalid = [
|
||||
category
|
||||
for category in REQUIRED_CATEGORIES
|
||||
if not isinstance(data.get(category), list)
|
||||
]
|
||||
if invalid:
|
||||
raise ValueError(
|
||||
f"Required categories must be lists in {path}: {', '.join(invalid)}"
|
||||
)
|
||||
|
||||
|
||||
def clean_text(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = WHITESPACE_RE.sub(" ", str(value)).strip()
|
||||
return text if text else None
|
||||
|
||||
|
||||
def split_legacy_string(value: str) -> list[str]:
|
||||
return [part.strip() for part in value.split("|")]
|
||||
|
||||
|
||||
def first_present(data: dict[str, Any], keys: tuple[str, ...]) -> str | None:
|
||||
for key in keys:
|
||||
text = clean_text(data.get(key))
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def parse_fact(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
"speaker": clean_text(value.get("speaker")),
|
||||
"text": first_present(value, ("statement", "fact", "text")),
|
||||
"status": clean_text(value.get("status")),
|
||||
"evidence": clean_text(value.get("evidence")),
|
||||
}
|
||||
|
||||
text = clean_text(value)
|
||||
parts = split_legacy_string(text or "")
|
||||
if len(parts) >= 4:
|
||||
return {
|
||||
"speaker": clean_text(parts[0]),
|
||||
"text": clean_text(parts[1]),
|
||||
"status": clean_text(parts[2]),
|
||||
"evidence": clean_text(" | ".join(parts[3:])),
|
||||
}
|
||||
if len(parts) >= 3:
|
||||
return {
|
||||
"speaker": None,
|
||||
"text": clean_text(parts[0]),
|
||||
"status": clean_text(parts[1]),
|
||||
"evidence": clean_text(" | ".join(parts[2:])),
|
||||
}
|
||||
return {"speaker": None, "text": text, "status": None, "evidence": None}
|
||||
|
||||
|
||||
def parse_decision(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
"text": first_present(value, ("decision", "text")),
|
||||
"evidence": clean_text(value.get("evidence")),
|
||||
}
|
||||
|
||||
text = clean_text(value)
|
||||
parts = split_legacy_string(text or "")
|
||||
if len(parts) >= 2:
|
||||
return {
|
||||
"text": clean_text(parts[0]),
|
||||
"evidence": clean_text(" | ".join(parts[1:])),
|
||||
}
|
||||
return {"text": text, "evidence": None}
|
||||
|
||||
|
||||
def parse_action_item(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
"text": first_present(value, ("task", "todo", "text")),
|
||||
"responsible": first_present(value, ("responsible", "owner")),
|
||||
"deadline": clean_text(value.get("deadline")),
|
||||
"evidence": clean_text(value.get("evidence")),
|
||||
}
|
||||
|
||||
text = clean_text(value)
|
||||
parts = split_legacy_string(text or "")
|
||||
if len(parts) >= 4:
|
||||
return {
|
||||
"text": clean_text(parts[0]),
|
||||
"responsible": clean_text(parts[1]),
|
||||
"deadline": clean_text(parts[2]),
|
||||
"evidence": clean_text(" | ".join(parts[3:])),
|
||||
}
|
||||
if len(parts) == 3:
|
||||
return {
|
||||
"text": clean_text(parts[0]),
|
||||
"responsible": clean_text(parts[1]),
|
||||
"deadline": None,
|
||||
"evidence": clean_text(parts[2]),
|
||||
}
|
||||
if len(parts) == 2:
|
||||
return {
|
||||
"text": clean_text(parts[0]),
|
||||
"responsible": None,
|
||||
"deadline": None,
|
||||
"evidence": clean_text(parts[1]),
|
||||
}
|
||||
return {
|
||||
"text": text,
|
||||
"responsible": None,
|
||||
"deadline": None,
|
||||
"evidence": None,
|
||||
}
|
||||
|
||||
|
||||
def parse_question(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
"text": first_present(value, ("question", "text")),
|
||||
"evidence": clean_text(value.get("evidence")),
|
||||
}
|
||||
|
||||
text = clean_text(value)
|
||||
parts = split_legacy_string(text or "")
|
||||
if len(parts) >= 2:
|
||||
return {
|
||||
"text": clean_text(parts[0]),
|
||||
"evidence": clean_text(" | ".join(parts[1:])),
|
||||
}
|
||||
return {"text": text, "evidence": None}
|
||||
|
||||
|
||||
def parse_position(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
"speaker": clean_text(value.get("speaker")),
|
||||
"text": first_present(value, ("position", "statement", "text")),
|
||||
"evidence": clean_text(value.get("evidence")),
|
||||
}
|
||||
|
||||
text = clean_text(value)
|
||||
parts = split_legacy_string(text or "")
|
||||
if len(parts) >= 3:
|
||||
return {
|
||||
"speaker": clean_text(parts[0]),
|
||||
"text": clean_text(parts[1]),
|
||||
"evidence": clean_text(" | ".join(parts[2:])),
|
||||
}
|
||||
if len(parts) == 2:
|
||||
return {
|
||||
"speaker": None,
|
||||
"text": clean_text(parts[0]),
|
||||
"evidence": clean_text(parts[1]),
|
||||
}
|
||||
return {"speaker": None, "text": text, "evidence": None}
|
||||
|
||||
|
||||
def parse_technical_detail(value: Any) -> dict[str, Any]:
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
"subject": clean_text(value.get("subject")),
|
||||
"text": first_present(value, ("statement", "technical", "text")),
|
||||
"status": clean_text(value.get("status")),
|
||||
"evidence": clean_text(value.get("evidence")),
|
||||
}
|
||||
|
||||
text = clean_text(value)
|
||||
parts = split_legacy_string(text or "")
|
||||
if len(parts) >= 4:
|
||||
return {
|
||||
"subject": clean_text(parts[0]),
|
||||
"text": clean_text(parts[1]),
|
||||
"status": clean_text(parts[2]),
|
||||
"evidence": clean_text(" | ".join(parts[3:])),
|
||||
}
|
||||
if len(parts) >= 3:
|
||||
return {
|
||||
"subject": None,
|
||||
"text": clean_text(parts[0]),
|
||||
"status": clean_text(parts[1]),
|
||||
"evidence": clean_text(" | ".join(parts[2:])),
|
||||
}
|
||||
return {"subject": None, "text": text, "status": None, "evidence": None}
|
||||
|
||||
|
||||
PARSERS = {
|
||||
"fact": parse_fact,
|
||||
"decision": parse_decision,
|
||||
"action_item": parse_action_item,
|
||||
"open_question": parse_question,
|
||||
"position": parse_position,
|
||||
"technical_detail": parse_technical_detail,
|
||||
}
|
||||
|
||||
|
||||
def source_reference(
|
||||
source_file: str,
|
||||
source_index: int,
|
||||
original_value: Any,
|
||||
evidence: str | None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"source_file": source_file,
|
||||
"source_index": source_index,
|
||||
"evidence": evidence,
|
||||
"original_value": original_value,
|
||||
}
|
||||
|
||||
|
||||
def semantic_key(item: dict[str, Any]) -> str:
|
||||
ignored = {
|
||||
"item_id",
|
||||
"source_file",
|
||||
"source_index",
|
||||
"original_value",
|
||||
"source_references",
|
||||
"duplicate_count",
|
||||
}
|
||||
comparable = {key: value for key, value in item.items() if key not in ignored}
|
||||
return json.dumps(comparable, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
def item_id_for(category: str, counts: Counter[str]) -> str:
|
||||
counts[category] += 1
|
||||
return f"{category}_{counts[category]:04d}"
|
||||
|
||||
|
||||
def canonicalize_value(
|
||||
category: str,
|
||||
value: Any,
|
||||
source_file: str,
|
||||
source_index: int,
|
||||
counts: Counter[str],
|
||||
) -> dict[str, Any]:
|
||||
parsed = PARSERS[category](value)
|
||||
item: dict[str, Any] = {
|
||||
"item_id": item_id_for(category, counts),
|
||||
"category": category,
|
||||
"text": parsed.pop("text", None),
|
||||
"evidence": parsed.pop("evidence", None),
|
||||
"source_file": source_file,
|
||||
"source_index": source_index,
|
||||
"original_value": value,
|
||||
}
|
||||
for key, parsed_value in parsed.items():
|
||||
item[key] = parsed_value
|
||||
item["source_references"] = [
|
||||
source_reference(source_file, source_index, value, item["evidence"])
|
||||
]
|
||||
return item
|
||||
|
||||
|
||||
def merge_exact_duplicates(items: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]:
|
||||
merged: list[dict[str, Any]] = []
|
||||
seen: dict[str, dict[str, Any]] = {}
|
||||
duplicates = 0
|
||||
|
||||
for item in items:
|
||||
key = semantic_key(item)
|
||||
existing = seen.get(key)
|
||||
if existing is None:
|
||||
item["duplicate_count"] = 1
|
||||
seen[key] = item
|
||||
merged.append(item)
|
||||
continue
|
||||
|
||||
duplicates += 1
|
||||
existing["duplicate_count"] += 1
|
||||
existing["source_references"].extend(item["source_references"])
|
||||
|
||||
return merged, duplicates
|
||||
|
||||
|
||||
def canonicalize_extractions(
|
||||
input_dir: Path,
|
||||
merge_duplicates: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
files = find_extraction_files(input_dir)
|
||||
if not files:
|
||||
raise ValueError(f"No chunk extraction JSON files found in {input_dir}")
|
||||
|
||||
counts: Counter[str] = Counter()
|
||||
input_counts: Counter[str] = Counter()
|
||||
items: list[dict[str, Any]] = []
|
||||
|
||||
for path in files:
|
||||
data = load_json_object(path)
|
||||
validate_required_categories(data, path)
|
||||
for raw_category in REQUIRED_CATEGORIES:
|
||||
category = CATEGORY_NAMES[raw_category]
|
||||
values = data[raw_category]
|
||||
input_counts[category] += len(values)
|
||||
for source_index, value in enumerate(values):
|
||||
items.append(
|
||||
canonicalize_value(
|
||||
category=category,
|
||||
value=value,
|
||||
source_file=path.name,
|
||||
source_index=source_index,
|
||||
counts=counts,
|
||||
)
|
||||
)
|
||||
|
||||
exact_duplicates = 0
|
||||
if merge_duplicates:
|
||||
items, exact_duplicates = merge_exact_duplicates(items)
|
||||
|
||||
output_counts = Counter(item["category"] for item in items)
|
||||
input_counts_by_category = {
|
||||
category: input_counts[category] for category in CANONICAL_CATEGORIES
|
||||
}
|
||||
output_counts_by_category = {
|
||||
category: output_counts[category] for category in CANONICAL_CATEGORIES
|
||||
}
|
||||
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"source_files": [path.name for path in files],
|
||||
"stats": {
|
||||
"input_item_count": sum(input_counts.values()),
|
||||
"output_item_count": len(items),
|
||||
"input_item_count_by_category": input_counts_by_category,
|
||||
"output_item_count_by_category": output_counts_by_category,
|
||||
"exact_duplicates_merged": exact_duplicates,
|
||||
},
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
def write_canonicalized(output: dict[str, Any], output_path: Path) -> Path:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(
|
||||
json.dumps(output, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return output_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
output = canonicalize_extractions(
|
||||
args.input_dir,
|
||||
merge_duplicates=not args.no_merge_exact_duplicates,
|
||||
)
|
||||
output_path = write_canonicalized(output, args.output)
|
||||
except (OSError, UnicodeError, ValueError) as exc:
|
||||
print(f"Error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
stats = output["stats"]
|
||||
print(f"Input directory: {args.input_dir}")
|
||||
print(f"Input files processed: {len(output['source_files'])}")
|
||||
print(f"Input item count by category: {stats['input_item_count_by_category']}")
|
||||
print(f"Output item count by category: {stats['output_item_count_by_category']}")
|
||||
print(f"Exact duplicates merged: {stats['exact_duplicates_merged']}")
|
||||
print(f"Output: {output_path}")
|
||||
print(f"Output JSON size: {output_path.stat().st_size} bytes")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,603 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LLM-backed V0 semantic consolidation for fact items only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from queue import Empty, Queue
|
||||
from threading import Thread
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
try:
|
||||
from meeting_lab.llm.prompts import load_prompt
|
||||
except ModuleNotFoundError: # pragma: no cover - used by repository-root tests.
|
||||
from src.meeting_lab.llm.prompts import load_prompt
|
||||
|
||||
|
||||
DEFAULT_MODEL = "qwen3.5:9b"
|
||||
DEFAULT_ENDPOINT = "http://127.0.0.1:11434/api/generate"
|
||||
DEFAULT_NUM_CTX = 32768
|
||||
DEFAULT_NUM_PREDICT = 4096
|
||||
DEFAULT_PROGRESS_INTERVAL = 30
|
||||
PROMPT_NAME = "consolidate_facts.md"
|
||||
|
||||
|
||||
class ConsolidationValidationError(ValueError):
|
||||
"""Raised when model consolidation output violates strict invariants."""
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Consolidate semantically equivalent fact items only."
|
||||
)
|
||||
parser.add_argument(
|
||||
"canonicalized_input",
|
||||
type=Path,
|
||||
help="Canonicalizer V1 JSON file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output-dir",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Directory for consolidated_extractions.json, report.md and raw response.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default=DEFAULT_MODEL,
|
||||
help=f"Ollama model name (default: {DEFAULT_MODEL}).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--endpoint",
|
||||
default=DEFAULT_ENDPOINT,
|
||||
help=f"Ollama generate endpoint (default: {DEFAULT_ENDPOINT}).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout",
|
||||
type=int,
|
||||
default=1800,
|
||||
help="HTTP timeout in seconds (default: 1800).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-ctx",
|
||||
type=int,
|
||||
default=DEFAULT_NUM_CTX,
|
||||
help=f"Context window tokens (default: {DEFAULT_NUM_CTX}).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-predict",
|
||||
type=int,
|
||||
default=DEFAULT_NUM_PREDICT,
|
||||
help=(
|
||||
"Maximum generated tokens. The default is bounded for the expected "
|
||||
f"fact-group JSON while leaving truncation headroom (default: {DEFAULT_NUM_PREDICT})."
|
||||
),
|
||||
)
|
||||
thinking = parser.add_mutually_exclusive_group()
|
||||
thinking.add_argument(
|
||||
"--think",
|
||||
dest="think",
|
||||
action="store_true",
|
||||
help="Enable Ollama thinking output when the selected model supports it.",
|
||||
)
|
||||
thinking.add_argument(
|
||||
"--no-think",
|
||||
dest="think",
|
||||
action="store_false",
|
||||
help="Disable Ollama thinking output for structured JSON consolidation.",
|
||||
)
|
||||
parser.set_defaults(think=False)
|
||||
parser.add_argument(
|
||||
"--progress-interval",
|
||||
type=int,
|
||||
default=DEFAULT_PROGRESS_INTERVAL,
|
||||
help=(
|
||||
"Seconds between waiting-status messages while the non-streaming "
|
||||
f"Ollama request is in flight (default: {DEFAULT_PROGRESS_INTERVAL})."
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_json_object(path: Path) -> dict[str, Any]:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(f"Invalid JSON in {path}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"JSON file must contain an object: {path}")
|
||||
return data
|
||||
|
||||
|
||||
def fact_items(canonicalized: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
items = canonicalized.get("items")
|
||||
if not isinstance(items, list):
|
||||
raise ValueError("Canonicalized input must contain an items list.")
|
||||
return [item for item in items if item.get("category") == "fact"]
|
||||
|
||||
|
||||
def non_fact_items(canonicalized: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
items = canonicalized.get("items")
|
||||
if not isinstance(items, list):
|
||||
raise ValueError("Canonicalized input must contain an items list.")
|
||||
return [item for item in items if item.get("category") != "fact"]
|
||||
|
||||
|
||||
def model_fact_payload(facts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
payload: list[dict[str, Any]] = []
|
||||
for item in facts:
|
||||
payload.append(
|
||||
{
|
||||
"item_id": item.get("item_id"),
|
||||
"text": item.get("text"),
|
||||
"evidence": item.get("evidence"),
|
||||
"speaker": item.get("speaker"),
|
||||
"status": item.get("status"),
|
||||
"source_file": item.get("source_file"),
|
||||
"source_index": item.get("source_index"),
|
||||
}
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def build_consolidation_prompt(facts: list[dict[str, Any]]) -> str:
|
||||
task_prompt = load_prompt(PROMPT_NAME)
|
||||
payload = json.dumps(
|
||||
{"fact_items": model_fact_payload(facts)},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
return f"{task_prompt}\n\nFACT ITEMS:\n{payload}\n"
|
||||
|
||||
|
||||
def response_text_from_ollama_data(data: dict[str, Any]) -> str | None:
|
||||
text = data.get("response")
|
||||
if isinstance(text, str) and text.strip():
|
||||
return text
|
||||
message = data.get("message")
|
||||
if isinstance(message, dict):
|
||||
content = message.get("content")
|
||||
if isinstance(content, str) and content.strip():
|
||||
return content
|
||||
return text if isinstance(text, str) else None
|
||||
|
||||
|
||||
def build_ollama_payload(
|
||||
model: str,
|
||||
prompt: str,
|
||||
num_ctx: int,
|
||||
num_predict: int,
|
||||
think: bool,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"think": think,
|
||||
"stream": False,
|
||||
"format": "json",
|
||||
"options": {
|
||||
"temperature": 0.0,
|
||||
"num_ctx": num_ctx,
|
||||
"num_predict": num_predict,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def print_response_metadata(data: dict[str, Any]) -> None:
|
||||
fields = [
|
||||
"total_duration",
|
||||
"load_duration",
|
||||
"prompt_eval_count",
|
||||
"prompt_eval_duration",
|
||||
"eval_count",
|
||||
"eval_duration",
|
||||
]
|
||||
present = [(field, data.get(field)) for field in fields if field in data]
|
||||
if not present:
|
||||
print("Ollama response metadata: unavailable")
|
||||
return
|
||||
print("Ollama response metadata:")
|
||||
for field, value in present:
|
||||
print(f" {field}: {value}")
|
||||
|
||||
|
||||
def post_with_progress(
|
||||
endpoint: str,
|
||||
payload: dict[str, Any],
|
||||
timeout: int,
|
||||
progress_interval: int,
|
||||
) -> tuple[requests.Response, float]:
|
||||
results: Queue[tuple[str, requests.Response | BaseException, float]] = Queue()
|
||||
|
||||
def worker() -> None:
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = requests.post(endpoint, json=payload, timeout=timeout)
|
||||
except BaseException as exc: # noqa: BLE001 - forwarded to main thread.
|
||||
results.put(("error", exc, time.perf_counter() - started))
|
||||
else:
|
||||
results.put(("response", response, time.perf_counter() - started))
|
||||
|
||||
started_wait = time.perf_counter()
|
||||
thread = Thread(target=worker, daemon=True)
|
||||
thread.start()
|
||||
while True:
|
||||
try:
|
||||
status, result, elapsed = results.get(timeout=max(progress_interval, 1))
|
||||
except Empty:
|
||||
print(
|
||||
"Waiting for Ollama response: "
|
||||
f"{time.perf_counter() - started_wait:.1f} seconds elapsed",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
if status == "error":
|
||||
raise result
|
||||
return result, elapsed
|
||||
|
||||
|
||||
def call_ollama(
|
||||
endpoint: str,
|
||||
model: str,
|
||||
prompt: str,
|
||||
timeout: int,
|
||||
num_ctx: int,
|
||||
num_predict: int,
|
||||
think: bool,
|
||||
progress_interval: int,
|
||||
) -> tuple[str, dict[str, Any], float]:
|
||||
payload = build_ollama_payload(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
num_ctx=num_ctx,
|
||||
num_predict=num_predict,
|
||||
think=think,
|
||||
)
|
||||
print(f"Ollama request start: {datetime.now().isoformat(timespec='seconds')}")
|
||||
print(f"Ollama endpoint: {endpoint}")
|
||||
print(f"Ollama model: {model}")
|
||||
print(f"Ollama stream: {payload['stream']}")
|
||||
print(f"Ollama think: {payload['think']}")
|
||||
print(f"Ollama timeout seconds: {timeout}")
|
||||
print(f"Ollama options: {json.dumps(payload['options'], sort_keys=True)}")
|
||||
response, elapsed = post_with_progress(
|
||||
endpoint=endpoint,
|
||||
payload=payload,
|
||||
timeout=timeout,
|
||||
progress_interval=progress_interval,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print_response_metadata(data)
|
||||
text = response_text_from_ollama_data(data)
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise ValueError("Ollama returned no usable response text.")
|
||||
return text.strip(), data, elapsed
|
||||
|
||||
|
||||
def parse_model_json(text: str) -> dict[str, Any]:
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ConsolidationValidationError(f"Invalid model JSON: {exc}") from exc
|
||||
if not isinstance(parsed, dict):
|
||||
raise ConsolidationValidationError("Model JSON must be an object.")
|
||||
return parsed
|
||||
|
||||
|
||||
def validate_model_groups(
|
||||
model_output: dict[str, Any],
|
||||
expected_fact_ids: set[str],
|
||||
) -> list[dict[str, Any]]:
|
||||
groups = model_output.get("groups")
|
||||
if not isinstance(groups, list):
|
||||
raise ConsolidationValidationError("Model output must contain a groups list.")
|
||||
|
||||
seen: list[str] = []
|
||||
validated: list[dict[str, Any]] = []
|
||||
for index, group in enumerate(groups):
|
||||
if not isinstance(group, dict):
|
||||
raise ConsolidationValidationError(f"Group {index} must be an object.")
|
||||
source_ids = group.get("source_item_ids")
|
||||
if not isinstance(source_ids, list) or not source_ids:
|
||||
raise ConsolidationValidationError(
|
||||
f"Group {index} must contain source_item_ids."
|
||||
)
|
||||
if not all(isinstance(item_id, str) for item_id in source_ids):
|
||||
raise ConsolidationValidationError(
|
||||
f"Group {index} source_item_ids must be strings."
|
||||
)
|
||||
unknown = sorted(set(source_ids) - expected_fact_ids)
|
||||
if unknown:
|
||||
raise ConsolidationValidationError(
|
||||
f"Group {index} contains unknown source item IDs: {unknown}"
|
||||
)
|
||||
duplicates_in_group = [
|
||||
item_id for item_id, count in Counter(source_ids).items() if count > 1
|
||||
]
|
||||
if duplicates_in_group:
|
||||
raise ConsolidationValidationError(
|
||||
f"Group {index} repeats source item IDs: {duplicates_in_group}"
|
||||
)
|
||||
canonical_text = group.get("canonical_text")
|
||||
if not isinstance(canonical_text, str) or not canonical_text.strip():
|
||||
raise ConsolidationValidationError(
|
||||
f"Group {index} must contain canonical_text."
|
||||
)
|
||||
merge_reason = group.get("merge_reason")
|
||||
if not isinstance(merge_reason, str) or not merge_reason.strip():
|
||||
raise ConsolidationValidationError(
|
||||
f"Group {index} must contain merge_reason."
|
||||
)
|
||||
seen.extend(source_ids)
|
||||
validated.append(
|
||||
{
|
||||
"canonical_text": canonical_text.strip(),
|
||||
"source_item_ids": source_ids,
|
||||
"merge_reason": merge_reason.strip(),
|
||||
}
|
||||
)
|
||||
|
||||
seen_counts = Counter(seen)
|
||||
duplicated = sorted(item_id for item_id, count in seen_counts.items() if count > 1)
|
||||
if duplicated:
|
||||
raise ConsolidationValidationError(
|
||||
f"Source item IDs appear in multiple groups: {duplicated}"
|
||||
)
|
||||
missing = sorted(expected_fact_ids - set(seen))
|
||||
if missing:
|
||||
raise ConsolidationValidationError(f"Missing source item IDs: {missing}")
|
||||
|
||||
return validated
|
||||
|
||||
|
||||
def validate_group_shapes(groups: list[dict[str, Any]]) -> None:
|
||||
for group in groups:
|
||||
source_count = len(group["source_item_ids"])
|
||||
if source_count < 1:
|
||||
raise ConsolidationValidationError("Groups must not be empty.")
|
||||
if source_count == 1:
|
||||
continue
|
||||
if source_count < 2:
|
||||
raise ConsolidationValidationError("Merged groups need at least two IDs.")
|
||||
|
||||
|
||||
def build_consolidated_fact_item(
|
||||
group: dict[str, Any],
|
||||
fact_by_id: dict[str, dict[str, Any]],
|
||||
sequence: int,
|
||||
) -> dict[str, Any]:
|
||||
source_ids = group["source_item_ids"]
|
||||
source_items = [fact_by_id[item_id] for item_id in source_ids]
|
||||
source_references: list[dict[str, Any]] = []
|
||||
evidence: list[str] = []
|
||||
for item in source_items:
|
||||
source_references.extend(item.get("source_references", []))
|
||||
item_evidence = item.get("evidence")
|
||||
if isinstance(item_evidence, str):
|
||||
evidence.append(item_evidence)
|
||||
return {
|
||||
"consolidated_id": f"fact_group_{sequence:04d}",
|
||||
"category": "fact",
|
||||
"canonical_text": group["canonical_text"],
|
||||
"source_item_ids": source_ids,
|
||||
"source_references": source_references,
|
||||
"evidence": evidence,
|
||||
"merge_reason": group["merge_reason"],
|
||||
}
|
||||
|
||||
|
||||
def build_consolidated_output(
|
||||
canonicalized: dict[str, Any],
|
||||
groups: list[dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
facts = fact_items(canonicalized)
|
||||
fact_by_id = {item["item_id"]: item for item in facts}
|
||||
consolidated_facts = [
|
||||
build_consolidated_fact_item(group, fact_by_id, index)
|
||||
for index, group in enumerate(groups, start=1)
|
||||
]
|
||||
output_items = consolidated_facts + non_fact_items(canonicalized)
|
||||
merged_groups = [item for item in consolidated_facts if len(item["source_item_ids"]) > 1]
|
||||
singleton_groups = [
|
||||
item for item in consolidated_facts if len(item["source_item_ids"]) == 1
|
||||
]
|
||||
|
||||
output = dict(canonicalized)
|
||||
output["schema_version"] = "semantic_consolidator_v0"
|
||||
output["items"] = output_items
|
||||
output["semantic_consolidation"] = {
|
||||
"scope": "facts_only",
|
||||
"merged_fact_group_count": len(merged_groups),
|
||||
"source_facts_in_merged_groups": sum(
|
||||
len(item["source_item_ids"]) for item in merged_groups
|
||||
),
|
||||
"singleton_fact_group_count": len(singleton_groups),
|
||||
}
|
||||
return output
|
||||
|
||||
|
||||
def validate_consolidated_output(
|
||||
canonicalized: dict[str, Any],
|
||||
output: dict[str, Any],
|
||||
) -> None:
|
||||
original_facts = fact_items(canonicalized)
|
||||
expected_fact_ids = {item["item_id"] for item in original_facts}
|
||||
output_items = output.get("items")
|
||||
if not isinstance(output_items, list):
|
||||
raise ConsolidationValidationError("Output items must be a list.")
|
||||
output_fact_groups = [item for item in output_items if item.get("category") == "fact"]
|
||||
seen: list[str] = []
|
||||
for item in output_fact_groups:
|
||||
source_ids = item.get("source_item_ids")
|
||||
if not isinstance(source_ids, list):
|
||||
raise ConsolidationValidationError("Fact groups need source_item_ids.")
|
||||
if len(source_ids) == 0:
|
||||
raise ConsolidationValidationError("Fact groups must not be empty.")
|
||||
if len(source_ids) > 1 and not item.get("merge_reason"):
|
||||
raise ConsolidationValidationError("Merged fact groups need merge_reason.")
|
||||
seen.extend(source_ids)
|
||||
source_refs = item.get("source_references")
|
||||
if not isinstance(source_refs, list) or not source_refs:
|
||||
raise ConsolidationValidationError("Fact groups need source references.")
|
||||
seen_counts = Counter(seen)
|
||||
duplicated = sorted(item_id for item_id, count in seen_counts.items() if count > 1)
|
||||
if duplicated:
|
||||
raise ConsolidationValidationError(
|
||||
f"Output duplicates source fact IDs: {duplicated}"
|
||||
)
|
||||
missing = sorted(expected_fact_ids - set(seen))
|
||||
if missing:
|
||||
raise ConsolidationValidationError(f"Output misses source fact IDs: {missing}")
|
||||
unknown = sorted(set(seen) - expected_fact_ids)
|
||||
if unknown:
|
||||
raise ConsolidationValidationError(f"Output has unknown source IDs: {unknown}")
|
||||
|
||||
original_non_facts = non_fact_items(canonicalized)
|
||||
output_non_facts = [item for item in output_items if item.get("category") != "fact"]
|
||||
if output_non_facts != original_non_facts:
|
||||
raise ConsolidationValidationError("Non-fact categories changed.")
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(data, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def write_report(
|
||||
path: Path,
|
||||
model: str,
|
||||
runtime: float,
|
||||
prompt_chars: int,
|
||||
prompt_token_estimate: int,
|
||||
fact_count: int,
|
||||
groups: list[dict[str, Any]],
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
merged = [group for group in groups if len(group["source_item_ids"]) > 1]
|
||||
singletons = [group for group in groups if len(group["source_item_ids"]) == 1]
|
||||
lines = [
|
||||
"# Semantic Consolidator V0 Report",
|
||||
"",
|
||||
"- Scope: facts only",
|
||||
f"- Model: `{model}`",
|
||||
"- LLM call count: 1",
|
||||
f"- Runtime: {runtime:.3f} seconds",
|
||||
f"- Fact item count: {fact_count}",
|
||||
f"- Prompt characters: {prompt_chars}",
|
||||
f"- Estimated prompt tokens: {prompt_token_estimate}",
|
||||
f"- Merged fact groups: {len(merged)}",
|
||||
f"- Source facts involved in merges: {sum(len(group['source_item_ids']) for group in merged)}",
|
||||
f"- Singleton fact groups: {len(singletons)}",
|
||||
f"- Output path: `{output_path}`",
|
||||
"",
|
||||
"## Actual Merges",
|
||||
"",
|
||||
]
|
||||
if not merged:
|
||||
lines.append("- None.")
|
||||
else:
|
||||
for group in merged:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {group['canonical_text']}",
|
||||
"",
|
||||
f"- Source fact IDs: {', '.join(group['source_item_ids'])}",
|
||||
f"- Merge reason: {group['merge_reason']}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
raw_response_path = args.output_dir / "raw_model_response.txt"
|
||||
output_path = args.output_dir / "consolidated_extractions.json"
|
||||
report_path = args.output_dir / "report.md"
|
||||
|
||||
try:
|
||||
canonicalized = load_json_object(args.canonicalized_input)
|
||||
facts = fact_items(canonicalized)
|
||||
prompt = build_consolidation_prompt(facts)
|
||||
prompt_chars = len(prompt)
|
||||
prompt_token_estimate = (prompt_chars + 3) // 4
|
||||
print(f"Fact item count: {len(facts)}")
|
||||
print(f"Estimated prompt size chars: {prompt_chars}")
|
||||
print(f"Estimated prompt tokens: {prompt_token_estimate}")
|
||||
print("Expected LLM call count: 1")
|
||||
print("Expected runtime: 5-10 minutes on current local benchmark basis")
|
||||
|
||||
raw_text, _response_data, runtime = call_ollama(
|
||||
endpoint=args.endpoint,
|
||||
model=args.model,
|
||||
prompt=prompt,
|
||||
timeout=args.timeout,
|
||||
num_ctx=args.num_ctx,
|
||||
num_predict=args.num_predict,
|
||||
think=args.think,
|
||||
progress_interval=args.progress_interval,
|
||||
)
|
||||
raw_response_path.write_text(raw_text + "\n", encoding="utf-8")
|
||||
model_output = parse_model_json(raw_text)
|
||||
expected_fact_ids = {item["item_id"] for item in facts}
|
||||
groups = validate_model_groups(model_output, expected_fact_ids)
|
||||
validate_group_shapes(groups)
|
||||
output = build_consolidated_output(canonicalized, groups)
|
||||
validate_consolidated_output(canonicalized, output)
|
||||
write_json(output_path, output)
|
||||
write_report(
|
||||
path=report_path,
|
||||
model=args.model,
|
||||
runtime=runtime,
|
||||
prompt_chars=prompt_chars,
|
||||
prompt_token_estimate=prompt_token_estimate,
|
||||
fact_count=len(facts),
|
||||
groups=groups,
|
||||
output_path=output_path,
|
||||
)
|
||||
except requests.ConnectionError as exc:
|
||||
print(f"Error: Ollama is not reachable at {args.endpoint}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
except requests.Timeout as exc:
|
||||
print(
|
||||
f"Error: Ollama request timed out after {args.timeout} seconds: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
except requests.HTTPError as exc:
|
||||
print(f"Error: Ollama returned an HTTP error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
except (OSError, UnicodeError, ValueError) as exc:
|
||||
print(f"Error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
merged = [group for group in groups if len(group["source_item_ids"]) > 1]
|
||||
print(f"Runtime seconds: {runtime:.3f}")
|
||||
print(f"Merged fact groups: {len(merged)}")
|
||||
print(
|
||||
"Source facts involved in merges: "
|
||||
f"{sum(len(group['source_item_ids']) for group in merged)}"
|
||||
)
|
||||
print(f"Singleton fact groups: {len(groups) - len(merged)}")
|
||||
print("Validation result: passed")
|
||||
print(f"Output: {output_path}")
|
||||
print(f"Report: {report_path}")
|
||||
print(f"Raw model response: {raw_response_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user