admin
950284e236
This commit significantly improves the robustness and determinism of the Meeting Lab processing pipeline and establishes the first Release Candidate baseline for end-to-end evaluation. Highlights - BUG-009 - Implement deterministic responsible-party validation - Normalize participant aliases using Meeting Context - Reject invalid responsible values (dates, locations, technical terms, projects, products, unknown entities) - Record structured responsibility validation metadata - Add focused regression tests - BUG-010 - Implement adaptive num_predict estimation for Semantic Consolidator - Eliminate JSON truncation caused by fixed output limits - Add deterministic source coverage repair - Preserve strict post-repair validation - Add regression tests - BUG-011 - Implement Working Protocol V2 renderer contract enforcement - Preserve raw renderer responses - Reject invalid protocol output instead of accepting malformed documents - Add deterministic cleanup for harmless formatting deviations - Add focused renderer regression tests - Meeting Context - Validate Meeting Context V1 - Integrate authoritative participant alias normalization - Documentation - Update architecture documentation - Update output documentation - Update regression bug tracker The pipeline now fails safely instead of silently accepting invalid intermediate or final artifacts. Remaining work focuses primarily on extraction quality and semantic classification (decisions, action items, protocol faithfulness), rather than pipeline robustness.
326 lines
11 KiB
Python
326 lines
11 KiB
Python
import unittest
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from src.meeting_lab.consolidation.consolidate_facts import (
|
|
ConsolidationValidationError,
|
|
DEFAULT_MIN_NUM_PREDICT,
|
|
DEFAULT_NUM_PREDICT,
|
|
build_consolidated_output,
|
|
build_ollama_payload,
|
|
estimate_response_tokens,
|
|
fact_items,
|
|
parse_model_json,
|
|
repair_model_group_coverage,
|
|
resolve_num_predict,
|
|
validate_consolidated_output,
|
|
validate_model_groups,
|
|
)
|
|
|
|
|
|
def canonicalized_fixture():
|
|
fact_one = {
|
|
"item_id": "fact_0001",
|
|
"category": "fact",
|
|
"text": "The lead maintains the list.",
|
|
"evidence": "lead maintains the list",
|
|
"source_file": "chunk_01_extraction.json",
|
|
"source_index": 0,
|
|
"original_value": "The lead maintains the list. | evidence",
|
|
"source_references": [
|
|
{
|
|
"source_file": "chunk_01_extraction.json",
|
|
"source_index": 0,
|
|
"evidence": "lead maintains the list",
|
|
"original_value": "The lead maintains the list. | evidence",
|
|
}
|
|
],
|
|
"duplicate_count": 1,
|
|
}
|
|
fact_two = {
|
|
"item_id": "fact_0002",
|
|
"category": "fact",
|
|
"text": "The head maintains the project list.",
|
|
"evidence": "head maintains the project list",
|
|
"source_file": "chunk_02_extraction.json",
|
|
"source_index": 0,
|
|
"original_value": "The head maintains the project list. | evidence",
|
|
"source_references": [
|
|
{
|
|
"source_file": "chunk_02_extraction.json",
|
|
"source_index": 0,
|
|
"evidence": "head maintains the project list",
|
|
"original_value": "The head maintains the project list. | evidence",
|
|
}
|
|
],
|
|
"duplicate_count": 1,
|
|
}
|
|
decision = {
|
|
"item_id": "decision_0001",
|
|
"category": "decision",
|
|
"text": "Ship it.",
|
|
"evidence": "Agreed.",
|
|
"source_file": "chunk_01_extraction.json",
|
|
"source_index": 0,
|
|
"original_value": "Ship it. | Agreed.",
|
|
"source_references": [],
|
|
"duplicate_count": 1,
|
|
}
|
|
return {
|
|
"schema_version": "1",
|
|
"source_files": ["chunk_01_extraction.json", "chunk_02_extraction.json"],
|
|
"items": [fact_one, fact_two, decision],
|
|
}
|
|
|
|
|
|
class ConsolidateFactsTests(unittest.TestCase):
|
|
def test_payload_construction_disables_streaming_and_thinking_by_default(self):
|
|
payload = build_ollama_payload(
|
|
model="qwen3.5:9B",
|
|
prompt="prompt",
|
|
num_ctx=32768,
|
|
num_predict=DEFAULT_NUM_PREDICT,
|
|
think=False,
|
|
)
|
|
|
|
self.assertEqual(payload["model"], "qwen3.5:9B")
|
|
self.assertEqual(payload["prompt"], "prompt")
|
|
self.assertIs(payload["stream"], False)
|
|
self.assertIs(payload["think"], False)
|
|
self.assertEqual(payload["format"], "json")
|
|
self.assertEqual(payload["options"]["temperature"], 0.0)
|
|
self.assertEqual(payload["options"]["num_ctx"], 32768)
|
|
self.assertEqual(payload["options"]["num_predict"], DEFAULT_NUM_PREDICT)
|
|
|
|
def test_payload_construction_can_enable_thinking_explicitly(self):
|
|
payload = build_ollama_payload(
|
|
model="qwen3.5:9B",
|
|
prompt="prompt",
|
|
num_ctx=16384,
|
|
num_predict=1024,
|
|
think=True,
|
|
)
|
|
|
|
self.assertIs(payload["think"], True)
|
|
self.assertEqual(payload["options"]["num_ctx"], 16384)
|
|
self.assertEqual(payload["options"]["num_predict"], 1024)
|
|
|
|
def test_explicit_num_predict_is_preserved(self):
|
|
facts = [canonicalized_fixture()["items"][0]]
|
|
|
|
self.assertEqual(
|
|
resolve_num_predict(
|
|
requested_num_predict=1234,
|
|
facts=facts,
|
|
prompt_token_estimate=100,
|
|
num_ctx=32768,
|
|
),
|
|
1234,
|
|
)
|
|
|
|
def test_adaptive_num_predict_scales_with_fact_payload(self):
|
|
fixture = canonicalized_fixture()
|
|
base_facts = fact_items(fixture)
|
|
larger_facts = []
|
|
for index in range(80):
|
|
item = dict(base_facts[index % len(base_facts)])
|
|
item["item_id"] = f"fact_{index + 1:04d}"
|
|
item["text"] = item["text"] + " " + ("detail " * 20)
|
|
item["evidence"] = item["evidence"] + " " + ("evidence " * 20)
|
|
larger_facts.append(item)
|
|
|
|
resolved = resolve_num_predict(
|
|
requested_num_predict=None,
|
|
facts=larger_facts,
|
|
prompt_token_estimate=9000,
|
|
num_ctx=32768,
|
|
)
|
|
|
|
self.assertGreater(resolved, DEFAULT_MIN_NUM_PREDICT)
|
|
self.assertLessEqual(resolved, 32768 - 9000 - 1024)
|
|
|
|
def test_progeo_context_benchmark_needs_more_than_fixed_default_when_available(self):
|
|
path = Path(
|
|
"samples/benchmarks/progeo_meeting_context_v1_20260804_110913/"
|
|
"canonicalizer/canonicalized_extractions.json"
|
|
)
|
|
if not path.exists():
|
|
self.skipTest("Progeo context benchmark artifact is not available.")
|
|
|
|
canonicalized = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
facts = fact_items(canonicalized)
|
|
|
|
self.assertGreater(len(facts), 60)
|
|
self.assertGreater(estimate_response_tokens(facts), DEFAULT_NUM_PREDICT)
|
|
|
|
def test_grouping_validation_accepts_complete_singletons(self):
|
|
groups = validate_model_groups(
|
|
{
|
|
"groups": [
|
|
{
|
|
"canonical_text": "The lead maintains the list.",
|
|
"source_item_ids": ["fact_0001"],
|
|
"merge_reason": "Singleton.",
|
|
},
|
|
{
|
|
"canonical_text": "The head maintains the project list.",
|
|
"source_item_ids": ["fact_0002"],
|
|
"merge_reason": "Singleton.",
|
|
},
|
|
]
|
|
},
|
|
{"fact_0001", "fact_0002"},
|
|
)
|
|
|
|
self.assertEqual(len(groups), 2)
|
|
|
|
def test_no_missing_source_ids(self):
|
|
with self.assertRaisesRegex(ConsolidationValidationError, "Missing"):
|
|
validate_model_groups(
|
|
{
|
|
"groups": [
|
|
{
|
|
"canonical_text": "Only one.",
|
|
"source_item_ids": ["fact_0001"],
|
|
"merge_reason": "Singleton.",
|
|
}
|
|
]
|
|
},
|
|
{"fact_0001", "fact_0002"},
|
|
)
|
|
|
|
def test_no_duplicate_source_ids(self):
|
|
with self.assertRaisesRegex(ConsolidationValidationError, "multiple"):
|
|
validate_model_groups(
|
|
{
|
|
"groups": [
|
|
{
|
|
"canonical_text": "One.",
|
|
"source_item_ids": ["fact_0001"],
|
|
"merge_reason": "Singleton.",
|
|
},
|
|
{
|
|
"canonical_text": "Again.",
|
|
"source_item_ids": ["fact_0001"],
|
|
"merge_reason": "Singleton.",
|
|
},
|
|
]
|
|
},
|
|
{"fact_0001"},
|
|
)
|
|
|
|
def test_source_coverage_repair_restores_missing_singletons(self):
|
|
fixture = canonicalized_fixture()
|
|
facts = fact_items(fixture)
|
|
repaired, changes = repair_model_group_coverage(
|
|
{
|
|
"groups": [
|
|
{
|
|
"canonical_text": "The lead maintains the list.",
|
|
"source_item_ids": ["fact_0001"],
|
|
"merge_reason": "Singleton.",
|
|
}
|
|
]
|
|
},
|
|
facts,
|
|
)
|
|
|
|
groups = validate_model_groups(repaired, {"fact_0001", "fact_0002"})
|
|
|
|
self.assertEqual(len(groups), 2)
|
|
self.assertEqual(groups[1]["source_item_ids"], ["fact_0002"])
|
|
self.assertEqual(
|
|
changes[0]["operation"],
|
|
"restore_missing_source_id_as_singleton",
|
|
)
|
|
|
|
def test_source_coverage_repair_removes_duplicate_occurrences(self):
|
|
fixture = canonicalized_fixture()
|
|
facts = fact_items(fixture)
|
|
repaired, changes = repair_model_group_coverage(
|
|
{
|
|
"groups": [
|
|
{
|
|
"canonical_text": "Merged.",
|
|
"source_item_ids": ["fact_0001", "fact_0002"],
|
|
"merge_reason": "Same.",
|
|
},
|
|
{
|
|
"canonical_text": "Duplicate.",
|
|
"source_item_ids": ["fact_0002"],
|
|
"merge_reason": "Duplicate.",
|
|
},
|
|
]
|
|
},
|
|
facts,
|
|
)
|
|
|
|
groups = validate_model_groups(repaired, {"fact_0001", "fact_0002"})
|
|
|
|
self.assertEqual(len(groups), 1)
|
|
self.assertEqual(groups[0]["source_item_ids"], ["fact_0001", "fact_0002"])
|
|
self.assertEqual(
|
|
[change["operation"] for change in changes],
|
|
["remove_duplicate_source_id", "remove_empty_group"],
|
|
)
|
|
|
|
def test_merged_group_validation(self):
|
|
groups = validate_model_groups(
|
|
{
|
|
"groups": [
|
|
{
|
|
"canonical_text": "The lead maintains the project list.",
|
|
"source_item_ids": ["fact_0001", "fact_0002"],
|
|
"merge_reason": "Same proposition.",
|
|
}
|
|
]
|
|
},
|
|
{"fact_0001", "fact_0002"},
|
|
)
|
|
|
|
output = build_consolidated_output(canonicalized_fixture(), groups)
|
|
validate_consolidated_output(canonicalized_fixture(), output)
|
|
self.assertEqual(output["items"][0]["source_item_ids"], ["fact_0001", "fact_0002"])
|
|
|
|
def test_preservation_of_non_fact_categories(self):
|
|
fixture = canonicalized_fixture()
|
|
groups = validate_model_groups(
|
|
{
|
|
"groups": [
|
|
{
|
|
"canonical_text": "The lead maintains the project list.",
|
|
"source_item_ids": ["fact_0001", "fact_0002"],
|
|
"merge_reason": "Same proposition.",
|
|
}
|
|
]
|
|
},
|
|
{"fact_0001", "fact_0002"},
|
|
)
|
|
|
|
output = build_consolidated_output(fixture, groups)
|
|
|
|
self.assertEqual(output["items"][1:], fixture["items"][2:])
|
|
|
|
def test_invalid_model_json(self):
|
|
with self.assertRaisesRegex(ConsolidationValidationError, "Invalid model JSON"):
|
|
parse_model_json("{invalid")
|
|
|
|
def test_unknown_source_item_ids(self):
|
|
with self.assertRaisesRegex(ConsolidationValidationError, "unknown"):
|
|
validate_model_groups(
|
|
{
|
|
"groups": [
|
|
{
|
|
"canonical_text": "Unknown.",
|
|
"source_item_ids": ["fact_9999"],
|
|
"merge_reason": "Bad ID.",
|
|
}
|
|
]
|
|
},
|
|
{"fact_0001"},
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|