martin/meeting-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stabilize Meeting Lab pipeline for RC1 evaluation

Browse Source 
This commit significantly improves the robustness and determinism of the Meeting Lab processing pipeline and establishes the first Release Candidate baseline for end-to-end evaluation.

Highlights

- BUG-009
  - Implement deterministic responsible-party validation
  - Normalize participant aliases using Meeting Context
  - Reject invalid responsible values (dates, locations, technical terms, projects, products, unknown entities)
  - Record structured responsibility validation metadata
  - Add focused regression tests

- BUG-010
  - Implement adaptive num_predict estimation for Semantic Consolidator
  - Eliminate JSON truncation caused by fixed output limits
  - Add deterministic source coverage repair
  - Preserve strict post-repair validation
  - Add regression tests

- BUG-011
  - Implement Working Protocol V2 renderer contract enforcement
  - Preserve raw renderer responses
  - Reject invalid protocol output instead of accepting malformed documents
  - Add deterministic cleanup for harmless formatting deviations
  - Add focused renderer regression tests

- Meeting Context
  - Validate Meeting Context V1
  - Integrate authoritative participant alias normalization

- Documentation
  - Update architecture documentation
  - Update output documentation
  - Update regression bug tracker

The pipeline now fails safely instead of silently accepting invalid intermediate or final artifacts.

Remaining work focuses primarily on extraction quality and semantic classification (decisions, action items, protocol faithfulness), rather than pipeline robustness.
This commit is contained in:
admin
2026-08-04 13:11:54 +02:00
parent 60a8acae91
commit 950284e236
10 changed files with 1816 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_canonicalize.py
+98
View File
@@ -19,6 +19,8 @@ EMPTY_EXTRACTION = {
"technical": [],
}
PROGEO_CONTEXT = Path("samples/real_live/progeo_meeting/meeting_context.yaml")
def write_extraction(directory: Path, name: str, data: dict) -> Path:
path = directory / name
@@ -163,6 +165,102 @@ class CanonicalizeTests(unittest.TestCase):
self.assertEqual(output["stats"]["input_item_count"], 0)
self.assertEqual(output["stats"]["output_item_count"], 0)
def test_responsible_aliases_normalize_with_meeting_context(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
data = dict(EMPTY_EXTRACTION)
data["todos"] = [
{"task": "A", "responsible": "Martin", "deadline": None, "evidence": "e"},
{"task": "B", "responsible": "Herr Tazl", "deadline": None, "evidence": "e"},
{"task": "C", "responsible": "Marlene", "deadline": None, "evidence": "e"},
{"task": "D", "responsible": "Gothard", "deadline": None, "evidence": "e"},
{"task": "E", "responsible": "Henning", "deadline": None, "evidence": "e"},
]
write_extraction(root, "chunk_01_extraction.json", data)
output = canonicalize_extractions(root, meeting_context_path=PROGEO_CONTEXT)
responsibles = [item["responsible"] for item in output["items"]]
self.assertEqual(
responsibles,
["Martin", "Martin", "Marleen", "Gotthard", "Henning"],
)
self.assertEqual(output["stats"]["responsibility_rejection_count"], 0)
self.assertEqual(output["stats"]["responsibility_normalization_count"], 3)
def test_invalid_responsible_values_are_cleared_with_validation_report(self) -> None:
invalid_values = [
"31. August",
"15. oder 16. September",
"27.8.",
"am 31.",
"morgen",
"nächste Woche",
"Frankfurt",
"Progeo",
"Secugrid",
"8:30 Uhr",
"Unbekannte Person",
]
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
data = dict(EMPTY_EXTRACTION)
data["todos"] = [
{
"task": f"Task {index}",
"responsible": value,
"deadline": None,
"evidence": "e",
}
for index, value in enumerate(invalid_values, start=1)
]
data["todos"].append(
{
"task": "Null task",
"responsible": None,
"deadline": None,
"evidence": "e",
}
)
write_extraction(root, "chunk_01_extraction.json", data)
output = canonicalize_extractions(root, meeting_context_path=PROGEO_CONTEXT)
self.assertTrue(all(item["responsible"] is None for item in output["items"]))
self.assertEqual(
output["stats"]["responsibility_rejection_count"],
len(invalid_values),
)
rejected = output["responsibility_validations"]
self.assertEqual([item["original_value"] for item in rejected], invalid_values)
self.assertTrue(all(item["cleared_to_null"] for item in rejected))
def test_null_responsible_remains_valid_without_warning(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
data = dict(EMPTY_EXTRACTION)
data["todos"] = [
{"task": "Task", "responsible": None, "deadline": None, "evidence": "e"}
]
write_extraction(root, "chunk_01_extraction.json", data)
output = canonicalize_extractions(root, meeting_context_path=PROGEO_CONTEXT)
self.assertIsNone(output["items"][0]["responsible"])
self.assertEqual(output["responsibility_validations"], [])
def test_responsible_validation_does_not_run_without_meeting_context(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
data = dict(EMPTY_EXTRACTION)
data["todos"] = ["Update docs | Mira | Friday | I will update docs"]
write_extraction(root, "chunk_01_extraction.json", data)
output = canonicalize_extractions(root)
self.assertEqual(output["items"][0]["responsible"], "Mira")
self.assertEqual(output["responsibility_validations"], [])
if __name__ == "__main__":
unittest.main()
tests/test_consolidate_facts.py
+110
View File
@@ -1,11 +1,18 @@
import unittest
import json
from pathlib import Path
from src.meeting_lab.consolidation.consolidate_facts import (
ConsolidationValidationError,
DEFAULT_MIN_NUM_PREDICT,
DEFAULT_NUM_PREDICT,
build_consolidated_output,
build_ollama_payload,
estimate_response_tokens,
fact_items,
parse_model_json,
repair_model_group_coverage,
resolve_num_predict,
validate_consolidated_output,
validate_model_groups,
)
@@ -98,6 +105,54 @@ class ConsolidateFactsTests(unittest.TestCase):
self.assertEqual(payload["options"]["num_ctx"], 16384)
self.assertEqual(payload["options"]["num_predict"], 1024)
def test_explicit_num_predict_is_preserved(self):
facts = [canonicalized_fixture()["items"][0]]
self.assertEqual(
resolve_num_predict(
requested_num_predict=1234,
facts=facts,
prompt_token_estimate=100,
num_ctx=32768,
),
1234,
)
def test_adaptive_num_predict_scales_with_fact_payload(self):
fixture = canonicalized_fixture()
base_facts = fact_items(fixture)
larger_facts = []
for index in range(80):
item = dict(base_facts[index % len(base_facts)])
item["item_id"] = f"fact_{index + 1:04d}"
item["text"] = item["text"] + " " + ("detail " * 20)
item["evidence"] = item["evidence"] + " " + ("evidence " * 20)
larger_facts.append(item)
resolved = resolve_num_predict(
requested_num_predict=None,
facts=larger_facts,
prompt_token_estimate=9000,
num_ctx=32768,
)
self.assertGreater(resolved, DEFAULT_MIN_NUM_PREDICT)
self.assertLessEqual(resolved, 32768 - 9000 - 1024)
def test_progeo_context_benchmark_needs_more_than_fixed_default_when_available(self):
path = Path(
"samples/benchmarks/progeo_meeting_context_v1_20260804_110913/"
"canonicalizer/canonicalized_extractions.json"
)
if not path.exists():
self.skipTest("Progeo context benchmark artifact is not available.")
canonicalized = json.loads(path.read_text(encoding="utf-8-sig"))
facts = fact_items(canonicalized)
self.assertGreater(len(facts), 60)
self.assertGreater(estimate_response_tokens(facts), DEFAULT_NUM_PREDICT)
def test_grouping_validation_accepts_complete_singletons(self):
groups = validate_model_groups(
{
@@ -154,6 +209,61 @@ class ConsolidateFactsTests(unittest.TestCase):
{"fact_0001"},
)
def test_source_coverage_repair_restores_missing_singletons(self):
fixture = canonicalized_fixture()
facts = fact_items(fixture)
repaired, changes = repair_model_group_coverage(
{
"groups": [
{
"canonical_text": "The lead maintains the list.",
"source_item_ids": ["fact_0001"],
"merge_reason": "Singleton.",
}
]
},
facts,
)
groups = validate_model_groups(repaired, {"fact_0001", "fact_0002"})
self.assertEqual(len(groups), 2)
self.assertEqual(groups[1]["source_item_ids"], ["fact_0002"])
self.assertEqual(
changes[0]["operation"],
"restore_missing_source_id_as_singleton",
)
def test_source_coverage_repair_removes_duplicate_occurrences(self):
fixture = canonicalized_fixture()
facts = fact_items(fixture)
repaired, changes = repair_model_group_coverage(
{
"groups": [
{
"canonical_text": "Merged.",
"source_item_ids": ["fact_0001", "fact_0002"],
"merge_reason": "Same.",
},
{
"canonical_text": "Duplicate.",
"source_item_ids": ["fact_0002"],
"merge_reason": "Duplicate.",
},
]
},
facts,
)
groups = validate_model_groups(repaired, {"fact_0001", "fact_0002"})
self.assertEqual(len(groups), 1)
self.assertEqual(groups[0]["source_item_ids"], ["fact_0001", "fact_0002"])
self.assertEqual(
[change["operation"] for change in changes],
["remove_duplicate_source_id", "remove_empty_group"],
)
def test_merged_group_validation(self):
groups = validate_model_groups(
{
tests/test_render_working_protocol.py
+179
View File
@@ -0,0 +1,179 @@
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from src.meeting_lab.protocol.render_working_protocol import (
REQUIRED_TITLE,
clean_working_protocol_markdown,
render_working_protocol,
validate_working_protocol_markdown,
)
VALID_PROTOCOL = """# Working Protocol
## Materialversuche
### Background
Die Materialversuche wurden besprochen.
### Decisions
- Der nächste Versuch wird vorbereitet.
### Action Items
- Verantwortung offen: Materialstatus prüfen.
### Open Questions
- Wann liegen die Ergebnisse vor?
"""
class WorkingProtocolRendererTests(unittest.TestCase):
def test_valid_protocol_beginning_with_required_heading_passes(self) -> None:
report = validate_working_protocol_markdown(VALID_PROTOCOL)
self.assertTrue(report.valid)
self.assertEqual(report.violations, [])
def test_explanatory_prose_before_valid_protocol_is_removed(self) -> None:
cleaned = clean_working_protocol_markdown(
"Hier ist das Protokoll:\n\n" + VALID_PROTOCOL
)
self.assertTrue(cleaned.startswith(REQUIRED_TITLE + "\n"))
self.assertNotIn("Hier ist das Protokoll", cleaned)
self.assertTrue(validate_working_protocol_markdown(cleaned).valid)
def test_leading_whitespace_before_valid_heading_is_removed(self) -> None:
cleaned = clean_working_protocol_markdown("\n \n # Working Protocol\n\n## Thema\n\n### Background\n\nText.\n")
self.assertTrue(cleaned.startswith(REQUIRED_TITLE + "\n"))
self.assertTrue(validate_working_protocol_markdown(cleaned).valid)
def test_output_without_required_heading_is_rejected(self) -> None:
report = validate_working_protocol_markdown("## Materialversuche\n\nText.\n")
self.assertFalse(report.valid)
self.assertEqual(report.violations[0]["type"], "missing_required_heading")
def test_categorized_summary_without_topic_body_is_rejected(self) -> None:
text = """Hier ist die Zusammenfassung:
### **Entscheidungen (Decisions)**
- Entscheidung.
### **Handlungsaufträge (Action Items)**
- Aufgabe.
"""
cleaned = clean_working_protocol_markdown(text)
report = validate_working_protocol_markdown(cleaned)
self.assertFalse(report.valid)
self.assertEqual(report.violations[0]["type"], "missing_required_heading")
def test_categorized_summary_after_heading_is_rejected(self) -> None:
text = """# Working Protocol
## Entscheidungen
- Entscheidung.
## Action Items
- Aufgabe.
"""
report = validate_working_protocol_markdown(text)
self.assertFalse(report.valid)
violation_types = {violation["type"] for violation in report.violations}
self.assertIn("generic_summary_framing", violation_types)
self.assertIn("missing_topic_sections", violation_types)
def test_malformed_markdown_is_rejected(self) -> None:
text = """# Working Protocol
## Materialversuche
### Background
```json
{"unterbrochen": true}
"""
report = validate_working_protocol_markdown(text)
self.assertFalse(report.valid)
self.assertIn(
"malformed_markdown",
{violation["type"] for violation in report.violations},
)
def test_raw_response_preserved_and_protocol_contains_only_validated_content(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
input_path = root / "input.json"
input_path.write_text('{"items": []}\n', encoding="utf-8")
output_dir = root / "working_protocol"
with patch(
"src.meeting_lab.protocol.render_working_protocol.call_ollama",
return_value=(
{
"response": "Einleitung.\n\n" + VALID_PROTOCOL,
"done": True,
"done_reason": "stop",
},
1.25,
),
):
metadata = render_working_protocol(input_path, output_dir)
raw = json.loads((output_dir / "raw_model_response.json").read_text(encoding="utf-8"))
self.assertEqual(raw["response"], "Einleitung.\n\n" + VALID_PROTOCOL)
self.assertTrue((output_dir / "cleaned_candidate.md").exists())
self.assertTrue((output_dir / "validation_report.json").exists())
self.assertTrue(metadata["valid"])
protocol = (output_dir / "working_protocol.md").read_text(encoding="utf-8")
self.assertEqual(protocol, clean_working_protocol_markdown(raw["response"]))
self.assertNotIn("Einleitung.", protocol)
def test_invalid_output_preserves_raw_but_does_not_write_protocol(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
input_path = root / "input.json"
input_path.write_text('{"items": []}\n', encoding="utf-8")
output_dir = root / "working_protocol"
with patch(
"src.meeting_lab.protocol.render_working_protocol.call_ollama",
return_value=(
{
"response": "Hier ist die Zusammenfassung:\n\n### Entscheidungen\n\n- X\n",
"done": True,
"done_reason": "stop",
},
1.25,
),
):
metadata = render_working_protocol(input_path, output_dir)
self.assertFalse(metadata["valid"])
self.assertTrue((output_dir / "raw_model_response.json").exists())
self.assertTrue((output_dir / "cleaned_candidate.md").exists())
self.assertTrue((output_dir / "validation_report.json").exists())
self.assertFalse((output_dir / "working_protocol.md").exists())
if __name__ == "__main__":
unittest.main()