From 58effcafc5706568740510d8accfc732cd2dee98 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 9 Aug 2026 14:33:46 +0200 Subject: [PATCH] Handle repetitive Semantic Consolidator output loops --- docs/architecture.md | 16 ++ docs/regression-bugs.md | 110 +++++++++++ scripts/run_meeting.py | 13 +- .../consolidation/consolidate_facts.py | 183 +++++++++++++++++- tests/test_consolidate_facts.py | 151 +++++++++++++++ 5 files changed, 462 insertions(+), 11 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 480ad8e..2713a00 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -375,6 +375,22 @@ extraction results into `meeting_protocol.md` for technical validation. The planned architecture separates Canonical Meeting Knowledge from the final Output Views documented in `output-views.md`. +## Semantic Consolidator failure handling + +Semantic Consolidator V0 preserves every raw model response before parsing. +Its normal path accepts parseable grouping JSON and leaves duplicate source-ID +and missing-source-ID correction to the existing deterministic coverage +repair. + +An invalid or truncated response is not retried by default. One controlled +retry is allowed only when deterministic inspection finds at least three +consecutive complete groups with an identical structural signature: +`canonical_text`, ordered `source_item_ids` and `merge_reason`. The retry keeps +the same model, temperature, context window and generation limit and adds only +an instruction not to emit an identical group more than once. Both attempts +and the detected repetition metadata are preserved. If the retry also fails, +the stage fails normally; it does not make another LLM call. + --- # Next Milestone diff --git a/docs/regression-bugs.md b/docs/regression-bugs.md index 8b5e4de..4a2c664 100644 --- a/docs/regression-bugs.md +++ b/docs/regression-bugs.md @@ -931,3 +931,113 @@ BUG-011 and does not invalidate this chunking hypothesis. This verifies the hypothesis for this transcript/configuration only. It does not claim the general chunker is fixed. + +## BUG-013 + +ID: BUG-013 + +Title: Semantic Consolidator repetitive output loop exhausts generation budget + +Pipeline stage: Semantic Consolidator + +Severity: High + +Status: Verified + +Date discovered: 2026-08-09 + +Version first observed: `progeo_north_linux_20260809_124954` + +Description: + +The Semantic Consolidator entered a pathological generation loop that emitted +the same complete semantic group 192 consecutive times. The repeated group had +the stable structural signature consisting of the same `canonical_text`, the +same ordered `source_item_ids` (`fact_0004`, `fact_0019`) and the same +`merge_reason`. Generation then truncated at an incomplete +`"canonical_text":` field. + +Expected behaviour: + +The consolidator should detect a structural repeated-group loop +deterministically. Invalid JSON should receive at most one controlled retry +only when that loop signature is present. The retry must use the same model, +temperature and context configuration and add only an instruction preventing +duplicate group emission. Unrelated malformed JSON must not be retried. + +Actual behaviour: + +The first model response was invalid JSON at line 974, column 24, character +71184. Semantic Consolidator runtime was 277.910 seconds. The raw response had +71,184 characters (71,568 UTF-8 bytes), ended at `"canonical_text":`, and +could not reach deterministic source-coverage repair because it was not +parseable JSON. + +Model metadata: + +- Resolved `num_predict`: 19,532 +- `num_ctx`: 32,768 +- `prompt_eval_count`: 14,114 +- `eval_count`: 18,654 +- `done_reason`: `length` +- `eval_count` did not equal resolved `num_predict`; it exactly consumed the + remaining evaluated context (`32768 - 14114 = 18654`) +- Complete groups before truncation: 194 +- Consecutive identical groups: 192, starting at group index 2 + +Root cause: + +The model produced a degenerate repeated semantic-group sequence until the +remaining context budget was exhausted. BUG-010 adaptive response sizing was +working as designed; increasing `num_predict` would not address this failure +mode. + +Implemented handling: + +- Extract complete group objects from a response even when its outer JSON is + truncated. +- Identify groups by normalized `canonical_text`, ordered `source_item_ids` + and normalized `merge_reason`. +- Classify three or more consecutive identical complete groups as a loop; two + identical occurrences remain an ordinary duplicate handled by deterministic + coverage repair when the JSON is valid. +- Preserve first-attempt raw text, raw Ollama JSON and repetition metadata. +- Retry at most once only when JSON parsing fails and a loop is detected. +- Preserve separate retry artifacts and fail normally if retry parsing fails. + +Related files: + +- `samples/benchmarks/progeo_north_linux_20260809_124954/` +- `samples/benchmarks/progeo_north_linux_20260809_124954/semantic_consolidator/raw_model_response.txt` +- `samples/benchmarks/progeo_north_linux_20260809_124954/semantic_consolidator/raw_ollama_response.json` +- `src/meeting_lab/consolidation/consolidate_facts.py` +- `scripts/run_meeting.py` +- `tests/test_consolidate_facts.py` + +Regression test available (yes/no): yes + +Current status: + +Verified for the repetitive-loop failure mode. A consolidator-only regression +used the preserved canonicalized input and did not run extraction, +canonicalization or rendering. + +The first attempt reproduced the original failure signature: 71,184 response +characters, `eval_count=18654`, `done_reason=length`, invalid JSON at character +71184, and a detected 192-group consecutive repetition run. This triggered the +only permitted retry. + +The retry used the same model and generation configuration, returned parseable +JSON after 7,362 evaluated tokens with `done_reason=stop`, and contained no +repetition loop (longest identical consecutive run: 1). Its source-coverage +validation before repair was invalid: 105 groups, 154 observed source-ID +occurrences, duplicate IDs, four missing IDs and one unknown ID. Deterministic +repair made 63 changes: 44 duplicate-ID removals, 15 empty-group removals and +four missing-ID singleton restorations. Validation after repair remained +invalid only because group 89 contained unknown `fact_0114`. + +The stage therefore failed cleanly after the single retry and preserved both +attempts. No third LLM call occurred. The remaining unknown-ID failure is not a +repetitive generation loop and is not silently repaired by the existing +coverage repair. This verification does not claim that general Semantic +Consolidator output quality is solved. diff --git a/scripts/run_meeting.py b/scripts/run_meeting.py index 4a7844b..233ce1d 100644 --- a/scripts/run_meeting.py +++ b/scripts/run_meeting.py @@ -43,7 +43,7 @@ from src.meeting_lab.consolidation.canonicalize import ( # noqa: E402 from src.meeting_lab.consolidation.consolidate_facts import ( # noqa: E402 build_consolidated_output, build_consolidation_prompt, - call_ollama as call_consolidator_ollama, + call_ollama_with_repetition_retry, fact_items, parse_model_json, repair_model_group_coverage, @@ -665,7 +665,7 @@ def run_semantic_consolidator_stage( prompt_token_estimate=prompt_token_estimate, num_ctx=num_ctx, ) - raw_text, response_data, runtime = call_consolidator_ollama( + raw_text, response_data, runtime, attempts = call_ollama_with_repetition_retry( endpoint=endpoint, model=model, prompt=prompt, @@ -674,15 +674,11 @@ def run_semantic_consolidator_stage( num_predict=resolved_num_predict, think=False, progress_interval=progress_interval, + output_dir=output_dir, ) - raw_response_path = output_dir / "raw_model_response.txt" - raw_json_path = output_dir / "raw_ollama_response.json" output_path = output_dir / "consolidated_extractions.json" report_path = output_dir / "report.md" repair_metadata_path = output_dir / "repair_metadata.json" - raw_response_path.write_text(raw_text + "\n", encoding="utf-8") - write_json(raw_json_path, response_data) - model_output = parse_model_json(raw_text) expected_fact_ids = {item["item_id"] for item in facts} validator_before = source_coverage_report_from_groups( @@ -734,9 +730,12 @@ def run_semantic_consolidator_stage( groups=groups, output_path=output_path, repair_changes=repair_changes, + llm_call_count=len(attempts), ) return { "runtime_seconds": runtime, + "llm_call_count": len(attempts), + "attempts": attempts, "fact_count": len(facts), "num_predict": resolved_num_predict, "validator_before": validator_before, diff --git a/src/meeting_lab/consolidation/consolidate_facts.py b/src/meeting_lab/consolidation/consolidate_facts.py index 4a55dc2..07c0412 100644 --- a/src/meeting_lab/consolidation/consolidate_facts.py +++ b/src/meeting_lab/consolidation/consolidate_facts.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse import copy import json +import re import sys import time from collections import Counter @@ -34,6 +35,15 @@ OUTPUT_TOKEN_ESTIMATE_CHARS = 4 OUTPUT_GROUP_OVERHEAD_CHARS = 320 OUTPUT_SAFETY_MARGIN = 1.35 PROMPT_NAME = "consolidate_facts.md" +REPETITION_LOOP_MIN_CONSECUTIVE_GROUPS = 3 +REPETITION_RETRY_INSTRUCTION = """ + +RETRY SAFETY INSTRUCTION: +Emit each semantic group exactly once. Before returning the JSON, check the +groups array for identical groups using canonical_text, source_item_ids and +merge_reason. If an identical group is already present, do not emit it again. +Finish the complete JSON object without repeating any group. +""".rstrip() class ConsolidationValidationError(ValueError): @@ -343,6 +353,169 @@ def parse_model_json(text: str) -> dict[str, Any]: return parsed +def group_repetition_signature(group: dict[str, Any]) -> tuple[str, tuple[str, ...], str] | None: + canonical_text = group.get("canonical_text") + source_item_ids = group.get("source_item_ids") + merge_reason = group.get("merge_reason") + if ( + not isinstance(canonical_text, str) + or not isinstance(source_item_ids, list) + or not all(isinstance(item_id, str) for item_id in source_item_ids) + or not isinstance(merge_reason, str) + ): + return None + normalize = lambda value: re.sub(r"\s+", " ", value).strip() + return ( + normalize(canonical_text), + tuple(source_item_ids), + normalize(merge_reason), + ) + + +def completed_group_objects(text: str) -> list[dict[str, Any]]: + """Extract complete JSON group objects even when the outer response is truncated.""" + groups_key = re.search(r'"groups"\s*:\s*\[', text) + if groups_key is None: + return [] + + decoder = json.JSONDecoder() + groups: list[dict[str, Any]] = [] + position = groups_key.end() + while position < len(text): + while position < len(text) and text[position] in " \t\r\n,": + position += 1 + if position >= len(text) or text[position] == "]": + break + if text[position] != "{": + break + try: + value, position = decoder.raw_decode(text, position) + except json.JSONDecodeError: + break + if not isinstance(value, dict): + break + groups.append(value) + return groups + + +def detect_repetitive_group_loop( + text: str, + minimum_consecutive: int = REPETITION_LOOP_MIN_CONSECUTIVE_GROUPS, +) -> dict[str, Any]: + groups = completed_group_objects(text) + longest_run = 0 + longest_signature: tuple[str, tuple[str, ...], str] | None = None + run_start = 0 + previous: tuple[str, tuple[str, ...], str] | None = None + current_run = 0 + + for index, group in enumerate(groups): + signature = group_repetition_signature(group) + if signature is not None and signature == previous: + current_run += 1 + else: + current_run = 1 if signature is not None else 0 + run_start = index + if current_run > longest_run: + longest_run = current_run + longest_signature = signature + longest_run_start = run_start + previous = signature + + detected = longest_signature is not None and longest_run >= minimum_consecutive + signature_data = None + if longest_signature is not None: + signature_data = { + "canonical_text": longest_signature[0], + "source_item_ids": list(longest_signature[1]), + "merge_reason": longest_signature[2], + } + return { + "detected": detected, + "threshold": minimum_consecutive, + "completed_group_count": len(groups), + "longest_consecutive_run": longest_run, + "run_start_group_index": longest_run_start if longest_run else None, + "signature": signature_data, + } + + +def call_ollama_with_repetition_retry( + *, + endpoint: str, + model: str, + prompt: str, + timeout: int, + num_ctx: int, + num_predict: int, + think: bool, + progress_interval: int, + output_dir: Path, + call_fn: Any = None, +) -> tuple[str, dict[str, Any], float, list[dict[str, Any]]]: + """Make one call, retrying once only for invalid JSON with a detected group loop.""" + if call_fn is None: + call_fn = call_ollama + output_dir.mkdir(parents=True, exist_ok=True) + attempts: list[dict[str, Any]] = [] + total_runtime = 0.0 + + for attempt_number in (1, 2): + attempt_prompt = ( + prompt + if attempt_number == 1 + else f"{prompt}\n{REPETITION_RETRY_INSTRUCTION}\n" + ) + raw_text, response_data, runtime = call_fn( + endpoint=endpoint, + model=model, + prompt=attempt_prompt, + timeout=timeout, + num_ctx=num_ctx, + num_predict=num_predict, + think=think, + progress_interval=progress_interval, + ) + total_runtime += runtime + suffix = "" if attempt_number == 1 else "_retry" + (output_dir / f"raw_model_response{suffix}.txt").write_text( + raw_text + "\n", encoding="utf-8" + ) + write_json(output_dir / f"raw_ollama_response{suffix}.json", response_data) + repetition = detect_repetitive_group_loop(raw_text) + attempt_metadata = { + "attempt": attempt_number, + "retry_instruction_added": attempt_number == 2, + "runtime_seconds": runtime, + "raw_response_chars": len(raw_text), + "raw_response_bytes": len(raw_text.encode("utf-8")), + "resolved_num_predict": num_predict, + "eval_count": response_data.get("eval_count"), + "done_reason": response_data.get("done_reason"), + "repetition": repetition, + } + attempts.append(attempt_metadata) + try: + parse_model_json(raw_text) + except ConsolidationValidationError as exc: + attempt_metadata["parse_error"] = str(exc) + write_json( + output_dir / "repetition_retry_metadata.json", + {"bug": "BUG-013", "attempts": attempts}, + ) + if attempt_number == 1 and repetition["detected"]: + continue + raise + + write_json( + output_dir / "repetition_retry_metadata.json", + {"bug": "BUG-013", "attempts": attempts}, + ) + return raw_text, response_data, total_runtime, attempts + + raise AssertionError("repetition retry loop exhausted unexpectedly") + + def validate_model_groups( model_output: dict[str, Any], expected_fact_ids: set[str], @@ -626,6 +799,7 @@ def write_report( groups: list[dict[str, Any]], output_path: Path, repair_changes: list[dict[str, Any]] | None = None, + llm_call_count: int = 1, ) -> None: merged = [group for group in groups if len(group["source_item_ids"]) > 1] singletons = [group for group in groups if len(group["source_item_ids"]) == 1] @@ -634,7 +808,7 @@ def write_report( "", "- Scope: facts only", f"- Model: `{model}`", - "- LLM call count: 1", + f"- LLM call count: {llm_call_count}", f"- Runtime: {runtime:.3f} seconds", f"- Fact item count: {fact_count}", f"- Prompt characters: {prompt_chars}", @@ -693,10 +867,10 @@ def main() -> int: print(f"Estimated prompt size chars: {prompt_chars}") print(f"Estimated prompt tokens: {prompt_token_estimate}") print(f"Resolved num_predict: {num_predict}") - print("Expected LLM call count: 1") + print("Expected LLM call count: 1; at most 2 only after detected BUG-013 loop") print("Expected runtime: 5-10 minutes on current local benchmark basis") - raw_text, _response_data, runtime = call_ollama( + raw_text, _response_data, runtime, attempts = call_ollama_with_repetition_retry( endpoint=args.endpoint, model=args.model, prompt=prompt, @@ -705,8 +879,8 @@ def main() -> int: num_predict=num_predict, think=args.think, progress_interval=args.progress_interval, + output_dir=args.output_dir, ) - raw_response_path.write_text(raw_text + "\n", encoding="utf-8") model_output = parse_model_json(raw_text) model_output, repair_changes = repair_model_group_coverage(model_output, facts) if repair_changes: @@ -736,6 +910,7 @@ def main() -> int: groups=groups, output_path=output_path, repair_changes=repair_changes, + llm_call_count=len(attempts), ) except requests.ConnectionError as exc: print(f"Error: Ollama is not reachable at {args.endpoint}: {exc}", file=sys.stderr) diff --git a/tests/test_consolidate_facts.py b/tests/test_consolidate_facts.py index 5fe3c50..afa3737 100644 --- a/tests/test_consolidate_facts.py +++ b/tests/test_consolidate_facts.py @@ -1,6 +1,8 @@ import unittest import json +import tempfile from pathlib import Path +from unittest.mock import Mock from src.meeting_lab.consolidation.consolidate_facts import ( ConsolidationValidationError, @@ -8,6 +10,8 @@ from src.meeting_lab.consolidation.consolidate_facts import ( DEFAULT_NUM_PREDICT, build_consolidated_output, build_ollama_payload, + call_ollama_with_repetition_retry, + detect_repetitive_group_loop, estimate_response_tokens, fact_items, parse_model_json, @@ -74,6 +78,14 @@ def canonicalized_fixture(): class ConsolidateFactsTests(unittest.TestCase): + @staticmethod + def group(text="A", ids=None, reason="Singleton."): + return { + "canonical_text": text, + "source_item_ids": ids or ["fact_0001"], + "merge_reason": reason, + } + def test_payload_construction_disables_streaming_and_thinking_by_default(self): payload = build_ollama_payload( model="qwen3.5:9B", @@ -174,6 +186,145 @@ class ConsolidateFactsTests(unittest.TestCase): self.assertEqual(len(groups), 2) + def test_valid_normal_consolidation_has_no_repetition_loop(self): + text = json.dumps( + {"groups": [self.group("A"), self.group("B", ["fact_0002"])]} + ) + + detection = detect_repetitive_group_loop(text) + + self.assertFalse(detection["detected"]) + self.assertEqual(detection["completed_group_count"], 2) + + def test_ordinary_duplicate_group_is_not_a_repetition_loop(self): + duplicate = self.group() + text = json.dumps({"groups": [duplicate, duplicate]}) + + detection = detect_repetitive_group_loop(text) + + self.assertFalse(detection["detected"]) + self.assertEqual(detection["longest_consecutive_run"], 2) + + def test_repetitive_identical_group_sequence_is_detected_structurally(self): + repeated = self.group("Same fact", ["fact_0001", "fact_0002"], "Equivalent.") + text = json.dumps({"groups": [repeated, repeated, repeated]}) + + detection = detect_repetitive_group_loop(text) + + self.assertTrue(detection["detected"]) + self.assertEqual(detection["longest_consecutive_run"], 3) + self.assertEqual( + detection["signature"], + { + "canonical_text": "Same fact", + "source_item_ids": ["fact_0001", "fact_0002"], + "merge_reason": "Equivalent.", + }, + ) + + def test_truncated_response_after_repetitive_loop_is_detected(self): + repeated = json.dumps(self.group(), separators=(",", ":")) + text = '{"groups":[' + ",".join([repeated] * 4) + ',{"canonical_text":' + + detection = detect_repetitive_group_loop(text) + + self.assertTrue(detection["detected"]) + self.assertEqual(detection["completed_group_count"], 4) + + def test_detected_invalid_loop_gets_one_controlled_retry(self): + repeated = json.dumps(self.group(), separators=(",", ":")) + truncated = '{"groups":[' + ",".join([repeated] * 3) + ',{"canonical_text":' + valid = json.dumps({"groups": [self.group()]}) + call_fn = Mock( + side_effect=[ + (truncated, {"eval_count": 100, "done_reason": "length"}, 1.0), + (valid, {"eval_count": 20, "done_reason": "stop"}, 0.5), + ] + ) + with tempfile.TemporaryDirectory() as directory: + raw_text, _data, runtime, attempts = call_ollama_with_repetition_retry( + endpoint="http://example.invalid", + model="test-model", + prompt="base prompt", + timeout=1, + num_ctx=1000, + num_predict=100, + think=False, + progress_interval=1, + output_dir=Path(directory), + call_fn=call_fn, + ) + + self.assertTrue((Path(directory) / "raw_model_response.txt").exists()) + self.assertTrue((Path(directory) / "raw_model_response_retry.txt").exists()) + metadata = json.loads( + (Path(directory) / "repetition_retry_metadata.json").read_text() + ) + + self.assertEqual(raw_text, valid) + self.assertEqual(runtime, 1.5) + self.assertEqual(len(attempts), 2) + self.assertEqual(call_fn.call_count, 2) + self.assertEqual(metadata["attempts"][0]["repetition"]["detected"], True) + first_prompt = call_fn.call_args_list[0].kwargs["prompt"] + retry_prompt = call_fn.call_args_list[1].kwargs["prompt"] + self.assertEqual(first_prompt, "base prompt") + self.assertIn("Emit each semantic group exactly once", retry_prompt) + + def test_unrelated_malformed_json_is_not_retried(self): + call_fn = Mock(return_value=("{invalid", {"done_reason": "length"}, 0.1)) + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ConsolidationValidationError): + call_ollama_with_repetition_retry( + endpoint="http://example.invalid", + model="test-model", + prompt="prompt", + timeout=1, + num_ctx=1000, + num_predict=100, + think=False, + progress_interval=1, + output_dir=Path(directory), + call_fn=call_fn, + ) + + self.assertEqual(call_fn.call_count, 1) + + def test_retry_failure_preserves_both_attempts_and_raises(self): + repeated = json.dumps(self.group(), separators=(",", ":")) + truncated = '{"groups":[' + ",".join([repeated] * 3) + ',{"canonical_text":' + call_fn = Mock( + side_effect=[ + (truncated, {"done_reason": "length"}, 1.0), + (truncated, {"done_reason": "length"}, 1.0), + ] + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaises(ConsolidationValidationError): + call_ollama_with_repetition_retry( + endpoint="http://example.invalid", + model="test-model", + prompt="prompt", + timeout=1, + num_ctx=1000, + num_predict=100, + think=False, + progress_interval=1, + output_dir=root, + call_fn=call_fn, + ) + + self.assertTrue((root / "raw_model_response.txt").exists()) + self.assertTrue((root / "raw_ollama_response.json").exists()) + self.assertTrue((root / "raw_model_response_retry.txt").exists()) + self.assertTrue((root / "raw_ollama_response_retry.json").exists()) + metadata = json.loads((root / "repetition_retry_metadata.json").read_text()) + + self.assertEqual(call_fn.call_count, 2) + self.assertEqual(len(metadata["attempts"]), 2) + self.assertIn("parse_error", metadata["attempts"][1]) + def test_no_missing_source_ids(self): with self.assertRaisesRegex(ConsolidationValidationError, "Missing"): validate_model_groups(