Add controlled Whisper and LLM benchmark results
Document and preserve the controlled Progeo benchmark series. Whisper comparison: - Compare whisper.cpp large-v3 and large-v3-turbo - Identify and verify BUG-012: extraction stability depends on chunk size - Repeat both transcription variants with identical reduced chunk budgets - Select large-v3-turbo as the current production transcription model LLM comparison: - Compare qwen3.5:9b with qwen3.5:35B-A3B - Preserve identical transcript, Meeting Context, prompts and chunking - Retain qwen3.5:9b as the production recommendation - Record runtime, repair burden and semantic-quality findings Current production benchmark configuration: - whisper.cpp large-v3-turbo - target_chars=4500 - max_chars=5500 - min_chars=2500 - overlap_blocks=0 - qwen3.5:9b - num_ctx=32768 - think=false - temperature=0 BUG-012 remains verified but not yet fixed in the production chunker.
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
{
|
||||
"large": {
|
||||
"input_segments": 3822,
|
||||
"input_segment_text_chars": 216500,
|
||||
"input_text_chars": 220321,
|
||||
"chunk_count": 25,
|
||||
"normalized_chunk_count": 25,
|
||||
"extraction_files": 25,
|
||||
"extracted_counts": {
|
||||
"facts": 138,
|
||||
"decisions": 1,
|
||||
"todos": 50,
|
||||
"questions": 36,
|
||||
"positions": 0,
|
||||
"technical": 64
|
||||
},
|
||||
"canonical_stats": {
|
||||
"input_item_count": 289,
|
||||
"output_item_count": 288,
|
||||
"input_item_count_by_category": {
|
||||
"fact": 138,
|
||||
"decision": 1,
|
||||
"action_item": 50,
|
||||
"open_question": 36,
|
||||
"position": 0,
|
||||
"technical_detail": 64
|
||||
},
|
||||
"output_item_count_by_category": {
|
||||
"fact": 138,
|
||||
"decision": 1,
|
||||
"action_item": 49,
|
||||
"open_question": 36,
|
||||
"position": 0,
|
||||
"technical_detail": 64
|
||||
},
|
||||
"exact_duplicates_merged": 1,
|
||||
"responsibility_validation_count": 5,
|
||||
"responsibility_rejection_count": 4,
|
||||
"responsibility_normalization_count": 1
|
||||
},
|
||||
"canonical_item_counts": {
|
||||
"fact": 138,
|
||||
"action_item": 49,
|
||||
"open_question": 36,
|
||||
"technical_detail": 64,
|
||||
"decision": 1
|
||||
},
|
||||
"consolidator_fact_group_count": 110,
|
||||
"repair_count": null,
|
||||
"repair_ops": [],
|
||||
"renderer_validation": {
|
||||
"valid": false,
|
||||
"violations": [
|
||||
{
|
||||
"type": "missing_required_heading",
|
||||
"expected": "# Working Protocol",
|
||||
"reason": "document must begin exactly with '# Working Protocol'"
|
||||
}
|
||||
]
|
||||
},
|
||||
"timings": {
|
||||
"normalization": {
|
||||
"exit_code": 0,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\normalization_stdout_stderr.txt",
|
||||
"runtime_seconds": 1.906
|
||||
},
|
||||
"chunking": {
|
||||
"exit_code": 0,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\chunking_stdout_stderr.txt",
|
||||
"runtime_seconds": 0.275
|
||||
},
|
||||
"extraction": {
|
||||
"runtime_seconds": 510.856,
|
||||
"exit_code": 0,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\extraction_stdout_stderr.txt"
|
||||
},
|
||||
"canonicalizer": {
|
||||
"runtime_seconds": 0.259,
|
||||
"exit_code": 0,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\canonicalizer_stdout_stderr.txt"
|
||||
},
|
||||
"semantic_consolidator": {
|
||||
"runtime_seconds": 117.423,
|
||||
"exit_code": 0,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\semantic_consolidator_stdout_stderr.txt"
|
||||
},
|
||||
"renderer": {
|
||||
"runtime_seconds": 26.675,
|
||||
"exit_code": 1,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\renderer_stdout_stderr.txt"
|
||||
}
|
||||
},
|
||||
"term_hits": {
|
||||
"Extruder": 36,
|
||||
"Verstrecken": 2,
|
||||
"Streckwerk": 3,
|
||||
"Reckwerk": 4,
|
||||
"Stabgeometrie": 2,
|
||||
"Orangenhaut": 2,
|
||||
"Virgin-Material": 0,
|
||||
"Masterbatch": 1,
|
||||
"MFI": 7,
|
||||
"Viskosit?t": 0,
|
||||
"Viskosit??t": 0,
|
||||
"Regranulat": 3,
|
||||
"Trommelsieb": 3,
|
||||
"Martin": 15,
|
||||
"Gotthard": 0,
|
||||
"James": 9,
|
||||
"David": 12,
|
||||
"Tim": 57,
|
||||
"Antonius": 7,
|
||||
"Marleen": 0,
|
||||
"Henning": 10,
|
||||
"MAS": 21,
|
||||
"Kockmann": 12,
|
||||
"Fraunhofer": 9,
|
||||
"Kiwa": 0,
|
||||
"Universit?t Leeds": 0,
|
||||
"Universit??t Leeds": 0,
|
||||
"100 %": 0,
|
||||
"20 %": 0,
|
||||
"126.000": 4
|
||||
}
|
||||
},
|
||||
"turbo": {
|
||||
"input_segments": 1211,
|
||||
"input_segment_text_chars": 83473,
|
||||
"input_text_chars": 84683,
|
||||
"chunk_count": 10,
|
||||
"normalized_chunk_count": 10,
|
||||
"extraction_files": 3,
|
||||
"extracted_counts": {
|
||||
"facts": 25,
|
||||
"decisions": 4,
|
||||
"todos": 8,
|
||||
"questions": 3,
|
||||
"positions": 0,
|
||||
"technical": 12
|
||||
},
|
||||
"canonical_stats": {},
|
||||
"canonical_item_counts": {},
|
||||
"consolidator_fact_group_count": null,
|
||||
"repair_count": null,
|
||||
"repair_ops": [],
|
||||
"renderer_validation": null,
|
||||
"timings": {
|
||||
"normalization": {
|
||||
"exit_code": 0,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\normalization_stdout_stderr.txt",
|
||||
"runtime_seconds": 0.788
|
||||
},
|
||||
"chunking": {
|
||||
"exit_code": 0,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\chunking_stdout_stderr.txt",
|
||||
"runtime_seconds": 0.269
|
||||
},
|
||||
"extraction": {
|
||||
"runtime_seconds": 198.349,
|
||||
"exit_code": 1,
|
||||
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\extraction_stdout_stderr.txt"
|
||||
}
|
||||
},
|
||||
"term_hits": {
|
||||
"Extruder": 17,
|
||||
"Verstrecken": 3,
|
||||
"Streckwerk": 3,
|
||||
"Reckwerk": 4,
|
||||
"Stabgeometrie": 3,
|
||||
"Orangenhaut": 2,
|
||||
"Virgin-Material": 4,
|
||||
"Masterbatch": 2,
|
||||
"MFI": 5,
|
||||
"Viskosit?t": 0,
|
||||
"Viskosit??t": 0,
|
||||
"Regranulat": 1,
|
||||
"Trommelsieb": 1,
|
||||
"Martin": 5,
|
||||
"Gotthard": 0,
|
||||
"James": 5,
|
||||
"David": 4,
|
||||
"Tim": 23,
|
||||
"Antonius": 3,
|
||||
"Marleen": 5,
|
||||
"Henning": 4,
|
||||
"MAS": 25,
|
||||
"Kockmann": 11,
|
||||
"Fraunhofer": 3,
|
||||
"Kiwa": 0,
|
||||
"Universit?t Leeds": 0,
|
||||
"Universit??t Leeds": 0,
|
||||
"100 %": 0,
|
||||
"20 %": 0,
|
||||
"126.000": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user