admin
03bc6b1d90
Document and preserve the controlled Progeo benchmark series. Whisper comparison: - Compare whisper.cpp large-v3 and large-v3-turbo - Identify and verify BUG-012: extraction stability depends on chunk size - Repeat both transcription variants with identical reduced chunk budgets - Select large-v3-turbo as the current production transcription model LLM comparison: - Compare qwen3.5:9b with qwen3.5:35B-A3B - Preserve identical transcript, Meeting Context, prompts and chunking - Retain qwen3.5:9b as the production recommendation - Record runtime, repair burden and semantic-quality findings Current production benchmark configuration: - whisper.cpp large-v3-turbo - target_chars=4500 - max_chars=5500 - min_chars=2500 - overlap_blocks=0 - qwen3.5:9b - num_ctx=32768 - think=false - temperature=0 BUG-012 remains verified but not yet fixed in the production chunker.
198 lines
5.2 KiB
JSON
198 lines
5.2 KiB
JSON
{
|
|
"large": {
|
|
"input_segments": 3822,
|
|
"input_segment_text_chars": 216500,
|
|
"input_text_chars": 220321,
|
|
"chunk_count": 25,
|
|
"normalized_chunk_count": 25,
|
|
"extraction_files": 25,
|
|
"extracted_counts": {
|
|
"facts": 138,
|
|
"decisions": 1,
|
|
"todos": 50,
|
|
"questions": 36,
|
|
"positions": 0,
|
|
"technical": 64
|
|
},
|
|
"canonical_stats": {
|
|
"input_item_count": 289,
|
|
"output_item_count": 288,
|
|
"input_item_count_by_category": {
|
|
"fact": 138,
|
|
"decision": 1,
|
|
"action_item": 50,
|
|
"open_question": 36,
|
|
"position": 0,
|
|
"technical_detail": 64
|
|
},
|
|
"output_item_count_by_category": {
|
|
"fact": 138,
|
|
"decision": 1,
|
|
"action_item": 49,
|
|
"open_question": 36,
|
|
"position": 0,
|
|
"technical_detail": 64
|
|
},
|
|
"exact_duplicates_merged": 1,
|
|
"responsibility_validation_count": 5,
|
|
"responsibility_rejection_count": 4,
|
|
"responsibility_normalization_count": 1
|
|
},
|
|
"canonical_item_counts": {
|
|
"fact": 138,
|
|
"action_item": 49,
|
|
"open_question": 36,
|
|
"technical_detail": 64,
|
|
"decision": 1
|
|
},
|
|
"consolidator_fact_group_count": 110,
|
|
"repair_count": null,
|
|
"repair_ops": [],
|
|
"renderer_validation": {
|
|
"valid": false,
|
|
"violations": [
|
|
{
|
|
"type": "missing_required_heading",
|
|
"expected": "# Working Protocol",
|
|
"reason": "document must begin exactly with '# Working Protocol'"
|
|
}
|
|
]
|
|
},
|
|
"timings": {
|
|
"normalization": {
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\normalization_stdout_stderr.txt",
|
|
"runtime_seconds": 1.906
|
|
},
|
|
"chunking": {
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\chunking_stdout_stderr.txt",
|
|
"runtime_seconds": 0.275
|
|
},
|
|
"extraction": {
|
|
"runtime_seconds": 510.856,
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\extraction_stdout_stderr.txt"
|
|
},
|
|
"canonicalizer": {
|
|
"runtime_seconds": 0.259,
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\canonicalizer_stdout_stderr.txt"
|
|
},
|
|
"semantic_consolidator": {
|
|
"runtime_seconds": 117.423,
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\semantic_consolidator_stdout_stderr.txt"
|
|
},
|
|
"renderer": {
|
|
"runtime_seconds": 26.675,
|
|
"exit_code": 1,
|
|
"log": "samples\\benchmarks\\progeo_whisper_large_v3_20260805_120903\\renderer_stdout_stderr.txt"
|
|
}
|
|
},
|
|
"term_hits": {
|
|
"Extruder": 36,
|
|
"Verstrecken": 2,
|
|
"Streckwerk": 3,
|
|
"Reckwerk": 4,
|
|
"Stabgeometrie": 2,
|
|
"Orangenhaut": 2,
|
|
"Virgin-Material": 0,
|
|
"Masterbatch": 1,
|
|
"MFI": 7,
|
|
"Viskosit?t": 0,
|
|
"Viskosit??t": 0,
|
|
"Regranulat": 3,
|
|
"Trommelsieb": 3,
|
|
"Martin": 15,
|
|
"Gotthard": 0,
|
|
"James": 9,
|
|
"David": 12,
|
|
"Tim": 57,
|
|
"Antonius": 7,
|
|
"Marleen": 0,
|
|
"Henning": 10,
|
|
"MAS": 21,
|
|
"Kockmann": 12,
|
|
"Fraunhofer": 9,
|
|
"Kiwa": 0,
|
|
"Universit?t Leeds": 0,
|
|
"Universit??t Leeds": 0,
|
|
"100 %": 0,
|
|
"20 %": 0,
|
|
"126.000": 4
|
|
}
|
|
},
|
|
"turbo": {
|
|
"input_segments": 1211,
|
|
"input_segment_text_chars": 83473,
|
|
"input_text_chars": 84683,
|
|
"chunk_count": 10,
|
|
"normalized_chunk_count": 10,
|
|
"extraction_files": 3,
|
|
"extracted_counts": {
|
|
"facts": 25,
|
|
"decisions": 4,
|
|
"todos": 8,
|
|
"questions": 3,
|
|
"positions": 0,
|
|
"technical": 12
|
|
},
|
|
"canonical_stats": {},
|
|
"canonical_item_counts": {},
|
|
"consolidator_fact_group_count": null,
|
|
"repair_count": null,
|
|
"repair_ops": [],
|
|
"renderer_validation": null,
|
|
"timings": {
|
|
"normalization": {
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\normalization_stdout_stderr.txt",
|
|
"runtime_seconds": 0.788
|
|
},
|
|
"chunking": {
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\chunking_stdout_stderr.txt",
|
|
"runtime_seconds": 0.269
|
|
},
|
|
"extraction": {
|
|
"runtime_seconds": 198.349,
|
|
"exit_code": 1,
|
|
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\extraction_stdout_stderr.txt"
|
|
}
|
|
},
|
|
"term_hits": {
|
|
"Extruder": 17,
|
|
"Verstrecken": 3,
|
|
"Streckwerk": 3,
|
|
"Reckwerk": 4,
|
|
"Stabgeometrie": 3,
|
|
"Orangenhaut": 2,
|
|
"Virgin-Material": 4,
|
|
"Masterbatch": 2,
|
|
"MFI": 5,
|
|
"Viskosit?t": 0,
|
|
"Viskosit??t": 0,
|
|
"Regranulat": 1,
|
|
"Trommelsieb": 1,
|
|
"Martin": 5,
|
|
"Gotthard": 0,
|
|
"James": 5,
|
|
"David": 4,
|
|
"Tim": 23,
|
|
"Antonius": 3,
|
|
"Marleen": 5,
|
|
"Henning": 4,
|
|
"MAS": 25,
|
|
"Kockmann": 11,
|
|
"Fraunhofer": 3,
|
|
"Kiwa": 0,
|
|
"Universit?t Leeds": 0,
|
|
"Universit??t Leeds": 0,
|
|
"100 %": 0,
|
|
"20 %": 0,
|
|
"126.000": 2
|
|
}
|
|
}
|
|
}
|