admin
03bc6b1d90
Document and preserve the controlled Progeo benchmark series. Whisper comparison: - Compare whisper.cpp large-v3 and large-v3-turbo - Identify and verify BUG-012: extraction stability depends on chunk size - Repeat both transcription variants with identical reduced chunk budgets - Select large-v3-turbo as the current production transcription model LLM comparison: - Compare qwen3.5:9b with qwen3.5:35B-A3B - Preserve identical transcript, Meeting Context, prompts and chunking - Retain qwen3.5:9b as the production recommendation - Record runtime, repair burden and semantic-quality findings Current production benchmark configuration: - whisper.cpp large-v3-turbo - target_chars=4500 - max_chars=5500 - min_chars=2500 - overlap_blocks=0 - qwen3.5:9b - num_ctx=32768 - think=false - temperature=0 BUG-012 remains verified but not yet fixed in the production chunker.
91 lines
2.4 KiB
JSON
91 lines
2.4 KiB
JSON
{
|
|
"preconditions": {
|
|
"branch": "feature/windowed-segmentation",
|
|
"commit": "dcc3a5c734169d6615959bcc05e6d90667de8fac",
|
|
"meeting_context": "samples/real_live/progeo_meeting/meeting_context.yaml",
|
|
"meeting_context_validation": "passed with .venv Python/PyYAML",
|
|
"ollama_model": "qwen3.5:9b available",
|
|
"ollama_ps_before": "no model jobs listed by ollama ps",
|
|
"settings": {
|
|
"think": false,
|
|
"temperature": 0,
|
|
"num_ctx": 32768,
|
|
"model": "qwen3.5:9b",
|
|
"semantic_retries": 0,
|
|
"manual_intervention": false
|
|
}
|
|
},
|
|
"input": "samples/real_live/progeo_meeting/progeo_whispercpp_vulkan_turbo_trimmed_converted.json",
|
|
"metrics": {
|
|
"input_segments": 1211,
|
|
"input_segment_text_chars": 83473,
|
|
"input_text_chars": 84683,
|
|
"chunk_count": 10,
|
|
"normalized_chunk_count": 10,
|
|
"extraction_files": 3,
|
|
"extracted_counts": {
|
|
"facts": 25,
|
|
"decisions": 4,
|
|
"todos": 8,
|
|
"questions": 3,
|
|
"positions": 0,
|
|
"technical": 12
|
|
},
|
|
"canonical_stats": {},
|
|
"canonical_item_counts": {},
|
|
"consolidator_fact_group_count": null,
|
|
"repair_count": null,
|
|
"repair_ops": [],
|
|
"renderer_validation": null,
|
|
"timings": {
|
|
"normalization": {
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\normalization_stdout_stderr.txt",
|
|
"runtime_seconds": 0.788
|
|
},
|
|
"chunking": {
|
|
"exit_code": 0,
|
|
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\chunking_stdout_stderr.txt",
|
|
"runtime_seconds": 0.269
|
|
},
|
|
"extraction": {
|
|
"runtime_seconds": 198.349,
|
|
"exit_code": 1,
|
|
"log": "samples\\benchmarks\\progeo_whisper_turbo_20260805_120903\\extraction_stdout_stderr.txt"
|
|
}
|
|
},
|
|
"term_hits": {
|
|
"Extruder": 17,
|
|
"Verstrecken": 3,
|
|
"Streckwerk": 3,
|
|
"Reckwerk": 4,
|
|
"Stabgeometrie": 3,
|
|
"Orangenhaut": 2,
|
|
"Virgin-Material": 4,
|
|
"Masterbatch": 2,
|
|
"MFI": 5,
|
|
"Viskosit?t": 0,
|
|
"Viskosit??t": 0,
|
|
"Regranulat": 1,
|
|
"Trommelsieb": 1,
|
|
"Martin": 5,
|
|
"Gotthard": 0,
|
|
"James": 5,
|
|
"David": 4,
|
|
"Tim": 23,
|
|
"Antonius": 3,
|
|
"Marleen": 5,
|
|
"Henning": 4,
|
|
"MAS": 25,
|
|
"Kockmann": 11,
|
|
"Fraunhofer": 3,
|
|
"Kiwa": 0,
|
|
"Universit?t Leeds": 0,
|
|
"Universit??t Leeds": 0,
|
|
"100 %": 0,
|
|
"20 %": 0,
|
|
"126.000": 2
|
|
}
|
|
}
|
|
}
|