Add windowed segmentation pipeline and review tooling
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
{
|
||||
"source": {
|
||||
"file": "chunk_01_normalized.txt",
|
||||
"block_count": 85,
|
||||
"target_block_chars": 600
|
||||
},
|
||||
"configuration": {
|
||||
"model": "qwen3:8b",
|
||||
"window_size": 20,
|
||||
"window_overlap": 3,
|
||||
"temperature": 0.0
|
||||
},
|
||||
"topic_changes": [
|
||||
17,
|
||||
24,
|
||||
27,
|
||||
31,
|
||||
47,
|
||||
49,
|
||||
51,
|
||||
68,
|
||||
74,
|
||||
80,
|
||||
83
|
||||
],
|
||||
"segments": [
|
||||
{
|
||||
"segment_id": "segment_001",
|
||||
"start_block": 1,
|
||||
"end_block": 17
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_002",
|
||||
"start_block": 18,
|
||||
"end_block": 24
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_003",
|
||||
"start_block": 25,
|
||||
"end_block": 27
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_004",
|
||||
"start_block": 28,
|
||||
"end_block": 31
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_005",
|
||||
"start_block": 32,
|
||||
"end_block": 47
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_006",
|
||||
"start_block": 48,
|
||||
"end_block": 49
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_007",
|
||||
"start_block": 50,
|
||||
"end_block": 51
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_008",
|
||||
"start_block": 52,
|
||||
"end_block": 68
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_009",
|
||||
"start_block": 69,
|
||||
"end_block": 74
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_010",
|
||||
"start_block": 75,
|
||||
"end_block": 80
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_011",
|
||||
"start_block": 81,
|
||||
"end_block": 83
|
||||
},
|
||||
{
|
||||
"segment_id": "segment_012",
|
||||
"start_block": 84,
|
||||
"end_block": 85
|
||||
}
|
||||
],
|
||||
"windows": [
|
||||
{
|
||||
"window_number": 1,
|
||||
"start_block": 1,
|
||||
"end_block": 20,
|
||||
"decision_start": 1,
|
||||
"decision_end": 17,
|
||||
"topic_changes": [
|
||||
17
|
||||
],
|
||||
"_run": {
|
||||
"model": "qwen3:8b",
|
||||
"elapsed_seconds": 26.392,
|
||||
"total_duration_ns": 26341490458,
|
||||
"load_duration_ns": 3196577000,
|
||||
"prompt_eval_count": 4093,
|
||||
"prompt_eval_duration_ns": 22488752000,
|
||||
"eval_count": 12,
|
||||
"eval_duration_ns": 638303000
|
||||
}
|
||||
},
|
||||
{
|
||||
"window_number": 2,
|
||||
"start_block": 18,
|
||||
"end_block": 37,
|
||||
"decision_start": 18,
|
||||
"decision_end": 34,
|
||||
"topic_changes": [
|
||||
24,
|
||||
27,
|
||||
31
|
||||
],
|
||||
"_run": {
|
||||
"model": "qwen3:8b",
|
||||
"elapsed_seconds": 22.944,
|
||||
"total_duration_ns": 22877936583,
|
||||
"load_duration_ns": 123779000,
|
||||
"prompt_eval_count": 3949,
|
||||
"prompt_eval_duration_ns": 21491350000,
|
||||
"eval_count": 20,
|
||||
"eval_duration_ns": 1087620000
|
||||
}
|
||||
},
|
||||
{
|
||||
"window_number": 3,
|
||||
"start_block": 35,
|
||||
"end_block": 54,
|
||||
"decision_start": 35,
|
||||
"decision_end": 51,
|
||||
"topic_changes": [
|
||||
47,
|
||||
49,
|
||||
51
|
||||
],
|
||||
"_run": {
|
||||
"model": "qwen3:8b",
|
||||
"elapsed_seconds": 24.082,
|
||||
"total_duration_ns": 24043266583,
|
||||
"load_duration_ns": 125136708,
|
||||
"prompt_eval_count": 4090,
|
||||
"prompt_eval_duration_ns": 22621129000,
|
||||
"eval_count": 20,
|
||||
"eval_duration_ns": 1097246000
|
||||
}
|
||||
},
|
||||
{
|
||||
"window_number": 4,
|
||||
"start_block": 52,
|
||||
"end_block": 71,
|
||||
"decision_start": 52,
|
||||
"decision_end": 68,
|
||||
"topic_changes": [
|
||||
68
|
||||
],
|
||||
"_run": {
|
||||
"model": "qwen3:8b",
|
||||
"elapsed_seconds": 24.062,
|
||||
"total_duration_ns": 24038158167,
|
||||
"load_duration_ns": 122426834,
|
||||
"prompt_eval_count": 4093,
|
||||
"prompt_eval_duration_ns": 23031993000,
|
||||
"eval_count": 12,
|
||||
"eval_duration_ns": 633425000
|
||||
}
|
||||
},
|
||||
{
|
||||
"window_number": 5,
|
||||
"start_block": 69,
|
||||
"end_block": 85,
|
||||
"decision_start": 69,
|
||||
"decision_end": 84,
|
||||
"topic_changes": [
|
||||
74,
|
||||
80,
|
||||
83
|
||||
],
|
||||
"_run": {
|
||||
"model": "qwen3:8b",
|
||||
"elapsed_seconds": 21.497,
|
||||
"total_duration_ns": 21473075042,
|
||||
"load_duration_ns": 126962584,
|
||||
"prompt_eval_count": 3481,
|
||||
"prompt_eval_duration_ns": 20075508000,
|
||||
"eval_count": 20,
|
||||
"eval_duration_ns": 1065423000
|
||||
}
|
||||
}
|
||||
],
|
||||
"_run": {
|
||||
"elapsed_seconds": 118.991
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user