Detect topic boundaries as continuous segments
This commit is contained in:
@@ -1,20 +1,23 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
Segment a normalized meeting transcript into discussion topics using Ollama.
|
Detect topic boundaries in a normalized meeting transcript using Ollama.
|
||||||
|
|
||||||
The script numbers the transcript blocks and asks the language model only to
|
The language model only identifies positions at which the discussion topic
|
||||||
identify the thematic structure of the discussion.
|
changes. Python then converts those boundaries into continuous, complete and
|
||||||
|
non-overlapping segments.
|
||||||
|
|
||||||
It does not extract facts, decisions, todos, questions or summaries.
|
The script does not:
|
||||||
|
|
||||||
Examples:
|
- collect or merge topics,
|
||||||
|
- identify resumed topics,
|
||||||
|
- extract facts,
|
||||||
|
- extract decisions,
|
||||||
|
- extract todos,
|
||||||
|
- summarize the discussion.
|
||||||
|
|
||||||
|
Example:
|
||||||
python src/meeting_lab/segmentation/segment_topics.py \
|
python src/meeting_lab/segmentation/segment_topics.py \
|
||||||
samples/chunks/chunk_01_normalized.txt
|
samples/chunks/chunk_01_normalized.txt
|
||||||
|
|
||||||
python src/meeting_lab/segmentation/segment_topics.py \
|
|
||||||
samples/chunks/chunk_01_normalized.txt \
|
|
||||||
--model qwen3:8b \
|
|
||||||
--output experiments/topic-segmentation/chunk_01_topics.json
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -32,131 +35,209 @@ import requests
|
|||||||
|
|
||||||
DEFAULT_MODEL = "qwen3:8b"
|
DEFAULT_MODEL = "qwen3:8b"
|
||||||
DEFAULT_ENDPOINT = "http://localhost:11434/api/generate"
|
DEFAULT_ENDPOINT = "http://localhost:11434/api/generate"
|
||||||
|
DEFAULT_TARGET_BLOCK_CHARS = 600
|
||||||
|
|
||||||
|
|
||||||
SYSTEM_INSTRUCTION = """
|
SYSTEM_INSTRUCTION = """
|
||||||
Du analysierst die thematische Struktur eines Meeting-Transkripts.
|
Du analysierst den chronologischen Verlauf eines Meeting-Transkripts.
|
||||||
|
|
||||||
Deine einzige Aufgabe ist die Themensegmentierung.
|
Deine einzige Aufgabe ist es, Stellen zu erkennen, an denen das Gespräch
|
||||||
|
inhaltlich zu einem anderen Thema wechselt.
|
||||||
|
|
||||||
Du sollst erkennen:
|
Du erhältst nummerierte Textblöcke.
|
||||||
|
|
||||||
1. wann ein Thema beginnt,
|
Gib ausschließlich die Nummern der Blöcke zurück, nach denen ein echter
|
||||||
2. wann ein Thema endet,
|
Themenwechsel stattfindet.
|
||||||
3. wann zu einem anderen Thema gewechselt wird,
|
|
||||||
4. wann ein früheres Thema erneut aufgenommen wird.
|
Beispiel:
|
||||||
|
|
||||||
|
Wenn die Blöcke 1 bis 4 ein Thema behandeln und mit Block 5 ein neues Thema
|
||||||
|
beginnt, gib 4 als Grenze zurück.
|
||||||
|
|
||||||
Regeln:
|
Regeln:
|
||||||
|
|
||||||
1. Extrahiere keine Fakten.
|
1. Gib nur echte inhaltliche Themenwechsel zurück.
|
||||||
2. Extrahiere keine Aufgaben.
|
2. Kleine Rückfragen sind kein Themenwechsel.
|
||||||
3. Extrahiere keine Entscheidungen.
|
3. Kurze Ergänzungen sind kein Themenwechsel.
|
||||||
4. Extrahiere keine offenen Fragen.
|
4. Sprecherwechsel sind kein Themenwechsel.
|
||||||
5. Fasse die Diskussion nicht zusammen.
|
5. Zustimmung oder Ablehnung sind kein Themenwechsel.
|
||||||
6. Bewerte die Aussagen nicht.
|
6. Eine kurze Abschweifung ist nur dann ein eigenes Segment, wenn sie einen
|
||||||
7. Erfinde keine Themen, die im Text nicht behandelt werden.
|
erkennbaren Gesprächsabschnitt bildet.
|
||||||
8. Verwende möglichst kurze und sachliche Themenbezeichnungen.
|
7. Organisatorische Einleitungen oder Verabschiedungen dürfen eigene Segmente
|
||||||
9. Jeder Block muss genau einem Thema zugeordnet werden.
|
bilden.
|
||||||
10. Mehrere direkt aufeinanderfolgende Blöcke desselben Themas bilden ein Segment.
|
8. Fasse keine Inhalte zusammen.
|
||||||
11. Wird ein früheres Thema später wieder aufgenommen, verwende dieselbe topic_id.
|
9. Vergib keine Themenbezeichnungen.
|
||||||
12. Kleine Rückfragen oder kurze Ergänzungen bleiben beim aktuellen Thema,
|
10. Extrahiere keine Fakten, Aufgaben, Entscheidungen oder Fragen.
|
||||||
sofern sie kein eigenständiges Thema eröffnen.
|
11. Die letzte Blocknummer darf nicht als Themenwechsel ausgegeben werden.
|
||||||
13. Organisatorische Übergänge dürfen als eigenes Thema erfasst werden, wenn
|
12. Die Blocknummern müssen aufsteigend und eindeutig sein.
|
||||||
sie einen erkennbaren Gesprächsabschnitt bilden.
|
13. Gib ausschließlich gültiges JSON aus.
|
||||||
14. Gib ausschließlich gültiges JSON aus.
|
14. Gib kein Markdown und keine zusätzlichen Erläuterungen aus.
|
||||||
15. Gib kein Markdown und keine Erläuterungen außerhalb des JSON aus.
|
|
||||||
""".strip()
|
""".strip()
|
||||||
|
|
||||||
|
|
||||||
OUTPUT_SCHEMA = {
|
TOPIC_CHANGE_JSON_SCHEMA: dict[str, Any] = {
|
||||||
"topics": [
|
"type": "object",
|
||||||
{
|
"properties": {
|
||||||
"topic_id": "topic_001",
|
"topic_changes": {
|
||||||
"title": "Kurze sachliche Themenbezeichnung",
|
"type": "array",
|
||||||
"segments": [
|
"items": {
|
||||||
{
|
"type": "integer",
|
||||||
"start_block": 1,
|
"minimum": 1,
|
||||||
"end_block": 5,
|
},
|
||||||
}
|
}
|
||||||
],
|
},
|
||||||
}
|
"required": ["topic_changes"],
|
||||||
]
|
"additionalProperties": False,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Segment a meeting transcript into discussion topics."
|
description=(
|
||||||
|
"Detect topic boundaries in a normalized meeting transcript."
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"input_file",
|
"input_file",
|
||||||
type=Path,
|
type=Path,
|
||||||
help="Normalized transcript or transcript chunk",
|
help="Normalized transcript file",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-o",
|
"-o",
|
||||||
"--output",
|
"--output",
|
||||||
type=Path,
|
type=Path,
|
||||||
help="Output JSON file; default: <input>_topics.json",
|
help="Output JSON file; default: <input>_segments.json",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--model",
|
"--model",
|
||||||
default=DEFAULT_MODEL,
|
default=DEFAULT_MODEL,
|
||||||
help=f"Ollama model name (default: {DEFAULT_MODEL})",
|
help=f"Ollama model name (default: {DEFAULT_MODEL})",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--endpoint",
|
"--endpoint",
|
||||||
default=DEFAULT_ENDPOINT,
|
default=DEFAULT_ENDPOINT,
|
||||||
help=f"Ollama generate endpoint (default: {DEFAULT_ENDPOINT})",
|
help=f"Ollama generate endpoint (default: {DEFAULT_ENDPOINT})",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--timeout",
|
"--timeout",
|
||||||
type=int,
|
type=int,
|
||||||
default=1800,
|
default=1800,
|
||||||
help="HTTP timeout in seconds (default: 1800)",
|
help="HTTP timeout in seconds (default: 1800)",
|
||||||
)
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--temperature",
|
"--temperature",
|
||||||
type=float,
|
type=float,
|
||||||
default=0.0,
|
default=0.0,
|
||||||
help="Sampling temperature (default: 0.0)",
|
help="Sampling temperature (default: 0.0)",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--target-block-chars",
|
||||||
|
type=int,
|
||||||
|
default=DEFAULT_TARGET_BLOCK_CHARS,
|
||||||
|
help=(
|
||||||
|
"Approximate size of analysis blocks in characters "
|
||||||
|
f"(default: {DEFAULT_TARGET_BLOCK_CHARS})"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
def split_into_blocks(text: str) -> list[str]:
|
def split_into_blocks(
|
||||||
|
text: str,
|
||||||
|
target_chars: int = DEFAULT_TARGET_BLOCK_CHARS,
|
||||||
|
) -> list[str]:
|
||||||
"""
|
"""
|
||||||
Split a transcript into stable, numbered discussion blocks.
|
Combine short transcript utterances into larger analysis blocks.
|
||||||
|
|
||||||
Blank-line-separated paragraphs are preferred. If the transcript contains
|
Whisper-derived transcripts often contain one short utterance per
|
||||||
no blank lines, individual non-empty lines are used as blocks.
|
paragraph. Adjacent utterances are therefore grouped until the approximate
|
||||||
|
target size is reached.
|
||||||
"""
|
"""
|
||||||
normalized = text.replace("\r\n", "\n").replace("\r", "\n").strip()
|
if target_chars < 1:
|
||||||
|
raise ValueError(
|
||||||
|
"target_chars must be greater than zero."
|
||||||
|
)
|
||||||
|
|
||||||
paragraphs = [
|
normalized = (
|
||||||
|
text.replace("\r\n", "\n")
|
||||||
|
.replace("\r", "\n")
|
||||||
|
.strip()
|
||||||
|
)
|
||||||
|
|
||||||
|
units = [
|
||||||
part.strip()
|
part.strip()
|
||||||
for part in re.split(r"\n\s*\n+", normalized)
|
for part in re.split(r"\n\s*\n+", normalized)
|
||||||
if part.strip()
|
if part.strip()
|
||||||
]
|
]
|
||||||
|
|
||||||
if len(paragraphs) > 1:
|
if not units:
|
||||||
return paragraphs
|
units = [
|
||||||
|
line.strip()
|
||||||
|
for line in normalized.splitlines()
|
||||||
|
if line.strip()
|
||||||
|
]
|
||||||
|
|
||||||
return [line.strip() for line in normalized.splitlines() if line.strip()]
|
blocks: list[str] = []
|
||||||
|
current_units: list[str] = []
|
||||||
|
current_length = 0
|
||||||
|
|
||||||
|
for unit in units:
|
||||||
|
separator_length = 1 if current_units else 0
|
||||||
|
|
||||||
|
projected_length = (
|
||||||
|
current_length
|
||||||
|
+ separator_length
|
||||||
|
+ len(unit)
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
current_units
|
||||||
|
and projected_length > target_chars
|
||||||
|
):
|
||||||
|
blocks.append("\n".join(current_units))
|
||||||
|
|
||||||
|
current_units = [unit]
|
||||||
|
current_length = len(unit)
|
||||||
|
|
||||||
|
else:
|
||||||
|
current_units.append(unit)
|
||||||
|
current_length = projected_length
|
||||||
|
|
||||||
|
if current_units:
|
||||||
|
blocks.append("\n".join(current_units))
|
||||||
|
|
||||||
|
return blocks
|
||||||
|
|
||||||
|
|
||||||
def render_numbered_blocks(blocks: list[str]) -> str:
|
def render_numbered_blocks(
|
||||||
|
blocks: list[str],
|
||||||
|
) -> str:
|
||||||
return "\n\n".join(
|
return "\n\n".join(
|
||||||
f"[BLOCK {number}]\n{block}"
|
f"[BLOCK {number}]\n{block}"
|
||||||
for number, block in enumerate(blocks, start=1)
|
for number, block in enumerate(
|
||||||
|
blocks,
|
||||||
|
start=1,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def build_prompt(source_name: str, blocks: list[str]) -> str:
|
def build_prompt(
|
||||||
schema_text = json.dumps(OUTPUT_SCHEMA, ensure_ascii=False, indent=2)
|
source_name: str,
|
||||||
numbered_transcript = render_numbered_blocks(blocks)
|
blocks: list[str],
|
||||||
|
) -> str:
|
||||||
|
numbered_transcript = render_numbered_blocks(
|
||||||
|
blocks
|
||||||
|
)
|
||||||
|
|
||||||
return f"""{SYSTEM_INSTRUCTION}
|
return f"""Bestimme die Stellen, an denen sich das Gesprächsthema ändert.
|
||||||
|
|
||||||
Quelldatei:
|
Quelldatei:
|
||||||
{source_name}
|
{source_name}
|
||||||
@@ -164,24 +245,19 @@ Quelldatei:
|
|||||||
Anzahl der Blöcke:
|
Anzahl der Blöcke:
|
||||||
{len(blocks)}
|
{len(blocks)}
|
||||||
|
|
||||||
Erwartete JSON-Struktur:
|
Erwartete Ausgabe:
|
||||||
{schema_text}
|
|
||||||
|
|
||||||
Zusätzliche Anforderungen:
|
{{
|
||||||
|
"topic_changes": [4, 12, 18]
|
||||||
|
}}
|
||||||
|
|
||||||
- Alle topic_id-Werte müssen dem Muster topic_001, topic_002 usw. folgen.
|
Bedeutung:
|
||||||
- Die Nummerierung beginnt mit topic_001.
|
|
||||||
- Die topic_id-Werte müssen innerhalb der Ausgabe eindeutig sein.
|
- Der Wert 4 bedeutet, dass nach Block 4 ein neues Thema beginnt.
|
||||||
- start_block und end_block beziehen sich auf die angegebenen Blocknummern.
|
- Der Wert 12 bedeutet, dass nach Block 12 ein neues Thema beginnt.
|
||||||
- start_block darf nicht größer als end_block sein.
|
- Der letzte Block darf nicht in der Liste stehen.
|
||||||
- Kein Block darf fehlen.
|
- Falls es keinen Themenwechsel gibt, gib eine leere Liste zurück.
|
||||||
- Kein Block darf mehreren Themen gleichzeitig zugeordnet werden.
|
- Gib nur eindeutige, aufsteigend sortierte Blocknummern zurück.
|
||||||
- Direkt aufeinanderfolgende Bereiche desselben Themas sollen als ein Segment
|
|
||||||
ausgegeben werden.
|
|
||||||
- Nicht zusammenhängende Bereiche desselben Themas sollen als mehrere Segmente
|
|
||||||
innerhalb desselben Themas ausgegeben werden.
|
|
||||||
- Themenbezeichnungen sollen knapp sein und keine Zusammenfassung enthalten.
|
|
||||||
- Der Schlüssel "topics" muss immer vorhanden sein.
|
|
||||||
|
|
||||||
TRANSKRIPT:
|
TRANSKRIPT:
|
||||||
--- BEGINN TRANSKRIPT ---
|
--- BEGINN TRANSKRIPT ---
|
||||||
@@ -199,20 +275,26 @@ def call_ollama(
|
|||||||
) -> tuple[str, dict[str, Any]]:
|
) -> tuple[str, dict[str, Any]]:
|
||||||
payload = {
|
payload = {
|
||||||
"model": model,
|
"model": model,
|
||||||
|
"system": SYSTEM_INSTRUCTION,
|
||||||
"prompt": prompt,
|
"prompt": prompt,
|
||||||
"stream": False,
|
"stream": False,
|
||||||
"format": "json",
|
"format": TOPIC_CHANGE_JSON_SCHEMA,
|
||||||
|
"think": False,
|
||||||
"options": {
|
"options": {
|
||||||
"temperature": temperature,
|
"temperature": temperature,
|
||||||
|
"seed": 42,
|
||||||
|
"num_ctx": 32768,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
started = time.perf_counter()
|
started = time.perf_counter()
|
||||||
|
|
||||||
response = requests.post(
|
response = requests.post(
|
||||||
endpoint,
|
endpoint,
|
||||||
json=payload,
|
json=payload,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
elapsed = time.perf_counter() - started
|
elapsed = time.perf_counter() - started
|
||||||
|
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
@@ -220,197 +302,226 @@ def call_ollama(
|
|||||||
data = response.json()
|
data = response.json()
|
||||||
response_text = data.get("response")
|
response_text = data.get("response")
|
||||||
|
|
||||||
if not isinstance(response_text, str) or not response_text.strip():
|
if (
|
||||||
raise ValueError("Ollama returned no usable response text.")
|
not isinstance(response_text, str)
|
||||||
|
or not response_text.strip()
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"Ollama returned no usable response text."
|
||||||
|
)
|
||||||
|
|
||||||
metadata = {
|
metadata = {
|
||||||
"model": data.get("model", model),
|
"model": data.get("model", model),
|
||||||
"elapsed_seconds": round(elapsed, 3),
|
"elapsed_seconds": round(
|
||||||
"total_duration_ns": data.get("total_duration"),
|
elapsed,
|
||||||
"load_duration_ns": data.get("load_duration"),
|
3,
|
||||||
"prompt_eval_count": data.get("prompt_eval_count"),
|
),
|
||||||
"prompt_eval_duration_ns": data.get("prompt_eval_duration"),
|
"total_duration_ns": data.get(
|
||||||
"eval_count": data.get("eval_count"),
|
"total_duration"
|
||||||
"eval_duration_ns": data.get("eval_duration"),
|
),
|
||||||
|
"load_duration_ns": data.get(
|
||||||
|
"load_duration"
|
||||||
|
),
|
||||||
|
"prompt_eval_count": data.get(
|
||||||
|
"prompt_eval_count"
|
||||||
|
),
|
||||||
|
"prompt_eval_duration_ns": data.get(
|
||||||
|
"prompt_eval_duration"
|
||||||
|
),
|
||||||
|
"eval_count": data.get(
|
||||||
|
"eval_count"
|
||||||
|
),
|
||||||
|
"eval_duration_ns": data.get(
|
||||||
|
"eval_duration"
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
return response_text.strip(), metadata
|
return response_text.strip(), metadata
|
||||||
|
|
||||||
|
|
||||||
def parse_json_response(text: str) -> dict[str, Any]:
|
def parse_json_response(
|
||||||
|
text: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
try:
|
try:
|
||||||
parsed = json.loads(text)
|
parsed = json.loads(text)
|
||||||
|
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
# Fallback for models that wrap JSON in Markdown or add text around it.
|
match = re.search(
|
||||||
match = re.search(r"\{.*\}", text, flags=re.DOTALL)
|
r"\{.*\}",
|
||||||
|
text,
|
||||||
|
flags=re.DOTALL,
|
||||||
|
)
|
||||||
|
|
||||||
if not match:
|
if not match:
|
||||||
raise
|
raise
|
||||||
parsed = json.loads(match.group(0))
|
|
||||||
|
parsed = json.loads(
|
||||||
|
match.group(0)
|
||||||
|
)
|
||||||
|
|
||||||
if not isinstance(parsed, dict):
|
if not isinstance(parsed, dict):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"The model response is valid JSON but not a JSON object."
|
"The model response is not a JSON object."
|
||||||
)
|
)
|
||||||
|
|
||||||
return parsed
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
def validate_topic_id(topic_id: Any) -> str:
|
def validate_topic_changes(
|
||||||
if not isinstance(topic_id, str):
|
result: dict[str, Any],
|
||||||
raise ValueError("Every topic must contain a string topic_id.")
|
|
||||||
|
|
||||||
if not re.fullmatch(r"topic_\d{3}", topic_id):
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid topic_id {topic_id!r}; expected format topic_001."
|
|
||||||
)
|
|
||||||
|
|
||||||
return topic_id
|
|
||||||
|
|
||||||
|
|
||||||
def validate_segment(
|
|
||||||
segment: Any,
|
|
||||||
topic_id: str,
|
|
||||||
block_count: int,
|
block_count: int,
|
||||||
) -> dict[str, int]:
|
) -> list[int]:
|
||||||
if not isinstance(segment, dict):
|
raw_changes = result.get("topic_changes")
|
||||||
|
|
||||||
|
if not isinstance(raw_changes, list):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Topic {topic_id} contains a segment that is not an object."
|
"Model output does not contain "
|
||||||
|
"a topic_changes list."
|
||||||
)
|
)
|
||||||
|
|
||||||
start_block = segment.get("start_block")
|
changes: list[int] = []
|
||||||
end_block = segment.get("end_block")
|
|
||||||
|
|
||||||
if not isinstance(start_block, int) or isinstance(start_block, bool):
|
for value in raw_changes:
|
||||||
|
if (
|
||||||
|
not isinstance(value, int)
|
||||||
|
or isinstance(value, bool)
|
||||||
|
):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Topic {topic_id} contains an invalid start_block."
|
"Every topic change must be "
|
||||||
|
"an integer block number."
|
||||||
)
|
)
|
||||||
|
|
||||||
if not isinstance(end_block, int) or isinstance(end_block, bool):
|
if value < 1:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Topic {topic_id} contains an invalid end_block."
|
f"Invalid topic change: {value}"
|
||||||
)
|
)
|
||||||
|
|
||||||
if start_block < 1 or end_block > block_count:
|
if value >= block_count:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Topic {topic_id} references blocks outside 1..{block_count}."
|
f"Topic change {value} is invalid. "
|
||||||
|
f"The last block is {block_count}."
|
||||||
)
|
)
|
||||||
|
|
||||||
if start_block > end_block:
|
changes.append(value)
|
||||||
|
|
||||||
|
if len(changes) != len(set(changes)):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Topic {topic_id} contains start_block > end_block."
|
"The model returned duplicate "
|
||||||
|
"topic changes."
|
||||||
)
|
)
|
||||||
|
|
||||||
return {
|
if changes != sorted(changes):
|
||||||
|
raise ValueError(
|
||||||
|
"The topic changes are not "
|
||||||
|
"sorted in ascending order."
|
||||||
|
)
|
||||||
|
|
||||||
|
return changes
|
||||||
|
|
||||||
|
|
||||||
|
def build_segments(
|
||||||
|
topic_changes: list[int],
|
||||||
|
block_count: int,
|
||||||
|
) -> list[dict[str, int | str]]:
|
||||||
|
"""
|
||||||
|
Convert topic boundaries into continuous and non-overlapping segments.
|
||||||
|
"""
|
||||||
|
segments: list[dict[str, int | str]] = []
|
||||||
|
|
||||||
|
start_block = 1
|
||||||
|
|
||||||
|
for number, end_block in enumerate(
|
||||||
|
topic_changes,
|
||||||
|
start=1,
|
||||||
|
):
|
||||||
|
segments.append(
|
||||||
|
{
|
||||||
|
"segment_id": (
|
||||||
|
f"segment_{number:03d}"
|
||||||
|
),
|
||||||
"start_block": start_block,
|
"start_block": start_block,
|
||||||
"end_block": end_block,
|
"end_block": end_block,
|
||||||
}
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
start_block = end_block + 1
|
||||||
|
|
||||||
def normalize_topics(
|
final_number = len(segments) + 1
|
||||||
result: dict[str, Any],
|
|
||||||
block_count: int,
|
|
||||||
) -> list[dict[str, Any]]:
|
|
||||||
raw_topics = result.get("topics")
|
|
||||||
|
|
||||||
if not isinstance(raw_topics, list):
|
segments.append(
|
||||||
raise ValueError("Model output does not contain a topics list.")
|
|
||||||
|
|
||||||
topics: list[dict[str, Any]] = []
|
|
||||||
seen_topic_ids: set[str] = set()
|
|
||||||
|
|
||||||
for raw_topic in raw_topics:
|
|
||||||
if not isinstance(raw_topic, dict):
|
|
||||||
raise ValueError("Every topic must be a JSON object.")
|
|
||||||
|
|
||||||
topic_id = validate_topic_id(raw_topic.get("topic_id"))
|
|
||||||
|
|
||||||
if topic_id in seen_topic_ids:
|
|
||||||
raise ValueError(f"Duplicate topic_id: {topic_id}")
|
|
||||||
seen_topic_ids.add(topic_id)
|
|
||||||
|
|
||||||
title = raw_topic.get("title")
|
|
||||||
if not isinstance(title, str) or not title.strip():
|
|
||||||
raise ValueError(f"Topic {topic_id} has no usable title.")
|
|
||||||
|
|
||||||
raw_segments = raw_topic.get("segments")
|
|
||||||
if not isinstance(raw_segments, list) or not raw_segments:
|
|
||||||
raise ValueError(f"Topic {topic_id} has no segments.")
|
|
||||||
|
|
||||||
segments = [
|
|
||||||
validate_segment(segment, topic_id, block_count)
|
|
||||||
for segment in raw_segments
|
|
||||||
]
|
|
||||||
segments.sort(key=lambda item: item["start_block"])
|
|
||||||
|
|
||||||
topics.append(
|
|
||||||
{
|
{
|
||||||
"topic_id": topic_id,
|
"segment_id": (
|
||||||
"title": title.strip(),
|
f"segment_{final_number:03d}"
|
||||||
"segments": segments,
|
),
|
||||||
|
"start_block": start_block,
|
||||||
|
"end_block": block_count,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
if not topics:
|
return segments
|
||||||
raise ValueError("The model returned no topics.")
|
|
||||||
|
|
||||||
topics.sort(
|
|
||||||
key=lambda topic: min(
|
|
||||||
segment["start_block"] for segment in topic["segments"]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return topics
|
|
||||||
|
|
||||||
|
|
||||||
def validate_block_coverage(
|
def validate_segments(
|
||||||
topics: list[dict[str, Any]],
|
segments: list[dict[str, int | str]],
|
||||||
block_count: int,
|
block_count: int,
|
||||||
) -> None:
|
) -> None:
|
||||||
assignments: dict[int, str] = {}
|
if not segments:
|
||||||
|
|
||||||
for topic in topics:
|
|
||||||
topic_id = topic["topic_id"]
|
|
||||||
|
|
||||||
for segment in topic["segments"]:
|
|
||||||
for block_number in range(
|
|
||||||
segment["start_block"],
|
|
||||||
segment["end_block"] + 1,
|
|
||||||
):
|
|
||||||
previous_topic = assignments.get(block_number)
|
|
||||||
|
|
||||||
if previous_topic is not None:
|
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Block {block_number} is assigned to both "
|
"No segments were created."
|
||||||
f"{previous_topic} and {topic_id}."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
assignments[block_number] = topic_id
|
expected_start = 1
|
||||||
|
|
||||||
missing_blocks = [
|
for segment in segments:
|
||||||
block_number
|
start_block = segment["start_block"]
|
||||||
for block_number in range(1, block_count + 1)
|
end_block = segment["end_block"]
|
||||||
if block_number not in assignments
|
|
||||||
]
|
|
||||||
|
|
||||||
if missing_blocks:
|
if not isinstance(start_block, int):
|
||||||
formatted = ", ".join(str(number) for number in missing_blocks)
|
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"The following blocks were not assigned to a topic: {formatted}"
|
"Invalid segment start block."
|
||||||
|
)
|
||||||
|
|
||||||
|
if not isinstance(end_block, int):
|
||||||
|
raise ValueError(
|
||||||
|
"Invalid segment end block."
|
||||||
|
)
|
||||||
|
|
||||||
|
if start_block != expected_start:
|
||||||
|
raise ValueError(
|
||||||
|
"Segments are not continuous."
|
||||||
|
)
|
||||||
|
|
||||||
|
if end_block < start_block:
|
||||||
|
raise ValueError(
|
||||||
|
"A segment ends before it starts."
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_start = end_block + 1
|
||||||
|
|
||||||
|
if expected_start != block_count + 1:
|
||||||
|
raise ValueError(
|
||||||
|
"Segments do not cover all blocks."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def build_output(
|
def build_output(
|
||||||
source_name: str,
|
source_name: str,
|
||||||
blocks: list[str],
|
blocks: list[str],
|
||||||
topics: list[dict[str, Any]],
|
topic_changes: list[int],
|
||||||
|
segments: list[dict[str, int | str]],
|
||||||
metadata: dict[str, Any],
|
metadata: dict[str, Any],
|
||||||
|
target_block_chars: int,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
return {
|
return {
|
||||||
"source": {
|
"source": {
|
||||||
"file": source_name,
|
"file": source_name,
|
||||||
"block_count": len(blocks),
|
"block_count": len(blocks),
|
||||||
|
"target_block_chars": (
|
||||||
|
target_block_chars
|
||||||
|
),
|
||||||
},
|
},
|
||||||
"topics": topics,
|
"topic_changes": topic_changes,
|
||||||
|
"segments": segments,
|
||||||
"_run": metadata,
|
"_run": metadata,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -419,8 +530,15 @@ def write_raw_response(
|
|||||||
output_path: Path,
|
output_path: Path,
|
||||||
raw_text: str,
|
raw_text: str,
|
||||||
) -> Path:
|
) -> Path:
|
||||||
raw_path = output_path.with_suffix(".raw.txt")
|
raw_path = output_path.with_suffix(
|
||||||
raw_path.write_text(raw_text + "\n", encoding="utf-8")
|
".raw.txt"
|
||||||
|
)
|
||||||
|
|
||||||
|
raw_path.write_text(
|
||||||
|
raw_text + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
return raw_path
|
return raw_path
|
||||||
|
|
||||||
|
|
||||||
@@ -430,7 +548,8 @@ def main() -> int:
|
|||||||
try:
|
try:
|
||||||
if not args.input_file.is_file():
|
if not args.input_file.is_file():
|
||||||
raise FileNotFoundError(
|
raise FileNotFoundError(
|
||||||
f"Input file not found: {args.input_file}"
|
f"Input file not found: "
|
||||||
|
f"{args.input_file}"
|
||||||
)
|
)
|
||||||
|
|
||||||
transcript = args.input_file.read_text(
|
transcript = args.input_file.read_text(
|
||||||
@@ -438,17 +557,35 @@ def main() -> int:
|
|||||||
).strip()
|
).strip()
|
||||||
|
|
||||||
if not transcript:
|
if not transcript:
|
||||||
raise ValueError("The input file is empty.")
|
raise ValueError(
|
||||||
|
"The input file is empty."
|
||||||
|
)
|
||||||
|
|
||||||
blocks = split_into_blocks(transcript)
|
blocks = split_into_blocks(
|
||||||
|
transcript,
|
||||||
|
target_chars=(
|
||||||
|
args.target_block_chars
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
if not blocks:
|
if not blocks:
|
||||||
raise ValueError("No discussion blocks could be created.")
|
raise ValueError(
|
||||||
|
"No analysis blocks could "
|
||||||
output_path = args.output or args.input_file.with_name(
|
"be created."
|
||||||
f"{args.input_file.stem}_topics.json"
|
)
|
||||||
|
|
||||||
|
output_path = (
|
||||||
|
args.output
|
||||||
|
or args.input_file.with_name(
|
||||||
|
f"{args.input_file.stem}"
|
||||||
|
"_segments.json"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
output_path.parent.mkdir(
|
||||||
|
parents=True,
|
||||||
|
exist_ok=True,
|
||||||
)
|
)
|
||||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
prompt = build_prompt(
|
prompt = build_prompt(
|
||||||
source_name=args.input_file.name,
|
source_name=args.input_file.name,
|
||||||
@@ -458,8 +595,12 @@ def main() -> int:
|
|||||||
print(f"Model: {args.model}")
|
print(f"Model: {args.model}")
|
||||||
print(f"Input: {args.input_file}")
|
print(f"Input: {args.input_file}")
|
||||||
print(f"Blocks: {len(blocks)}")
|
print(f"Blocks: {len(blocks)}")
|
||||||
|
print(
|
||||||
|
"Block target: "
|
||||||
|
f"{args.target_block_chars} characters"
|
||||||
|
)
|
||||||
print(f"Output: {output_path}")
|
print(f"Output: {output_path}")
|
||||||
print("Segmenting discussion topics ...")
|
print("Detecting topic boundaries ...")
|
||||||
|
|
||||||
raw_text, metadata = call_ollama(
|
raw_text, metadata = call_ollama(
|
||||||
endpoint=args.endpoint,
|
endpoint=args.endpoint,
|
||||||
@@ -470,28 +611,53 @@ def main() -> int:
|
|||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
parsed = parse_json_response(raw_text)
|
parsed = parse_json_response(
|
||||||
topics = normalize_topics(
|
raw_text
|
||||||
|
)
|
||||||
|
|
||||||
|
topic_changes = (
|
||||||
|
validate_topic_changes(
|
||||||
result=parsed,
|
result=parsed,
|
||||||
block_count=len(blocks),
|
block_count=len(blocks),
|
||||||
)
|
)
|
||||||
validate_block_coverage(
|
)
|
||||||
topics=topics,
|
|
||||||
|
segments = build_segments(
|
||||||
|
topic_changes=topic_changes,
|
||||||
block_count=len(blocks),
|
block_count=len(blocks),
|
||||||
)
|
)
|
||||||
except (json.JSONDecodeError, ValueError) as exc:
|
|
||||||
raw_path = write_raw_response(output_path, raw_text)
|
validate_segments(
|
||||||
|
segments=segments,
|
||||||
|
block_count=len(blocks),
|
||||||
|
)
|
||||||
|
|
||||||
|
except (
|
||||||
|
json.JSONDecodeError,
|
||||||
|
ValueError,
|
||||||
|
) as exc:
|
||||||
|
raw_path = write_raw_response(
|
||||||
|
output_path,
|
||||||
|
raw_text,
|
||||||
|
)
|
||||||
|
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Model output could not be validated. "
|
"Model output could not be "
|
||||||
f"Raw output saved to: {raw_path}. "
|
"validated. "
|
||||||
|
f"Raw output saved to: "
|
||||||
|
f"{raw_path}. "
|
||||||
f"Reason: {exc}"
|
f"Reason: {exc}"
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
result = build_output(
|
result = build_output(
|
||||||
source_name=args.input_file.name,
|
source_name=args.input_file.name,
|
||||||
blocks=blocks,
|
blocks=blocks,
|
||||||
topics=topics,
|
topic_changes=topic_changes,
|
||||||
|
segments=segments,
|
||||||
metadata=metadata,
|
metadata=metadata,
|
||||||
|
target_block_chars=(
|
||||||
|
args.target_block_chars
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
output_path.write_text(
|
output_path.write_text(
|
||||||
@@ -504,40 +670,54 @@ def main() -> int:
|
|||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
|
||||||
segment_count = sum(
|
print(
|
||||||
len(topic["segments"]) for topic in topics
|
f"Done in "
|
||||||
|
f"{metadata['elapsed_seconds']:.1f} "
|
||||||
|
"seconds."
|
||||||
)
|
)
|
||||||
resumed_topic_count = sum(
|
print(
|
||||||
1 for topic in topics if len(topic["segments"]) > 1
|
f"Topic changes: "
|
||||||
|
f"{len(topic_changes)}"
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
f"Segments: "
|
||||||
|
f"{len(segments)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
print(f"Done in {metadata['elapsed_seconds']:.1f} seconds.")
|
|
||||||
print(f"Topics: {len(topics)}")
|
|
||||||
print(f"Segments: {segment_count}")
|
|
||||||
print(f"Resumed topics: {resumed_topic_count}")
|
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
except requests.ConnectionError:
|
except requests.ConnectionError:
|
||||||
print(
|
print(
|
||||||
"Error: Ollama is not reachable. Is `ollama serve` running?",
|
"Error: Ollama is not reachable. "
|
||||||
|
"Is `ollama serve` running?",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
except requests.Timeout:
|
except requests.Timeout:
|
||||||
print(
|
print(
|
||||||
"Error: The Ollama request timed out.",
|
"Error: The Ollama request timed out.",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
except requests.HTTPError as exc:
|
except requests.HTTPError as exc:
|
||||||
print(
|
print(
|
||||||
f"Error: Ollama returned an HTTP error: {exc}",
|
"Error: Ollama returned an HTTP "
|
||||||
|
f"error: {exc}",
|
||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
return 1
|
return 1
|
||||||
except (OSError, UnicodeError, ValueError) as exc:
|
|
||||||
print(f"Error: {exc}", file=sys.stderr)
|
except (
|
||||||
|
OSError,
|
||||||
|
UnicodeError,
|
||||||
|
ValueError,
|
||||||
|
) as exc:
|
||||||
|
print(
|
||||||
|
f"Error: {exc}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user