Add historical radioeins parser support

2026-08-09 12:19:14 +02:00
parent 355ea3e509
commit bbf0f1bf0a
5 changed files with 141 additions and 6 deletions
@@ -160,6 +160,47 @@ def extract_votes_from_table(
    return [rows_by_rank[rank] for rank in range(1, 11)]


+def extract_votes_from_historical_text(
+    soup,
+    juror: str,
+    source_url: str,
+) -> list[Vote] | None:
+    rows_by_rank: dict[int, Vote] = {}
+
+    for element in soup.find_all(["tr", "li", "p"]):
+        text = clean(element.get_text(" ", strip=True))
+        match = re.fullmatch(r"(10|[1-9])\.\s*(.+)", text)
+        if not match:
+            continue
+
+        rank = int(match.group(1))
+        entry = match.group(2)
+        artist, separator, title = entry.partition(":")
+        if not separator:
+            artist, separator, title = entry.partition(" - ")
+        if not separator:
+            continue
+
+        artist = clean(artist)
+        title = clean(title)
+        if not artist or not title:
+            continue
+
+        rows_by_rank[rank] = Vote(
+            juror=juror,
+            rank=rank,
+            points=POINTS[rank],
+            artist=artist,
+            title=title,
+            source_url=source_url,
+        )
+
+    if set(rows_by_rank) != set(range(1, 11)):
+        return None
+
+    return [rows_by_rank[rank] for rank in range(1, 11)]
+
+
 def parse_jury_page(
    html: str,
    fallback_name: str,
@@ -173,6 +214,10 @@ def parse_jury_page(
        if votes is not None:
            return votes

+    votes = extract_votes_from_historical_text(soup, juror, source_url)
+    if votes is not None:
+        return votes
+
    raise RuntimeError("Keine vollständige Top-10-Tabelle gefunden")


@@ -0,0 +1,18 @@
+<!doctype html>
+<html lang="de">
+<head><meta property="og:title" content="Andreas Löffler"></head>
+<body>
+<table>
+<tr><td>1. Die Ärzte: Schrei nach Liebe</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>2. Peter Fox: Haus am See</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>3. Fettes Brot: Jein</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>4. Die Toten Hosen: Hier kommt Alex</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>5. Rammstein: Engel</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>6. Marteria: OMG</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>7. Herbert Grönemeyer:&nbsp;Mensch</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>8. Sportfreunde Stiller: Ein Kompliment</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>9. Falco: Der Kommissar</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>10. Rio Reiser: Junimond</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+</table>
+</body>
+</html>
@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="de">
+<head><meta property="og:title" content="Andreas Ulrich"></head>
+<body>
+<h4>Meine Top 10</h4>
+<table>
+<tr><td>1.&nbsp;City: Am Fenster</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>2.&nbsp;Rosenstolz: Königin</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>3.&nbsp;Marteria: Kids ( 2 Finger am Kopf)</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>4.&nbsp;Ton Steine Scherben: Macht kaputt was euch kaputt macht</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>5.&nbsp;Heinz Rudolf Kunze: Wunderkinder</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>6.&nbsp;Annenmaykantrereit: Pocahontas<br></td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>7.&nbsp;Die Toten Hosen: Tage wie diese<br></td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>8.&nbsp;Pankow: Langeweile<br></td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>9.&nbsp;Rio Reiser: Junimond<br></td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>10.&nbsp;Wir Sind Helden: Müssen nur wollen</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+</table>
+</body>
+</html>
@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="de">
+<head><meta property="og:title" content="Aydo Abay"></head>
+<body>
+<h4>Meine Top 10</h4>
+<table>
+<tr><td>1. Kraftwerk: Radioaktivität</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>2. Trio: Los Paul</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>3. Udo Lindenberg: Leider nur ein Vakuum</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>4. Dauerfisch: So gut</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>5. D.A.F. : Der Mussolini</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>6. Element of Crime: Am Ende denk ich immer nur an dich</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>7. Porf: Schönes Bild Wenn</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>8. Boxhmasters - Mogli</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>9. All Diese Gewalt: Jeder Traum eine Falle</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+<tr><td>10. Udo Jürgens: Merci Chérie</td><td>&nbsp;</td><td>&nbsp;</td></tr>
+</table>
+</body>
+</html>
@@ -5,6 +5,14 @@ from radioeins_top100 import POINTS, derive_output_dir, parse_jury_page


 class RadioeinsTop100Tests(unittest.TestCase):
+    def parse_fixture(self, name):
+        fixture = Path(__file__).parent / "fixtures" / name
+        return parse_jury_page(
+            fixture.read_text(encoding="utf-8"),
+            fallback_name="Fallback",
+            source_url=f"https://example.test/{name}",
+        )
+
    def test_output_directory_is_derived_from_url_path(self):
        base = "https://www.radioeins.de/musik/top_100/2026"
        self.assertEqual(derive_output_dir(f"{base}/zeit/"), Path("output_zeit"))
@@ -20,12 +28,7 @@ class RadioeinsTop100Tests(unittest.TestCase):
        )

    def test_parse_existing_jury_fixture(self):
-        fixture = Path(__file__).parent / "fixtures" / "aditya-sharma.html"
-        votes = parse_jury_page(
-            fixture.read_text(encoding="utf-8"),
-            fallback_name="Fallback",
-            source_url="https://example.test/aditya-sharma.html",
-        )
+        votes = self.parse_fixture("aditya-sharma.html")

        self.assertEqual(len(votes), 10)
        self.assertEqual(votes[0].juror, "Aditya Sharma")
@@ -33,6 +36,37 @@ class RadioeinsTop100Tests(unittest.TestCase):
        self.assertEqual((votes[1].artist, votes[1].title), ("Samy Deluxe", "Weck Mich Auf"))
        self.assertEqual((votes[-1].rank, votes[-1].points), (10, 1))

+    def test_parse_historical_page_without_heading(self):
+        votes = self.parse_fixture("andreas-loeffler.html")
+
+        self.assertEqual(len(votes), 10)
+        self.assertEqual(votes[0].juror, "Andreas Löffler")
+        self.assertEqual((votes[0].artist, votes[0].title), ("Die Ärzte", "Schrei nach Liebe"))
+        self.assertEqual((votes[-1].rank, votes[-1].points), (10, 1))
+
+    def test_parse_historical_page_with_html_spacing(self):
+        votes = self.parse_fixture("andreas-ulrich.html")
+
+        self.assertEqual(len(votes), 10)
+        self.assertEqual(votes[5].title, "Pocahontas")
+        self.assertEqual(votes[-1].artist, "Wir Sind Helden")
+
+    def test_historical_title_preserves_additional_colons(self):
+        fixture = Path(__file__).parent / "fixtures" / "andreas-loeffler.html"
+        html = fixture.read_text(encoding="utf-8").replace(
+            "Schrei nach Liebe", "Live: Schrei nach Liebe"
+        )
+        votes = parse_jury_page(html, "Fallback", "https://example.test/list.html")
+
+        self.assertEqual(votes[0].title, "Live: Schrei nach Liebe")
+
+    def test_historical_page_accepts_spaced_hyphen_separator(self):
+        votes = self.parse_fixture("aydo-abay.html")
+
+        self.assertEqual(len(votes), 10)
+        self.assertEqual(votes[7].artist, "Boxhmasters")
+        self.assertEqual(votes[7].title, "Mogli")
+

 if __name__ == "__main__":
    unittest.main()