From be599d28b35d07cf3f08238f9e71ff30b613616a Mon Sep 17 00:00:00 2001 From: Martin Tazl Date: Wed, 29 Jul 2026 11:31:17 +0200 Subject: [PATCH] Add RollCalc JSON importer --- CHANGELOG.md | 1 + PROJECT_KNOWLEDGE.md | 5 +- README.md | 14 + docs/importers.md | 60 ++++ .../importers/rollcalc_json.py | 161 +++++++++++ src/article_data_manager/models/__init__.py | 4 +- src/article_data_manager/models/article.py | 58 ++++ tests/unit/importers/test_rollcalc_json.py | 270 ++++++++++++++++++ 8 files changed, 569 insertions(+), 4 deletions(-) create mode 100644 docs/importers.md create mode 100644 src/article_data_manager/importers/rollcalc_json.py create mode 100644 tests/unit/importers/test_rollcalc_json.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e0362c..43f56d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- RollCalc-JSON-Importer mit strikter Struktur- und Typvalidierung ergänzt. - Initiale Projektstruktur angelegt. - Dokumentation fuer Architektur, Datenmodell, Datenwoerterbuch und Merge-Regeln erstellt. - Reduzierte Beispieldaten und Test-Fixtures ergaenzt. diff --git a/PROJECT_KNOWLEDGE.md b/PROJECT_KNOWLEDGE.md index 2beacd4..b2861d6 100644 --- a/PROJECT_KNOWLEDGE.md +++ b/PROJECT_KNOWLEDGE.md @@ -40,7 +40,7 @@ Produktive ERP-Daten, lokale Quelldaten, generierte Dateien und Reports sind per ## Aktueller Stand -Phase 0: Struktur, Dokumentation, Fixtures, minimale Python-Bausteine und Tests. +Phase 0: Struktur, Dokumentation, Fixtures, minimale Python-Bausteine, Tests und RollCalc-JSON-Importer. ## Offene fachliche Fragen @@ -48,13 +48,14 @@ Siehe `docs/data-model.md` und `docs/merge-rules.md`. ## Naechste Schritte -- Importer fuer RollCalc-JSON, ERP-CSV und manuelle CSV-Datei implementieren. +- ERP-CSV-Importer und Import manueller CSV-Datei implementieren. - Validierungs- und Merge-Regeln produktiv ausbauen. - Reports und deterministischen Export ergaenzen. ## Wichtige Dateien - `docs/data-dictionary.md` +- `docs/importers.md` - `docs/merge-rules.md` - `tests/fixtures/` - `src/article_data_manager/` diff --git a/README.md b/README.md index aed3af4..692ec64 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,20 @@ article-data-manager validate data/generated/article-data.json Die CLI-Kommandos sind in Phase 0 nur als Platzhalter vorgesehen. +## Implementierte Importer + +Der RollCalc-JSON-Importer ist als Python-API verfügbar: + +```python +from pathlib import Path + +from article_data_manager.importers.rollcalc_json import load_rollcalc_articles + +articles = load_rollcalc_articles(Path("data/source/rollcalc/article-data.json")) +``` + +Er validiert die bestehende RollCalc-JSON-Struktur strikt, erhält Artikelnummern als Strings und verändert die Quelldatei nicht. Details stehen in `docs/importers.md`. + ## Tests ```bash diff --git a/docs/importers.md b/docs/importers.md new file mode 100644 index 0000000..29dfe6e --- /dev/null +++ b/docs/importers.md @@ -0,0 +1,60 @@ +# Importer + +## RollCalc-JSON-Importer + +Der RollCalc-JSON-Importer liest die bestehende RollCalc-Datei `article-data.json` ein und überführt gültige Datensätze in typisierte `RollCalcArticle`-Objekte. + +Öffentliche API: + +```python +from pathlib import Path + +from article_data_manager.importers.rollcalc_json import load_rollcalc_articles + +articles = load_rollcalc_articles(Path("data/source/rollcalc/article-data.json")) +``` + +## Eingabeformat + +Erwartet wird eine UTF-8-Datei mit einem JSON-Array. Jeder Array-Eintrag muss ein JSON-Objekt sein. + +Pflichtfelder: + +- `nr` +- `name` +- `thickness` +- `area_weight` +- `core_type` + +Datentypen: + +- `nr`: String, nicht leer, ohne führende oder nachgestellte Leerzeichen +- `name`: String +- `thickness`: JSON-Zahl, kein Boolean +- `area_weight`: JSON-Zahl, kein Boolean +- `core_type`: JSON-Zahl, kein Boolean + +Integer-Zahlen aus JSON werden intern als Python-`float` gespeichert, sofern sie in numerischen Feldern stehen. Strings wie `"6.722"`, `null` und Booleans werden nicht als Zahlen akzeptiert. + +## Unbekannte Felder + +Zusätzliche unbekannte Felder werden akzeptiert und ignoriert. Sie werden nicht in das aktuelle interne Modell übernommen. Dadurch bleibt der Importer kompatibel mit möglichen RollCalc-Erweiterungen, validiert die bekannten Felder aber weiterhin strikt. + +## Dubletten + +Doppelte Artikelnummern innerhalb einer Datei sind ein Validierungsfehler. Maßgeblich ist die exakte Stringdarstellung, daher sind `"00001"` und `"1"` unterschiedliche Artikelnummern. + +## Fehlerverhalten + +Der Importer verwendet eigene Exceptions: + +- `RollCalcImportError` +- `RollCalcFileError` +- `RollCalcJsonSyntaxError` +- `RollCalcValidationError` + +Fehlermeldungen enthalten Datei, Datensatzindex und Feldname, soweit anwendbar. Vollständige Datensätze werden nicht in Fehlermeldungen ausgegeben. + +## Quelldatei + +Der Importer liest die Quelldatei nur. Er schreibt, verändert oder repariert die Datei nicht und erzeugt keine Ausgabe auf stdout oder stderr. diff --git a/src/article_data_manager/importers/rollcalc_json.py b/src/article_data_manager/importers/rollcalc_json.py new file mode 100644 index 0000000..01810eb --- /dev/null +++ b/src/article_data_manager/importers/rollcalc_json.py @@ -0,0 +1,161 @@ +"""Importer for the existing RollCalc `article-data.json` source file.""" + +from __future__ import annotations + +import json +from json import JSONDecodeError +from pathlib import Path +from typing import Any + +from article_data_manager.models import RollCalcArticle + +REQUIRED_FIELDS: tuple[str, ...] = ("nr", "name", "thickness", "area_weight", "core_type") +NUMERIC_FIELDS: frozenset[str] = frozenset({"thickness", "area_weight", "core_type"}) + + +class RollCalcImportError(Exception): + """Base class for RollCalc import failures.""" + + +class RollCalcFileError(RollCalcImportError): + """Raised when the source file cannot be read as a regular UTF-8 file.""" + + +class RollCalcJsonSyntaxError(RollCalcImportError): + """Raised when the source file does not contain syntactically valid JSON.""" + + +class RollCalcValidationError(RollCalcImportError): + """Raised when valid JSON does not match the expected RollCalc structure.""" + + +def load_rollcalc_articles(path: Path) -> list[RollCalcArticle]: + """Load and validate an existing RollCalc JSON article file. + + The source must be a UTF-8 JSON array. Known RollCalc fields are validated + strictly, unknown object fields are accepted and ignored for forward + compatibility. Article numbers are never trimmed, reformatted, or interpreted + numerically; leading or trailing whitespace is treated as invalid input. + """ + + source_path = Path(path) + payload = _read_json(source_path) + if not isinstance(payload, list): + raise RollCalcValidationError( + f"{source_path}: RollCalc JSON root must be an array, got {_type_name(payload)}." + ) + + articles: list[RollCalcArticle] = [] + seen_article_numbers: dict[str, int] = {} + for index, item in enumerate(payload): + article = _parse_article(source_path, index, item) + first_index = seen_article_numbers.get(article.nr) + if first_index is not None: + raise RollCalcValidationError( + f"{source_path}: Duplicate article number {article.nr!r} at index {index}; " + f"first occurrence at index {first_index}." + ) + seen_article_numbers[article.nr] = index + articles.append(article) + + return articles + + +def _read_json(path: Path) -> Any: + if not path.exists(): + raise RollCalcFileError(f"{path}: RollCalc JSON file does not exist.") + if not path.is_file(): + raise RollCalcFileError(f"{path}: RollCalc JSON path must be a regular file.") + + try: + content = path.read_text(encoding="utf-8") + except UnicodeDecodeError as exc: + raise RollCalcFileError(f"{path}: RollCalc JSON file must be readable as UTF-8.") from exc + + try: + return json.loads(content, parse_constant=_reject_non_standard_json_constant) + except JSONDecodeError as exc: + raise RollCalcJsonSyntaxError( + f"{path}: invalid JSON at line {exc.lineno}, column {exc.colno}: {exc.msg}." + ) from exc + except ValueError as exc: + raise RollCalcJsonSyntaxError(f"{path}: invalid JSON: {exc}.") from exc + + +def _reject_non_standard_json_constant(value: str) -> None: + raise ValueError(f"non-standard numeric constant {value!r} is not allowed") + + +def _parse_article(path: Path, index: int, item: Any) -> RollCalcArticle: + if not isinstance(item, dict): + raise RollCalcValidationError( + f"{path}: Invalid RollCalc article at index {index}: " + f"entry must be an object, got {_type_name(item)}." + ) + + for field_name in REQUIRED_FIELDS: + if field_name not in item: + raise RollCalcValidationError( + f"{path}: Invalid RollCalc article at index {index}: " + f"missing required field {field_name!r}." + ) + + nr = _require_string(path, index, "nr", item["nr"]) + if nr == "": + raise RollCalcValidationError( + f"{path}: Invalid RollCalc article at index {index}: field 'nr' must not be empty." + ) + if nr != nr.strip(): + raise RollCalcValidationError( + f"{path}: Invalid RollCalc article at index {index}: " + "field 'nr' must not contain leading or trailing whitespace." + ) + + return RollCalcArticle.from_validated_values( + nr=nr, + name=_require_string(path, index, "name", item["name"]), + thickness=_require_number(path, index, "thickness", item["thickness"]), + area_weight=_require_number(path, index, "area_weight", item["area_weight"]), + core_type=_require_number(path, index, "core_type", item["core_type"]), + ) + + +def _require_string(path: Path, index: int, field_name: str, value: Any) -> str: + if not isinstance(value, str): + raise RollCalcValidationError( + f"{path}: Invalid RollCalc article at index {index}: field {field_name!r} " + f"must be a string, got {_type_name(value)} ({_format_value(value)})." + ) + return value + + +def _require_number(path: Path, index: int, field_name: str, value: Any) -> int | float: + if not _is_json_number(value): + raise RollCalcValidationError( + f"{path}: Invalid RollCalc article at index {index}: field {field_name!r} " + f"must be a JSON number, got {_type_name(value)} ({_format_value(value)})." + ) + return value + + +def _is_json_number(value: Any) -> bool: + return isinstance(value, int | float) and not isinstance(value, bool) + + +def _type_name(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "bool" + if isinstance(value, list): + return "array" + if isinstance(value, dict): + return "object" + return type(value).__name__ + + +def _format_value(value: Any, *, max_length: int = 80) -> str: + formatted = repr(value) + if len(formatted) > max_length: + return formatted[: max_length - 3] + "..." + return formatted diff --git a/src/article_data_manager/models/__init__.py b/src/article_data_manager/models/__init__.py index f9d40c1..b66c8c5 100644 --- a/src/article_data_manager/models/__init__.py +++ b/src/article_data_manager/models/__init__.py @@ -1,5 +1,5 @@ """Domain models.""" -from article_data_manager.models.article import Article +from article_data_manager.models.article import Article, RollCalcArticle -__all__ = ["Article"] +__all__ = ["Article", "RollCalcArticle"] diff --git a/src/article_data_manager/models/article.py b/src/article_data_manager/models/article.py index 6b47daa..dccac66 100644 --- a/src/article_data_manager/models/article.py +++ b/src/article_data_manager/models/article.py @@ -1,6 +1,7 @@ """Canonical article model for Phase 0 assumptions.""" from dataclasses import dataclass +from typing import Any @dataclass(frozen=True, slots=True) @@ -24,3 +25,60 @@ class Article: raise TypeError("Article number 'nr' must be a string.") if self.nr == "": raise ValueError("Article number 'nr' must not be empty.") + + +@dataclass(frozen=True, slots=True) +class RollCalcArticle: + """Article shape imported from the existing RollCalc JSON source. + + This model preserves the currently observed RollCalc fields. The numeric + `core_type` value is intentionally not translated because its meaning is + still fachlich offen. + """ + + nr: str + name: str + thickness: float + area_weight: float + core_type: float + + def __post_init__(self) -> None: + if not isinstance(self.nr, str): + raise TypeError("Article number 'nr' must be a string.") + if self.nr == "": + raise ValueError("Article number 'nr' must not be empty.") + + @classmethod + def from_validated_values( + cls, + *, + nr: str, + name: str, + thickness: int | float, + area_weight: int | float, + core_type: int | float, + ) -> "RollCalcArticle": + """Create an article after importer validation. + + Integer JSON numbers are stored as floats to keep numeric model fields + consistent without changing their value. + """ + + return cls( + nr=nr, + name=name, + thickness=float(thickness), + area_weight=float(area_weight), + core_type=float(core_type), + ) + + def to_dict(self) -> dict[str, Any]: + """Return the imported RollCalc fields as a plain dictionary.""" + + return { + "nr": self.nr, + "name": self.name, + "thickness": self.thickness, + "area_weight": self.area_weight, + "core_type": self.core_type, + } diff --git a/tests/unit/importers/test_rollcalc_json.py b/tests/unit/importers/test_rollcalc_json.py new file mode 100644 index 0000000..3636107 --- /dev/null +++ b/tests/unit/importers/test_rollcalc_json.py @@ -0,0 +1,270 @@ +import json +from pathlib import Path + +import pytest + +from article_data_manager.importers.rollcalc_json import ( + RollCalcFileError, + RollCalcJsonSyntaxError, + RollCalcValidationError, + load_rollcalc_articles, +) + + +def write_json(path: Path, payload: object) -> Path: + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + +def valid_article(**overrides: object) -> dict[str, object]: + article: dict[str, object] = { + "nr": "214700", + "name": "Example Article", + "thickness": 6.722, + "area_weight": 0.0, + "core_type": 0.0, + } + article.update(overrides) + return article + + +def test_loads_valid_file_with_one_article(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article()]) + + articles = load_rollcalc_articles(path) + + assert len(articles) == 1 + assert articles[0].nr == "214700" + assert articles[0].name == "Example Article" + assert articles[0].thickness == 6.722 + assert articles[0].area_weight == 0.0 + assert articles[0].core_type == 0.0 + + +def test_loads_valid_file_with_multiple_articles(tmp_path: Path) -> None: + path = write_json( + tmp_path / "article-data.json", + [ + valid_article(nr="100001"), + valid_article(nr="100002"), + ], + ) + + articles = load_rollcalc_articles(path) + + assert [article.nr for article in articles] == ["100001", "100002"] + + +def test_preserves_source_order(tmp_path: Path) -> None: + path = write_json( + tmp_path / "article-data.json", + [ + valid_article(nr="300003"), + valid_article(nr="100001"), + valid_article(nr="200002"), + ], + ) + + articles = load_rollcalc_articles(path) + + assert [article.nr for article in articles] == ["300003", "100001", "200002"] + + +def test_preserves_leading_zero_in_article_number(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(nr="000123")]) + + articles = load_rollcalc_articles(path) + + assert articles[0].nr == "000123" + + +def test_numeric_integer_is_stored_as_float(tmp_path: Path) -> None: + path = write_json( + tmp_path / "article-data.json", + [valid_article(thickness=5, area_weight=350, core_type=1)], + ) + + article = load_rollcalc_articles(path)[0] + + assert article.thickness == 5.0 + assert article.area_weight == 350.0 + assert article.core_type == 1.0 + assert isinstance(article.thickness, float) + assert isinstance(article.area_weight, float) + assert isinstance(article.core_type, float) + + +def test_accepts_and_ignores_unknown_fields(tmp_path: Path) -> None: + path = write_json( + tmp_path / "article-data.json", + [valid_article(width=6.0, unknown_nested={"value": "ignored"})], + ) + + article = load_rollcalc_articles(path)[0] + + assert not hasattr(article, "width") + assert not hasattr(article, "unknown_nested") + assert article.to_dict() == { + "nr": "214700", + "name": "Example Article", + "thickness": 6.722, + "area_weight": 0.0, + "core_type": 0.0, + } + + +def test_empty_json_array_returns_empty_list(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", []) + + assert load_rollcalc_articles(path) == [] + + +def test_missing_file_raises_file_error(tmp_path: Path) -> None: + path = tmp_path / "missing.json" + + with pytest.raises(RollCalcFileError, match="does not exist"): + load_rollcalc_articles(path) + + +def test_directory_path_raises_file_error(tmp_path: Path) -> None: + with pytest.raises(RollCalcFileError, match="regular file"): + load_rollcalc_articles(tmp_path) + + +def test_invalid_utf8_raises_file_error(tmp_path: Path) -> None: + path = tmp_path / "article-data.json" + path.write_bytes(b"\xff\xfe\xfa") + + with pytest.raises(RollCalcFileError, match="UTF-8"): + load_rollcalc_articles(path) + + +def test_empty_file_raises_json_syntax_error(tmp_path: Path) -> None: + path = tmp_path / "article-data.json" + path.write_text("", encoding="utf-8") + + with pytest.raises(RollCalcJsonSyntaxError, match="invalid JSON"): + load_rollcalc_articles(path) + + +def test_syntactically_invalid_json_raises_json_syntax_error(tmp_path: Path) -> None: + path = tmp_path / "article-data.json" + path.write_text("[}", encoding="utf-8") + + with pytest.raises(RollCalcJsonSyntaxError, match="invalid JSON"): + load_rollcalc_articles(path) + + +def test_json_root_must_be_array(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", {"nr": "214700"}) + + with pytest.raises(RollCalcValidationError, match="root must be an array"): + load_rollcalc_articles(path) + + +def test_array_entry_must_be_object(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(), "invalid"]) + + with pytest.raises(RollCalcValidationError, match="index 1: entry must be an object"): + load_rollcalc_articles(path) + + +def test_missing_required_field(tmp_path: Path) -> None: + article = valid_article() + del article["name"] + path = write_json(tmp_path / "article-data.json", [article]) + + with pytest.raises(RollCalcValidationError, match="missing required field 'name'"): + load_rollcalc_articles(path) + + +def test_nr_must_be_string(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(nr=214700)]) + + with pytest.raises(RollCalcValidationError, match="field 'nr' must be a string, got int"): + load_rollcalc_articles(path) + + +def test_nr_must_not_be_empty(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(nr="")]) + + with pytest.raises(RollCalcValidationError, match="field 'nr' must not be empty"): + load_rollcalc_articles(path) + + +def test_nr_must_not_contain_surrounding_whitespace(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(nr=" 000123")]) + + with pytest.raises(RollCalcValidationError, match="leading or trailing whitespace"): + load_rollcalc_articles(path) + + +def test_name_must_be_string(tmp_path: Path) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(name=123)]) + + with pytest.raises(RollCalcValidationError, match="field 'name' must be a string"): + load_rollcalc_articles(path) + + +@pytest.mark.parametrize("field_name", ["thickness", "area_weight", "core_type"]) +def test_numeric_field_must_not_be_string(tmp_path: Path, field_name: str) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(**{field_name: "6.722"})]) + + with pytest.raises( + RollCalcValidationError, + match=f"field '{field_name}' must be a JSON number, got str", + ): + load_rollcalc_articles(path) + + +@pytest.mark.parametrize("field_name", ["thickness", "area_weight", "core_type"]) +def test_numeric_field_must_not_be_null(tmp_path: Path, field_name: str) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(**{field_name: None})]) + + with pytest.raises( + RollCalcValidationError, + match=f"field '{field_name}' must be a JSON number, got null", + ): + load_rollcalc_articles(path) + + +@pytest.mark.parametrize("field_name", ["thickness", "area_weight", "core_type"]) +def test_numeric_field_must_not_be_boolean(tmp_path: Path, field_name: str) -> None: + path = write_json(tmp_path / "article-data.json", [valid_article(**{field_name: True})]) + + with pytest.raises( + RollCalcValidationError, + match=f"field '{field_name}' must be a JSON number, got bool", + ): + load_rollcalc_articles(path) + + +def test_duplicate_article_number_raises_validation_error(tmp_path: Path) -> None: + path = write_json( + tmp_path / "article-data.json", + [valid_article(nr="00001"), valid_article(nr="1"), valid_article(nr="00001")], + ) + + with pytest.raises( + RollCalcValidationError, + match="Duplicate article number '00001' at index 2; first occurrence at index 0", + ): + load_rollcalc_articles(path) + + +def test_multiple_errors_report_first_deterministically(tmp_path: Path) -> None: + path = write_json( + tmp_path / "article-data.json", + [ + valid_article(name=123, thickness="6.722"), + valid_article(nr="214700"), + ], + ) + + with pytest.raises(RollCalcValidationError) as exc_info: + load_rollcalc_articles(path) + + message = str(exc_info.value) + assert "index 0" in message + assert "field 'name'" in message + assert "thickness" not in message