Add central material calibration support

src/production_analytics/calculations/calibrations.py

@@ -0,0 +1,89 @@
+"""Generic, version-controlled calibration values and measurement provenance."""
+
+from dataclasses import dataclass
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from production_analytics.calculations.config import (
+    CalculationConfigError,
+    _UniqueLoader,
+    finite_number,
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Calibration:
+    type: str
+    value: float
+    unit: str
+    calibrated_at: date
+    method: str
+    description: str
+    reference: dict[str, Any]
+
+
+def load_calibrations(path: str | Path) -> dict[str, Calibration]:
+    """Validate every entry, retaining opaque IDs and free-form reference metadata."""
+    try:
+        with Path(path).open(encoding="utf-8") as stream:
+            document = yaml.load(stream, Loader=_UniqueLoader)
+    except (OSError, UnicodeError) as exc:
+        raise CalculationConfigError(f"Cannot read calibration configuration: {path}") from exc
+    except yaml.YAMLError as exc:
+        raise CalculationConfigError(f"Invalid YAML in calibration configuration: {path}") from exc
+    if not isinstance(document, dict) or set(document) != {"calibrations"}:
+        raise CalculationConfigError("Calibration configuration must contain only 'calibrations'")
+    entries = document["calibrations"]
+    if not isinstance(entries, dict) or not entries:
+        raise CalculationConfigError("calibrations must be a non-empty mapping")
+    result = {}
+    required = {"type", "value", "unit", "calibrated_at", "method", "description", "reference"}
+    for identifier, entry in entries.items():
+        prefix = f"calibrations[{identifier!r}]"
+        if not identifier.strip():
+            raise CalculationConfigError("Calibration IDs must be non-empty strings")
+        if not isinstance(entry, dict) or set(entry) != required:
+            raise CalculationConfigError(
+                f"{prefix}: required fields are {', '.join(sorted(required))}"
+            )
+        for name in ("type", "unit", "method", "description"):
+            if not isinstance(entry[name], str) or not entry[name].strip():
+                raise CalculationConfigError(f"{prefix}.{name} must be a non-empty string")
+        calibrated_at = entry["calibrated_at"]
+        if isinstance(calibrated_at, str):
+            try:
+                parsed = date.fromisoformat(calibrated_at)
+                if parsed.isoformat() != calibrated_at:
+                    raise ValueError
+                calibrated_at = parsed
+            except ValueError as exc:
+                raise CalculationConfigError(f"{prefix}.calibrated_at must be YYYY-MM-DD") from exc
+        if type(calibrated_at) is not date:
+            raise CalculationConfigError(f"{prefix}.calibrated_at must be YYYY-MM-DD")
+        if not isinstance(entry["reference"], dict) or not entry["reference"]:
+            raise CalculationConfigError(f"{prefix}.reference must be a non-empty mapping")
+        result[identifier] = Calibration(
+            **{**entry, "calibrated_at": calibrated_at,
+               "value": finite_number(entry["value"], f"{prefix}.value")},
+        )
+    return result
+
+
+def resolve_calibration(
+    calibrations: dict[str, Calibration], reference: str, *, expected_type: str, expected_unit: str,
+) -> Calibration:
+    """Require explicit compatibility; no implicit conversion or ID interpretation."""
+    if not isinstance(reference, str) or not reference.strip():
+        raise CalculationConfigError("calibration_ref must be a non-empty string")
+    if reference not in calibrations:
+        raise CalculationConfigError(f"Missing calibration ID: {reference!r}")
+    calibration = calibrations[reference]
+    for field, expected in (("type", expected_type), ("unit", expected_unit)):
+        if getattr(calibration, field) != expected:
+            raise CalculationConfigError(
+                f"Calibration {reference!r}: incompatible {field}; expected {expected!r}"
+            )
+    return calibration

src/production_analytics/calculations/config.py

@@ -58,8 +58,10 @@ class MaterialCalculationConfig:
     application_signal_refs: tuple[str, ...] = ()
     rotational_speed_signal_refs: tuple[str, ...] = ()
     specific_discharge_kg_per_rev_m: float | None = None
+    calibration_ref: str | None = None
     erp_workplace: str = ""
     production_order_format: str = ""
+    process_application_calculation_id: str | None = None
 
 
 def load_material_calculation(
@@ -81,6 +83,7 @@ def load_material_calculation(
     required = {field.name for field in fields(MaterialCalculationConfig)
                 if field.default is MISSING}
     instances = {}
+    calibrations = None
     for index, entry in enumerate(entries):
         prefix = f"calculations[{index}]"
         if not isinstance(entry, dict):
@@ -148,6 +151,24 @@ def load_material_calculation(
             "erp_workplace", "production_order_format",
         )):
             raise CalculationConfigError("Width context fields require a width-based source mode")
+        if "calibration_ref" in entry:
+            if not rotational:
+                raise CalculationConfigError("calibration_ref requires rotational_discharge")
+            if "specific_discharge_kg_per_rev_m" in entry:
+                raise CalculationConfigError(
+                    "Specify calibration_ref or direct specific discharge, not both"
+                )
+            from production_analytics.calculations.calibrations import (
+                load_calibrations,
+                resolve_calibration,
+            )
+            if calibrations is None:
+                calibrations = load_calibrations(Path(path).parent / "material-calibrations.yaml")
+            calibration = resolve_calibration(
+                calibrations, entry["calibration_ref"],
+                expected_type="rotational_discharge", expected_unit="kg_per_rev_m",
+            )
+            entry["specific_discharge_kg_per_rev_m"] = calibration.value
         if rotational:
             entry["specific_discharge_kg_per_rev_m"] = finite_number(
                 entry.get("specific_discharge_kg_per_rev_m"),
@@ -155,9 +176,24 @@ def load_material_calculation(
             )
         elif "specific_discharge_kg_per_rev_m" in entry:
             raise CalculationConfigError("Specific discharge requires rotational_discharge")
+        if "process_application_calculation_id" in entry:
+            process_id = entry["process_application_calculation_id"]
+            if (not isinstance(process_id, str) or not process_id.strip()
+                    or "\x00" in process_id or process_id == entry["id"]):
+                raise CalculationConfigError(
+                    "process application ID must be a distinct non-empty string"
+                )
+            if not rotational or not entry["gate_signal_ref"] or entry["gate_threshold"] <= 0:
+                raise CalculationConfigError(
+                    "Process application requires rotational_discharge and a positive speed gate"
+                )
         if entry["id"] in instances:
             raise CalculationConfigError("Calculation ids must be unique")
         instances[entry["id"]] = MaterialCalculationConfig(**entry)
+    process_ids = [c.process_application_calculation_id for c in instances.values()
+                   if c.process_application_calculation_id is not None]
+    if len(set(process_ids)) != len(process_ids) or set(process_ids) & instances.keys():
+        raise CalculationConfigError("Calculation ids must be unique across all outputs")
     if calculation_id is not None:
         if calculation_id not in instances:
             raise CalculationConfigError("Requested calculation id was not found")

src/production_analytics/calculations/material_application.py

@@ -0,0 +1,25 @@
+"""Width-independent process application from actual rotational speeds."""
+
+from collections.abc import Iterable
+from math import isfinite
+
+
+def rotational_application_g_m2(
+    rotational_speeds_rpm: Iterable[float], specific_discharge_kg_per_rev_m: float,
+    line_speed_m_min: float, gate_threshold: float,
+) -> float | None:
+    speeds = tuple(rotational_speeds_rpm)
+    if not speeds or not all(isfinite(value) for value in speeds):
+        raise ValueError("rotational speeds must be non-empty and finite")
+    if not isfinite(specific_discharge_kg_per_rev_m) or specific_discharge_kg_per_rev_m <= 0:
+        raise ValueError("specific discharge must be finite and positive")
+    if not isfinite(gate_threshold) or gate_threshold <= 0:
+        raise ValueError("speed gate threshold must be finite and positive")
+    if not isfinite(line_speed_m_min):
+        raise ValueError("line speed must be finite")
+    if line_speed_m_min <= gate_threshold:
+        return None
+    value = sum(speeds) * specific_discharge_kg_per_rev_m * 1000 / line_speed_m_min
+    if not isfinite(value):
+        raise ValueError("derived application must be finite")
+    return value

src/production_analytics/calculations/material_consumption.py

@@ -13,6 +13,7 @@ class MaterialSample:
     timestamp: datetime
     material_rate_kg_per_hour: float
     gate_value: float
+    application_g_m2: float | None = None
 
 
 @dataclass(frozen=True, slots=True)

src/production_analytics/enlyze/gateway.py

@@ -97,6 +97,7 @@ class EnlyzeApiGateway:
         nominal_width_m: float | None = None,
         rotational_speed_variable_ids: tuple[str, ...] = (),
         specific_discharge_kg_per_rev_m: float | None = None,
+        process_application_gate_threshold: float | None = None,
     ) -> list[MaterialSample]:
         for name, value in (("start", start), ("end", end)):
             if value.tzinfo is None or value.utcoffset() is None:
@@ -159,9 +160,21 @@ class EnlyzeApiGateway:
                             rate = area_application_rate_kg_per_hour(sources, nominal_width_m, gate)
                         else:
                             rate = sources[0]
+                        application = None
+                        if process_application_gate_threshold is not None:
+                            if not rotational_speed_variable_ids or gate_variable_id is None:
+                                raise ValueError("Process application requires rpm and speed gate")
+                            from production_analytics.calculations.material_application import (
+                                rotational_application_g_m2,
+                            )
+                            application = rotational_application_g_m2(
+                                sources, specific_discharge_kg_per_rev_m, gate,
+                                process_application_gate_threshold,
+                            )
                         samples.append(MaterialSample(
                             timestamp=_parse_timestamp(record[time_index]),
                             material_rate_kg_per_hour=float(rate), gate_value=float(gate),
+                            application_g_m2=application,
                         ))
                     except (TypeError, ValueError) as exc:
                         raise ValueError(f"malformed record {index}: {exc}") from exc

src/production_analytics/service/material_efficiency.py

@@ -38,12 +38,14 @@ class MaterialEfficiencyService:
         self, erp: WorkplaceStatusReader, materials: MaterialSnapshotRepository, *,
         workplace: str, machine_id: str, calculation_id: str, format_template: str,
         erp_timezone: ZoneInfo | None = None,
+        output_calculation_id: str | None = None,
     ) -> None:
         self.erp = erp
         self.materials = materials
         self.workplace = workplace
         self.machine_id = machine_id
         self.calculation_id = calculation_id
+        self.output_calculation_id = output_calculation_id or calculation_id
         self.format_template = format_template
         self.erp_timezone = erp_timezone
 
@@ -58,7 +60,7 @@ class MaterialEfficiencyService:
         Configuration, timestamp and adapter contract errors raise ValueError.
         Database failures propagate, rather than being treated as missing data.
         """
-        if status.workplace != self.workplace:
+        if status.workplace.strip().casefold() != self.workplace.strip().casefold():
             raise ValueError("ERP workplace does not match configured workplace")
         order = build_enlyze_production_order(status.production_order, self.format_template)
         quantity = status.good_quantity_m2
@@ -88,7 +90,8 @@ class MaterialEfficiencyService:
             return None
         return MaterialEfficiencySnapshot(
             workplace=status.workplace, machine_id=self.machine_id,
-            calculation_id=self.calculation_id, erp_production_order=status.production_order,
+            calculation_id=self.output_calculation_id,
+            erp_production_order=status.production_order,
             enlyze_production_order=order, article_number=status.article_number,
             article_description=status.article_description,
             nominal_width_m=extract_nominal_width_m(status.article_description),

src/production_analytics/service/material_efficiency_runtime.py

@@ -45,10 +45,10 @@ def build_material_efficiency_runner(
     if (
         not isinstance(document, dict)
         or not required <= document.keys()
-        or document.keys() - required - {"poll_interval_seconds"}
+        or document.keys() - required - {"poll_interval_seconds", "output_calculation_id"}
     ):
         raise CalculationConfigError("Invalid material efficiency configuration fields")
-    for name in required:
+    for name in required | ({"output_calculation_id"} & document.keys()):
         value = document[name]
         if not isinstance(value, str) or not value.strip() or "\x00" in value:
             raise CalculationConfigError(f"{name} must be a non-empty string without NUL bytes")
@@ -76,6 +76,7 @@ def build_material_efficiency_runner(
             calculation_id=document["calculation_id"],
             format_template=document["production_order_format"],
             erp_timezone=timezone,
+            output_calculation_id=document.get("output_calculation_id"),
         ),
         PostgresMaterialEfficiencyWriter(postgres),
         poll_interval_seconds=interval,

src/production_analytics/service/material_polling.py

@@ -14,6 +14,7 @@ from production_analytics.enlyze.gateway import (
     EnlyzeApiGateway,
     EnlyzeProductionRun,
 )
+from production_analytics.service.postgres_material_application import MaterialApplicationWriter
 
 
 @dataclass(frozen=True, slots=True)
@@ -58,9 +59,20 @@ class MaterialPollingService:
         rotational_speed_variable_ids: tuple[str, ...] = (),
         specific_discharge_kg_per_rev_m: float | None = None,
         nominal_width_provider: Callable[[str], float] | None = None,
+        process_application_calculation_id: str | None = None,
+        application_writer: MaterialApplicationWriter | None = None,
     ) -> None:
         if application_variable_ids and rotational_speed_variable_ids:
             raise ValueError("Material source modes are mutually exclusive")
+        if process_application_calculation_id is not None and (
+            not rotational_speed_variable_ids or not gate_variable_id
+            or gate_threshold <= 0 or application_writer is None
+        ):
+            raise ValueError(
+                "Process application requires rotational source, positive gate and writer"
+            )
+        self._process_application_calculation_id = process_application_calculation_id
+        self._application_writer = application_writer
         self._rotational_speed_variable_ids = rotational_speed_variable_ids
         self._specific_discharge = specific_discharge_kg_per_rev_m
         self._application_variable_ids = application_variable_ids
@@ -107,7 +119,8 @@ class MaterialPollingService:
                 if previous.end is None or previous.end > run.start:
                     raise ValueError("Historical production run overlaps the current open run")
                 integrated = self._integrate(
-                    initial_state, start=previous.start, end=previous.end, width=width,
+                    initial_state, start=previous.start, end=previous.end,
+                    width=width, run=previous,
                 )
                 initial_state = MaterialIntegrationState(
                     cumulative_consumption_kg=integrated.cumulative_consumption_kg,
@@ -132,7 +145,7 @@ class MaterialPollingService:
         if now < start:
             return None
 
-        state = self._integrate(initial_state, start=start, end=now, width=width)
+        state = self._integrate(initial_state, start=start, end=now, width=width, run=run)
 
         self._state_store.save(
             self._machine_id,
@@ -150,7 +163,7 @@ class MaterialPollingService:
 
     def _integrate(
         self, initial_state: MaterialIntegrationState, *, start: datetime, end: datetime,
-        width: float | None = None,
+        width: float | None = None, run: EnlyzeProductionRun | None = None,
     ) -> MaterialIntegrationState:
         integrator = MaterialConsumptionIntegrator(self._config, initial_state=initial_state)
         source_options = {}
@@ -163,6 +176,8 @@ class MaterialPollingService:
                 rotational_speed_variable_ids=self._rotational_speed_variable_ids,
                 specific_discharge_kg_per_rev_m=self._specific_discharge, nominal_width_m=width,
             )
+        if self._process_application_calculation_id is not None:
+            source_options["process_application_gate_threshold"] = self._config.gate_threshold
         samples = self._gateway.get_material_samples(
             machine_id=self._machine_id,
             rate_variable_id=self._rate_variable_id,
@@ -171,4 +186,13 @@ class MaterialPollingService:
             end=end,
             **source_options,
         )
-        return integrator.process_many(samples)
+        state = integrator.process_many(samples)
+        if self._process_application_calculation_id is not None:
+            assert self._application_writer is not None and run is not None
+            # Persist before advancing the checkpoint, so failed writes can be retried.
+            self._application_writer.write(
+                calculation_id=self._process_application_calculation_id,
+                machine_id=self._machine_id, production_order=run.production_order,
+                run_id=run.uuid, samples=samples,
+            )
+        return state

src/production_analytics/service/material_runtime.py

@@ -26,6 +26,9 @@ from production_analytics.service.postgres_material import (
     PostgresMaterialSnapshotWriter,
     PostgresSettings,
 )
+from production_analytics.service.postgres_material_application import (
+    PostgresMaterialApplicationWriter,
+)
 
 
 class _ReportingStateStore:
@@ -88,6 +91,11 @@ def build_material_runner(
             rotational_speed_variable_ids=calculation.rotational_speed_signal_refs,
             specific_discharge_kg_per_rev_m=calculation.specific_discharge_kg_per_rev_m,
         )
+    if calculation.process_application_calculation_id is not None:
+        source_options.update(
+            process_application_calculation_id=calculation.process_application_calculation_id,
+            application_writer=PostgresMaterialApplicationWriter(postgres_settings),
+        )
     service = MaterialPollingService(
         gateway=EnlyzeApiGateway(ExplorationClient(settings)),
         state_store=_ReportingStateStore(JsonMaterialStateStore(state_directory)),

src/production_analytics/service/postgres_material_application.py

@@ -0,0 +1,53 @@
+"""Generic process application snapshots, timestamped at the source sample."""
+
+from collections.abc import Sequence
+from typing import Protocol
+
+import psycopg
+
+from production_analytics.calculations.config import finite_number
+from production_analytics.calculations.material_consumption import MaterialSample
+from production_analytics.service.postgres_material import PostgresSettings
+
+
+class MaterialApplicationWriter(Protocol):
+    def write(
+        self, *, calculation_id: str, machine_id: str, production_order: str,
+        run_id: str, samples: Sequence[MaterialSample],
+    ) -> None: ...
+
+
+class PostgresMaterialApplicationWriter:
+    def __init__(self, settings: PostgresSettings) -> None:
+        self.settings = settings
+
+    def write(
+        self, *, calculation_id: str, machine_id: str, production_order: str,
+        run_id: str, samples: Sequence[MaterialSample],
+    ) -> None:
+        rows = []
+        for sample in samples:
+            if sample.application_g_m2 is None:
+                continue
+            if sample.timestamp.utcoffset() is None:
+                raise ValueError("Application timestamp must be timezone-aware")
+            finite_number(sample.application_g_m2, "application_g_m2")
+            rows.append((sample.timestamp, calculation_id, machine_id, production_order,
+                         run_id, sample.application_g_m2))
+        if not rows:
+            return
+        with psycopg.connect(
+            host=self.settings.host, port=self.settings.port, dbname=self.settings.dbname,
+            user=self.settings.user, password=self.settings.password, connect_timeout=10,
+            options="-c statement_timeout=10000",
+        ) as connection:
+            with connection.cursor() as cursor:
+                cursor.executemany(
+                    """INSERT INTO material_application_snapshots
+                       (timestamp, calculation_id, machine_id, production_order,
+                        run_id, application_g_m2)
+                       VALUES (%s, %s, %s, %s, %s, %s)
+                       ON CONFLICT (calculation_id, machine_id, production_order, timestamp)
+                       DO NOTHING""",
+                    rows,
+                )