Add central material calibration support
This commit is contained in:
@@ -177,11 +177,11 @@ def test_bento_rpm_bootstrap_and_persistent_resume(tmp_path, inter_run_gap_secon
|
||||
partial = service().poll_once(now=runs[1].start + timedelta(seconds=300))
|
||||
result = service().poll_once(now=runs[1].end)
|
||||
assert result.state.cumulative_consumption_kg - partial.state.cumulative_consumption_kg == (
|
||||
pytest.approx((3.739 + 3.956) * 5 * 3.12 * 326 / 60)
|
||||
pytest.approx((3.739 + 3.956) * 5 * 2.75 * 326 / 60)
|
||||
)
|
||||
assert result.state.integrated_running_seconds / 60 == pytest.approx(1256)
|
||||
assert result.state.cumulative_consumption_kg == pytest.approx(
|
||||
(3.739 + 3.956) * 5 * 3.12 * 1256,
|
||||
(3.739 + 3.956) * 5 * 2.75 * 1256,
|
||||
)
|
||||
assert service().poll_once(now=runs[1].end).state == result.state
|
||||
assert gateway.get_production_runs.call_count == 1
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
from dataclasses import replace
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from production_analytics.calculations.calibrations import load_calibrations, resolve_calibration
|
||||
from production_analytics.calculations.config import (
|
||||
CalculationConfigError,
|
||||
load_material_calculation,
|
||||
)
|
||||
from production_analytics.calculations.material_application import rotational_application_g_m2
|
||||
from production_analytics.calculations.material_consumption import (
|
||||
MaterialConsumptionIntegrator,
|
||||
MaterialIntegratorConfig,
|
||||
)
|
||||
from production_analytics.enlyze.gateway import EnlyzeApiGateway
|
||||
|
||||
CALIBRATIONS = Path('config/material-calibrations.yaml')
|
||||
BENTO = Path('config/bento1-material-consumption.yaml')
|
||||
ID = 'bento1-spreader-1-2'
|
||||
|
||||
|
||||
def resolve(entries, identifier=ID):
|
||||
return resolve_calibration(entries, identifier, expected_type='rotational_discharge',
|
||||
expected_unit='kg_per_rev_m')
|
||||
|
||||
|
||||
def test_load_and_resolve():
|
||||
calibration = resolve(load_calibrations(CALIBRATIONS))
|
||||
assert calibration.value == 2.75
|
||||
assert calibration.calibrated_at == date(2026, 9, 8)
|
||||
assert calibration.method == 'gravimetric_tray'
|
||||
assert calibration.reference == {
|
||||
'measured_application_g_m2': 4068, 'line_speed_m_min': 2.3,
|
||||
'signal_values': {'left': 1.65, 'right': 1.75},
|
||||
}
|
||||
assert load_material_calculation(BENTO).specific_discharge_kg_per_rev_m == 2.75
|
||||
|
||||
|
||||
@pytest.mark.parametrize('field', ['type', 'value', 'unit', 'calibrated_at', 'method',
|
||||
'description', 'reference'])
|
||||
def test_missing_metadata(tmp_path, field):
|
||||
document = yaml.safe_load(CALIBRATIONS.read_text())
|
||||
del document['calibrations'][ID][field]
|
||||
path = tmp_path / 'material-calibrations.yaml'
|
||||
path.write_text(yaml.safe_dump(document))
|
||||
with pytest.raises(CalculationConfigError, match='required fields'):
|
||||
load_calibrations(path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('field,value', [
|
||||
('type', ''), ('unit', None), ('method', 12), ('description', ' '),
|
||||
('calibrated_at', '2026-02-30'), ('calibrated_at', 123),
|
||||
('reference', []), ('reference', {}), ('value', True), ('value', float('nan')),
|
||||
])
|
||||
def test_invalid_metadata(tmp_path, field, value):
|
||||
document = yaml.safe_load(CALIBRATIONS.read_text())
|
||||
document['calibrations'][ID][field] = value
|
||||
path = tmp_path / 'material-calibrations.yaml'
|
||||
path.write_text(yaml.safe_dump(document))
|
||||
with pytest.raises(CalculationConfigError, match=field):
|
||||
load_calibrations(path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('field,value,error', [
|
||||
('type', 'other', 'incompatible type'), ('unit', 'kg_per_rev', 'incompatible unit'),
|
||||
('value', 0, 'greater than zero'),
|
||||
])
|
||||
def test_incompatible_at_config_load(tmp_path, field, value, error):
|
||||
document = yaml.safe_load(CALIBRATIONS.read_text())
|
||||
document['calibrations'][ID][field] = value
|
||||
(tmp_path / CALIBRATIONS.name).write_text(yaml.safe_dump(document))
|
||||
path = tmp_path / BENTO.name
|
||||
path.write_text(BENTO.read_text())
|
||||
with pytest.raises(CalculationConfigError, match=error):
|
||||
load_material_calculation(path)
|
||||
|
||||
|
||||
def test_missing_id_and_file(tmp_path):
|
||||
with pytest.raises(CalculationConfigError, match='Missing calibration ID'):
|
||||
resolve(load_calibrations(CALIBRATIONS), 'unknown')
|
||||
path = tmp_path / BENTO.name
|
||||
path.write_text(BENTO.read_text())
|
||||
with pytest.raises(CalculationConfigError, match='Cannot read calibration'):
|
||||
load_material_calculation(path)
|
||||
(tmp_path / CALIBRATIONS.name).write_text(CALIBRATIONS.read_text())
|
||||
path.write_text(BENTO.read_text().replace(ID, 'unknown'))
|
||||
with pytest.raises(CalculationConfigError, match='Missing calibration ID'):
|
||||
load_material_calculation(path)
|
||||
|
||||
|
||||
def test_exact_direct_value_equivalence(tmp_path):
|
||||
referenced = load_material_calculation(BENTO)
|
||||
document = yaml.safe_load(BENTO.read_text())
|
||||
entry = document['calculations'][0]
|
||||
del entry['calibration_ref']
|
||||
entry['specific_discharge_kg_per_rev_m'] = 2.75
|
||||
path = tmp_path / 'direct.yaml'
|
||||
path.write_text(yaml.safe_dump(document))
|
||||
direct = load_material_calculation(path)
|
||||
assert replace(referenced, calibration_ref=None) == direct
|
||||
start = datetime(2026, 9, 8, tzinfo=UTC)
|
||||
outputs = []
|
||||
for config in (referenced, direct):
|
||||
factor = config.specific_discharge_kg_per_rev_m
|
||||
application = rotational_application_g_m2([1.65, 1.75], factor, 2.3, 0.3)
|
||||
client = Mock()
|
||||
client.post_json.return_value.body = {'data': {
|
||||
'columns': ['time', 'left', 'right', 'speed'],
|
||||
'records': [[(start + timedelta(seconds=t)).isoformat(), 1.65, 1.75, speed]
|
||||
for t, speed in [(0, 2.3), (10, 0), (20, 2.3), (50, 2.3), (60, 2.3)]],
|
||||
}}
|
||||
samples = EnlyzeApiGateway(client).get_material_samples(
|
||||
machine_id='m', rate_variable_id='unused', gate_variable_id='speed',
|
||||
rotational_speed_variable_ids=('left', 'right'), nominal_width_m=5,
|
||||
specific_discharge_kg_per_rev_m=factor, start=start,
|
||||
end=start + timedelta(seconds=60),
|
||||
)
|
||||
integrator = MaterialConsumptionIntegrator(MaterialIntegratorConfig(0.3, 20))
|
||||
state = integrator.process_many(samples)
|
||||
assert samples[0].material_rate_kg_per_hour == 2805.0
|
||||
assert state.integrated_running_seconds == 20
|
||||
outputs.append((application, samples, state))
|
||||
assert outputs[0] == outputs[1]
|
||||
assert outputs[0][0] == pytest.approx(4065.217391304348)
|
||||
|
||||
|
||||
def test_update_only_central_file(tmp_path):
|
||||
path = tmp_path / BENTO.name
|
||||
path.write_text(BENTO.read_text())
|
||||
(tmp_path / CALIBRATIONS.name).write_text(CALIBRATIONS.read_text().replace('2.75', '2.8'))
|
||||
assert load_material_calculation(path).specific_discharge_kg_per_rev_m == 2.8
|
||||
|
||||
|
||||
def test_k7_without_calibration_file(tmp_path):
|
||||
original = Path('config/k7-material-consumption.yaml')
|
||||
path = tmp_path / original.name
|
||||
path.write_text(original.read_text())
|
||||
assert load_material_calculation(path) == load_material_calculation(original)
|
||||
assert load_material_calculation(path).calibration_ref is None
|
||||
@@ -0,0 +1,158 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from production_analytics.calculations.config import (
|
||||
CalculationConfigError,
|
||||
load_material_calculation,
|
||||
)
|
||||
from production_analytics.calculations.material_application import rotational_application_g_m2
|
||||
from production_analytics.calculations.material_consumption import MaterialSample
|
||||
from production_analytics.enlyze.gateway import EnlyzeApiGateway, EnlyzeProductionRun
|
||||
from production_analytics.service.material_polling import MaterialPollingService
|
||||
from production_analytics.service.postgres_material import PostgresSettings
|
||||
from production_analytics.service.postgres_material_application import (
|
||||
PostgresMaterialApplicationWriter,
|
||||
)
|
||||
|
||||
NOW = datetime(2026, 9, 7, tzinfo=UTC)
|
||||
|
||||
|
||||
def test_realistic_application_and_generic_factor():
|
||||
assert rotational_application_g_m2([3.739, 3.956], 3.12, 8, 0.3) == pytest.approx(3001.05)
|
||||
assert rotational_application_g_m2([2, 3], 1.5, 10, 0.5) == 750
|
||||
|
||||
|
||||
@pytest.mark.parametrize('speed', [-10, 0, 1e-300, 0.299999999, 0.3])
|
||||
def test_inactive_and_near_zero(speed):
|
||||
assert rotational_application_g_m2([3.739, 3.956], 3.12, speed, 0.3) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize('speeds,factor,speed,threshold', [
|
||||
([], 3.12, 8, 0.3), ([float('nan')], 3.12, 8, 0.3),
|
||||
([1], 0, 8, 0.3), ([1], float('inf'), 8, 0.3),
|
||||
([1], 3.12, float('nan'), 0.3), ([1], 3.12, 8, 0),
|
||||
([1], 3.12, 8, float('inf')), ([1e308], 3.12, 8, 0.3),
|
||||
])
|
||||
def test_invalid_inputs(speeds, factor, speed, threshold):
|
||||
with pytest.raises(ValueError):
|
||||
rotational_application_g_m2(speeds, factor, speed, threshold)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('width', [1, 4.85, 5, 100])
|
||||
def test_gateway_width_cancels_and_only_act_signals_requested(width):
|
||||
config = load_material_calculation('config/bento1-material-consumption.yaml')
|
||||
refs = [*config.rotational_speed_signal_refs, config.gate_signal_ref]
|
||||
client = Mock()
|
||||
client.post_json.return_value.body = {'data': {
|
||||
'columns': ['time', *refs],
|
||||
'records': [[NOW.isoformat(), 3.739, 3.956, 8],
|
||||
[(NOW + timedelta(seconds=10)).isoformat(), 3.739, 3.956, 0]],
|
||||
}}
|
||||
samples = EnlyzeApiGateway(client).get_material_samples(
|
||||
machine_id=config.machine_ref, rate_variable_id='unused',
|
||||
gate_variable_id=config.gate_signal_ref, start=NOW, end=NOW + timedelta(seconds=10),
|
||||
rotational_speed_variable_ids=config.rotational_speed_signal_refs,
|
||||
nominal_width_m=width, specific_discharge_kg_per_rev_m=3.12,
|
||||
process_application_gate_threshold=0.3,
|
||||
)
|
||||
assert samples[0].application_g_m2 == pytest.approx(3001.05)
|
||||
assert samples[1].application_g_m2 is None
|
||||
assert samples[0].material_rate_kg_per_hour == pytest.approx(7.695 * width * 3.12 * 60)
|
||||
assert client.post_json.call_args.args[1]['variables'] == [{'uuid': ref} for ref in refs]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('changes', [
|
||||
{'process_application_calculation_id': ''}, {'process_application_calculation_id': None},
|
||||
{'process_application_calculation_id': 3}, {'process_application_calculation_id': 'x\x00'},
|
||||
{'process_application_calculation_id': 'bento1-fresh-bentonite-consumption'},
|
||||
{'gate_signal_ref': None}, {'gate_threshold': 0}, {'gate_threshold': -0.3},
|
||||
])
|
||||
def test_process_config_validation(tmp_path, changes):
|
||||
document = yaml.safe_load(Path('config/bento1-material-consumption.yaml').read_text())
|
||||
document['calculations'][0].update(changes)
|
||||
path = tmp_path / 'config.yaml'
|
||||
path.write_text(yaml.safe_dump(document))
|
||||
with pytest.raises(CalculationConfigError):
|
||||
load_material_calculation(path)
|
||||
|
||||
|
||||
def test_persistence_before_checkpoint_retries_and_disjoint_runs():
|
||||
old = EnlyzeProductionRun('old', 'machine', None, 'order', NOW, NOW + timedelta(seconds=10))
|
||||
current = EnlyzeProductionRun(
|
||||
'new', 'machine', None, 'order', NOW + timedelta(seconds=15), None,
|
||||
)
|
||||
gateway = Mock()
|
||||
gateway.get_open_production_run.return_value = current
|
||||
gateway.get_production_runs.return_value = [old, current]
|
||||
|
||||
def samples(**kwargs):
|
||||
start = kwargs['start']
|
||||
assert kwargs['process_application_gate_threshold'] == 0.3
|
||||
return [MaterialSample(start, 7202.52, 8, 3001.05),
|
||||
MaterialSample(start + timedelta(seconds=10), 7202.52, 8, 3001.05)]
|
||||
|
||||
gateway.get_material_samples.side_effect = samples
|
||||
store = Mock(load=Mock(return_value=None))
|
||||
writer = Mock()
|
||||
writer.write.side_effect = [None, psycopg.OperationalError(), None, None]
|
||||
service = MaterialPollingService(
|
||||
gateway=gateway, state_store=store, machine_id='machine', rate_variable_id='unused',
|
||||
gate_variable_id='speed', gate_threshold=0.3, max_sample_gap_seconds=20,
|
||||
rotational_speed_variable_ids=('right', 'left'), specific_discharge_kg_per_rev_m=3.12,
|
||||
nominal_width_provider=lambda order: 5, process_application_calculation_id='application',
|
||||
application_writer=writer,
|
||||
)
|
||||
with pytest.raises(psycopg.OperationalError):
|
||||
service.poll_once(now=NOW + timedelta(seconds=25))
|
||||
store.save.assert_not_called()
|
||||
result = service.poll_once(now=NOW + timedelta(seconds=25))
|
||||
assert result.state.cumulative_consumption_kg == pytest.approx(40.014)
|
||||
assert result.state.integrated_running_seconds == 20
|
||||
assert [c.kwargs['run_id'] for c in writer.write.call_args_list] == ['old', 'new', 'old', 'new']
|
||||
store.save.assert_called_once()
|
||||
|
||||
|
||||
def test_sql_persistence_idempotent_and_inactive_omitted(monkeypatch):
|
||||
import sqlite3
|
||||
|
||||
database = sqlite3.connect(':memory:')
|
||||
database.executescript(Path('db/schema.sql').read_text())
|
||||
connection = MagicMock()
|
||||
cursor = connection.__enter__.return_value.cursor.return_value.__enter__.return_value
|
||||
cursor.executemany.side_effect = lambda sql, rows: database.executemany(
|
||||
sql.replace('%s', '?'),
|
||||
[(row[0].isoformat(), *row[1:]) for row in rows],
|
||||
)
|
||||
connect = Mock(return_value=connection)
|
||||
monkeypatch.setattr(psycopg, 'connect', connect)
|
||||
writer = PostgresMaterialApplicationWriter(PostgresSettings('host', 5432, 'db', 'u', 'p'))
|
||||
kwargs = dict(calculation_id='application', machine_id='machine', production_order='order',
|
||||
run_id='run', samples=[MaterialSample(NOW, 7202.52, 8, 3001.05),
|
||||
MaterialSample(NOW + timedelta(seconds=10), 7202.52, 0)])
|
||||
writer.write(**kwargs)
|
||||
writer.write(**kwargs)
|
||||
assert database.execute('SELECT * FROM material_application_snapshots').fetchall() == [
|
||||
(NOW.isoformat(), 'application', 'machine', 'order', 'run', 3001.05),
|
||||
]
|
||||
writer.write(**(kwargs | {'samples': [MaterialSample(NOW, 1, 0)]}))
|
||||
assert connect.call_count == 2
|
||||
database.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('collision', ['output', 'consumption'])
|
||||
def test_output_ids_unique_across_config_entries(tmp_path, collision):
|
||||
document = yaml.safe_load(Path('config/bento1-material-consumption.yaml').read_text())
|
||||
first = document['calculations'][0]
|
||||
second = dict(first, id='second-consumption')
|
||||
if collision == 'consumption':
|
||||
second['process_application_calculation_id'] = first['id']
|
||||
document['calculations'].append(second)
|
||||
path = tmp_path / 'config.yaml'
|
||||
path.write_text(yaml.safe_dump(document))
|
||||
with pytest.raises(CalculationConfigError, match='unique'):
|
||||
load_material_calculation(path, first['id'])
|
||||
@@ -445,3 +445,44 @@ def test_commit_failure_does_not_log_success(storage):
|
||||
)
|
||||
assert output.getvalue() == ""
|
||||
assert errors == "Material efficiency cycle failed: OperationalError\n"
|
||||
|
||||
|
||||
def test_bento_uses_generic_efficiency_and_distinct_output_id(runtime_files, storage):
|
||||
_, postgres, erp = runtime_files
|
||||
runner = build_material_efficiency_runner(
|
||||
Path('config/bento1-material-efficiency.yaml'),
|
||||
secrets_file=postgres, erp_secrets_file=erp,
|
||||
)
|
||||
from production_analytics.service.material_efficiency import MaterialEfficiencyService
|
||||
|
||||
assert type(runner.service) is MaterialEfficiencyService
|
||||
service = runner.service
|
||||
status = CurrentWorkplaceStatus(
|
||||
'Bento 1', '00123', None, None, NOW, None, 800, None, None, None,
|
||||
)
|
||||
service.erp = Mock(get_current_workplace_status=Mock(return_value=status))
|
||||
service.materials = Mock(latest_at_or_before=Mock(return_value=MaterialConsumptionSnapshot(
|
||||
NOW, 'bento1-fresh-bentonite-consumption', service.machine_id, 'Bento 1-00123', 'run', 2400,
|
||||
)))
|
||||
snapshot = service.evaluate_current()
|
||||
assert snapshot.calculation_id == 'bento1-fresh-bentonite-efficiency'
|
||||
assert snapshot.material_consumption_g_per_m2 == 3000
|
||||
assert snapshot.nominal_width_m is None
|
||||
service.materials.latest_at_or_before.assert_called_once_with(
|
||||
calculation_id='bento1-fresh-bentonite-consumption', machine_id=service.machine_id,
|
||||
production_order='Bento 1-00123', timestamp=NOW,
|
||||
)
|
||||
runner.writer.write(snapshot)
|
||||
row = storage[0].execute(
|
||||
'SELECT calculation_id, material_consumption_g_per_m2 FROM material_efficiency_snapshots'
|
||||
).fetchone()
|
||||
assert row == ('bento1-fresh-bentonite-efficiency', 3000)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('value', ['', None, 12, 'bad\x00id'])
|
||||
def test_invalid_output_calculation_id(runtime_files, value):
|
||||
config, postgres, erp = runtime_files
|
||||
document = yaml.safe_load(config.read_text()) | {'output_calculation_id': value}
|
||||
config.write_text(yaml.safe_dump(document))
|
||||
with pytest.raises(CalculationConfigError, match='output_calculation_id'):
|
||||
build_material_efficiency_runner(config, secrets_file=postgres, erp_secrets_file=erp)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
@@ -78,10 +79,13 @@ def test_invalid_config(tmp_path, field, value):
|
||||
|
||||
def test_optional_gate_config(tmp_path):
|
||||
document = yaml.safe_load(open('config/bento1-material-consumption.yaml'))
|
||||
del document['calculations'][0]['process_application_calculation_id']
|
||||
del document['calculations'][0]['gate_signal_ref']
|
||||
del document['calculations'][0]['gate_threshold']
|
||||
path = tmp_path / 'config.yaml'
|
||||
path.write_text(yaml.safe_dump(document))
|
||||
(tmp_path / 'material-calibrations.yaml').write_text(
|
||||
Path('config/material-calibrations.yaml').read_text())
|
||||
config = load_material_calculation(path)
|
||||
assert config.gate_signal_ref is None
|
||||
assert config.gate_threshold == 0
|
||||
@@ -102,11 +106,16 @@ def test_bento_runtime_wiring(tmp_path, monkeypatch):
|
||||
)
|
||||
service = runner.service
|
||||
assert config.source_mode == 'rotational_discharge'
|
||||
assert service._process_application_calculation_id == 'bento1-fresh-bentonite-application'
|
||||
from production_analytics.service.postgres_material_application import (
|
||||
PostgresMaterialApplicationWriter,
|
||||
)
|
||||
assert isinstance(service._application_writer, PostgresMaterialApplicationWriter)
|
||||
assert service._rotational_speed_variable_ids == (
|
||||
'6e5d2d94-98f9-4cc1-8a88-7987c6282525', 'cd7385c4-337b-4759-ab32-45d65beaf190',
|
||||
)
|
||||
assert service._application_variable_ids == ()
|
||||
assert service._specific_discharge == 3.12
|
||||
assert service._specific_discharge == 2.75
|
||||
assert service._gate_variable_id == 'fef41976-1103-4090-b780-eaeecc02fdfa'
|
||||
assert service._config == MaterialIntegratorConfig(0.3, 20)
|
||||
assert service._nominal_width_provider.workplace == 'Bento 1'
|
||||
|
||||
Reference in New Issue
Block a user