124 lines
3.8 KiB
Python
124 lines
3.8 KiB
Python
"""collection of configured data pipelines, intended to be invoked from C#"""
|
|
|
|
import time
|
|
from datetime import datetime as Datetime
|
|
from typing import Final
|
|
|
|
import sqlalchemy as sql
|
|
|
|
from delta_barth import databases as db
|
|
from delta_barth.analysis import forecast
|
|
from delta_barth.constants import DEFAULT_DB_ERR_CODE
|
|
from delta_barth.errors import STATUS_HANDLER, wrap_result
|
|
from delta_barth.logging import logger_pipelines as logger
|
|
from delta_barth.management import SESSION
|
|
from delta_barth.types import JsonExportResponse, PipelineMetrics
|
|
|
|
|
|
def _write_performance_metrics(
|
|
pipeline_name: str,
|
|
time_start: int,
|
|
time_end: int,
|
|
) -> PipelineMetrics:
|
|
if time_end < time_start:
|
|
raise ValueError("Ending time smaller than starting time")
|
|
execution_duration = (time_end - time_start) / 1e9
|
|
metrics = PipelineMetrics(
|
|
pipeline_name=pipeline_name,
|
|
execution_duration=execution_duration,
|
|
)
|
|
|
|
with SESSION.db_engine.begin() as con:
|
|
con.execute(sql.insert(db.perf_meas).values(**metrics))
|
|
|
|
return metrics
|
|
|
|
|
|
@wrap_result(code_on_error=DEFAULT_DB_ERR_CODE)
|
|
def _write_performance_metrics_wrapped(
|
|
pipeline_name: str,
|
|
time_start: int,
|
|
time_end: int,
|
|
) -> PipelineMetrics:
|
|
return _write_performance_metrics(pipeline_name, time_start, time_end)
|
|
|
|
|
|
def pipeline_sales_forecast(
|
|
company_id: int | None,
|
|
start_date: Datetime | None,
|
|
) -> JsonExportResponse:
|
|
PIPELINE_NAME: Final[str] = "sales_forecast"
|
|
logger.info("[EXT-CALL PIPELINES] Starting main sales forecast pipeline...")
|
|
t_start = time.perf_counter_ns()
|
|
result = forecast.pipeline_sales_forecast(
|
|
SESSION, company_id=company_id, start_date=start_date
|
|
)
|
|
export = JsonExportResponse(result.model_dump_json())
|
|
t_end = time.perf_counter_ns()
|
|
logger.info("[EXT-CALL PIPELINES] Main sales forecast pipeline successful")
|
|
logger.info("[EXT-CALL PIPELINES] Writing performance metrics...")
|
|
res = _write_performance_metrics_wrapped(
|
|
pipeline_name=PIPELINE_NAME,
|
|
time_start=t_start,
|
|
time_end=t_end,
|
|
)
|
|
if res.status != STATUS_HANDLER.SUCCESS:
|
|
logger.error(
|
|
(
|
|
"[DB-WRITE][METRICS] Pipeline: >%s< - Error on writing "
|
|
"pipeline metrics to database: %s"
|
|
),
|
|
PIPELINE_NAME,
|
|
res.status,
|
|
)
|
|
else:
|
|
metrics = res.unwrap()
|
|
logger.info(
|
|
"[METRICS] Pipeline: >%s< - Execution time: %.6f",
|
|
PIPELINE_NAME,
|
|
metrics["execution_duration"],
|
|
)
|
|
|
|
return export
|
|
|
|
|
|
def pipeline_sales_forecast_dummy(
|
|
company_id: int | None,
|
|
start_date: Datetime | None,
|
|
) -> JsonExportResponse:
|
|
PIPELINE_NAME: Final[str] = "sales_forecast_dummy"
|
|
logger.info("[EXT-CALL PIPELINES] Starting dummy sales forecast pipeline...")
|
|
t_start = time.perf_counter_ns()
|
|
result = forecast.pipeline_sales_dummy(
|
|
SESSION,
|
|
company_id=company_id,
|
|
start_date=start_date,
|
|
)
|
|
export = JsonExportResponse(result.model_dump_json())
|
|
t_end = time.perf_counter_ns()
|
|
logger.info("[EXT-CALL PIPELINES] Dummy sales forecast pipeline successful")
|
|
logger.info("[EXT-CALL PIPELINES] Writing performance metrics...")
|
|
res = _write_performance_metrics_wrapped(
|
|
pipeline_name=PIPELINE_NAME,
|
|
time_start=t_start,
|
|
time_end=t_end,
|
|
)
|
|
if res.status != STATUS_HANDLER.SUCCESS:
|
|
logger.error(
|
|
(
|
|
"[DB-WRITE][METRICS] Pipeline: >%s< - Error on writing "
|
|
"pipeline metrics to database: %s"
|
|
),
|
|
PIPELINE_NAME,
|
|
res.status,
|
|
)
|
|
else:
|
|
metrics = res.unwrap()
|
|
logger.info(
|
|
"[METRICS] Pipeline: >%s< - Execution time: %.6f",
|
|
PIPELINE_NAME,
|
|
metrics["execution_duration"],
|
|
)
|
|
|
|
return export
|