From 5d1f5199d318d00c1cf3fd304dda1737be2ee0aa Mon Sep 17 00:00:00 2001 From: foefl Date: Fri, 11 Apr 2025 10:37:49 +0200 Subject: [PATCH] prototype ideas --- src/delta_barth/analysis/forecast.py | 76 ++++++++++++++++++---------- src/delta_barth/constants.py | 4 +- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/src/delta_barth/analysis/forecast.py b/src/delta_barth/analysis/forecast.py index bc402bc..55e42ac 100644 --- a/src/delta_barth/analysis/forecast.py +++ b/src/delta_barth/analysis/forecast.py @@ -1,8 +1,7 @@ from __future__ import annotations +import copy import datetime -# --- new: for calculating timedelta -from dateutil.relativedelta import relativedelta import math from collections.abc import Mapping, Set from dataclasses import asdict @@ -13,6 +12,9 @@ import numpy as np import pandas as pd import scipy.stats import sqlalchemy as sql + +# --- new: for calculating timedelta +from dateutil.relativedelta import relativedelta from sklearn.metrics import mean_absolute_error, r2_score from sklearn.model_selection import KFold, RandomizedSearchCV from xgboost import XGBRegressor @@ -185,16 +187,14 @@ def _process_sales( PipeResult _description_ """ - # cust_data: CustomerDataSalesForecast = CustomerDataSalesForecast() - # filter data data = pipe.data assert data is not None, "processing not existing pipe result" DATE_FEAT: Final[str] = "buchungs_datum" SALES_FEAT: Final[str] = "betrag" - df_firma = data[(data["betrag"] > 0)] - df_cust = df_firma.copy() + df_filter = data[(data["betrag"] > 0)] + df_cust = df_filter.copy() df_cust = df_cust.sort_values(by=DATE_FEAT).reset_index() len_ds = len(df_cust) @@ -218,9 +218,11 @@ def _process_sales( features = ["jahr", "monat"] target = SALES_FEAT - # --- new: dates and forecast - last_date = pd.to_datetime(datetime.now().strftime("%m.%Y"), format="%m.%Y") - future_dates = pd.date_range(start=last_date + pd.DateOffset(months=1), periods=6, freq="MS") + # ?? --- new: dates and forecast + last_date = pd.to_datetime(datetime.datetime.now().strftime("%m.%Y"), format="%m.%Y") + future_dates = pd.date_range( + start=last_date + pd.DateOffset(months=1), periods=6, freq="MS" + ) forecast = pd.DataFrame({"datum": future_dates.strftime("%m.%Y")}).set_index("datum") # Randomized Search @@ -236,30 +238,42 @@ def _process_sales( "early_stopping_rounds": [20, 50], } - # --- new: best_estimator (internal usage only) + # ?? --- new: best_estimator (internal usage only) best_estimator = None - best_params: BestParametersXGBRegressor | None = None best_score_mae: float | None = float("inf") best_score_r2: float | None = None best_start_year: int | None = None too_few_month_points: bool = True - forecast: pd.DataFrame | None = None - + # forecast: pd.DataFrame | None = None + # TODO: write routine to pad missing values in datetime row + # TODO problem: continuous timeline expected, but values can be empty for multiple months + # TODO: therefore, stepping with fixed value n does not result in timedelta of n episodes + # Option A: pad data frame with zero values --> could impede forecast algorithm + # Option B: calculate next index based on timedelta dates = monthly_sum.index - # --- new: use monthly basis for time windows - starting_date = datetime.now() - relativedelta(months=36) - #starting_date = dates.max() - relativedelta(months=36) - start_index = next((i for i, date in enumerate(dates) if date >= starting_date), len(dates) - 1) + # print("dates: ", dates) + # ?? --- new: use monthly basis for time windows + # baseline: 3 years - 36 months + starting_date = datetime.datetime.now() - relativedelta(months=12) + # starting_date = dates.max() - relativedelta(months=36) + start_index = next( + (i for i, date in enumerate(dates) if date >= starting_date), len(dates) - 1 + ) + print("start idx: ", start_index, "length dates: ", len(dates)) - for index, i in enumerate(range(start_index, -1, -12)): - first_date = dates[i] + for add_year, date_idx in enumerate(range(start_index, -1, -12)): + print("date_idx: ", date_idx) + first_date = dates[date_idx] + print("first date: ", first_date) split_date = dates[-6] train = cast( pd.DataFrame, monthly_sum.loc[first_date:split_date].copy(), # type: ignore ) + print(train) + print("Length train: ", len(train)) test = cast( pd.DataFrame, monthly_sum.loc[split_date:].copy(), # type: ignore @@ -267,8 +281,10 @@ def _process_sales( X_train, X_test = train[features], test[features] y_train, y_test = train[target], test[target] - # --- new: adapted condition to fit new for-loop - if len(train) >= 30 + 10 * index: + # ?? --- new: adapted condition to fit new for-loop + # test set size fixed at 6 --> first iteration: baseline - 6 entries + # for each new year 10 new data points needed + if len(train) >= 30 + 10 * add_year: too_few_month_points = False rand = RandomizedSearchCV( @@ -294,14 +310,16 @@ def _process_sales( # --- new: use first_date for best_start_year best_start_year = first_date.year # --- new: store best_estimator - best_estimator = rand.best_estimator_ + best_estimator = copy.copy(rand.best_estimator_) - # --- new: use best_estimator to calculate future values and store them in forecast + # ?? --- new: use best_estimator to calculate future values and store them in forecast if best_estimator is not None: - X_future = pd.DataFrame({"jahr": future_dates.year, "monat": future_dates.month}, index=future_dates) - y_future = best_estimator.predict(X_future) - forecast["vorhersage"] = y_future - + X_future = pd.DataFrame( + {"jahr": future_dates.year, "monat": future_dates.month}, index=future_dates + ) + y_future = best_estimator.predict(X_future) # type: ignore + forecast["vorhersage"] = y_future + best_score_mae = best_score_mae if not math.isinf(best_score_mae) else None if too_few_month_points: @@ -317,7 +335,9 @@ def _process_sales( pipe.stats(stats) return pipe - assert forecast is not None, "forecast is None, but was attempted to be returned" + assert "vorhersage" in forecast.columns, ( + "forecast does not contain prognosis values, but was attempted to be returned" + ) status = STATUS_HANDLER.SUCCESS pipe.success(forecast, status) stats = SalesForecastStatistics( diff --git a/src/delta_barth/constants.py b/src/delta_barth/constants.py index 45c74ba..5f163ae 100644 --- a/src/delta_barth/constants.py +++ b/src/delta_barth/constants.py @@ -17,7 +17,7 @@ DUMMY_DATA_PATH: Final[Path] = dummy_data_pth # ** logging ENABLE_LOGGING: Final[bool] = True LOGGING_TO_FILE: Final[bool] = True -LOGGING_TO_STDERR: Final[bool] = True +LOGGING_TO_STDERR: Final[bool] = False LOG_FILENAME: Final[str] = "dopt-delbar.log" # ** databases @@ -40,7 +40,7 @@ class KnownDelBarApiErrorCodes(enum.Enum): # ** API -API_CON_TIMEOUT: Final[float] = 5.0 # secs to response +API_CON_TIMEOUT: Final[float] = 10.0 # secs to response # ** API response parsing # ** column mapping [API-Response --> Target-Features] COL_MAP_SALES_PROGNOSIS: Final[DualDict[str, str]] = DualDict(