Umbreit/umbreit-py
1
0
Fork 0
generated from dopt-python/py311
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

writing results and temp tables

Browse Source
This commit is contained in:
Florian Förster
2026-01-16 14:13:55 +01:00
parent 4ea9d35a91
commit 3b2e0e5773
4 changed files with 247 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
data_analysis/02-3_oracle_workflow_test.py
View File

@@ -23,7 +23,6 @@ from umbreit import db, types
# types = importlib.reload(types)
# db = importlib.reload(db)
# %%
p_cfg = io.search_file_iterative(
starting_path=Path.cwd(),
@@ -61,7 +60,7 @@ def set_fetch_sizes(conn, cursor, statement, parameters, context, executemany):
########### RESULTS ###########
# temporary
res_engine = sql.create_engine("sqlite://")
db.metadata.create_all(res_engine, tables=(db.results,))
db.metadata.create_all(res_engine, tables=(db.results_local,))
# %%
@@ -70,13 +69,13 @@ def delete_results(
res_engine: sql.Engine,
) -> None:
with res_engine.begin() as conn:
res = conn.execute(sql.delete(db.results))
res = conn.execute(sql.delete(db.results_local))
print("Rows deleted: ", res.rowcount)
delete_results(res_engine)
stmt = sql.select(db.results.c.bedarf_nr, db.results.c.bedarf_sequenz)
stmt = sql.select(db.results_local.c.bedarf_nr, db.results_local.c.bedarf_sequenz)
with res_engine.connect() as conn:
res = conn.execute(stmt)
print(res.all())
@@ -173,9 +172,9 @@ df_order
# %%
# df.select(pl.col("BEDP_MENGE_BEDARF").is_null().sum())
print(f"Query duration: {elapsed:.4f} sec")
print("Number of entries: ", len(df))
print(f"Estimated size in memory: {df.estimated_size(unit='mb')} MB")
# print(f"Query duration: {elapsed:.4f} sec")
# print("Number of entries: ", len(df))
# print(f"Estimated size in memory: {df.estimated_size(unit='mb')} MB")
# %%
# try title_info parsing
@@ -202,10 +201,10 @@ stmt = sql.select(
db.ext_bedpbed.c.BEDP_TITELNR,
db.ext_bedpbed.c.BEDP_MAN,
db.ext_bedpbed.c.BEDP_MENGE_BEDARF_VM,
sql.case(
(db.ext_bedpbed.c.BEDP_MENGE_BEDARF_VM <= -1, sql.null()),
else_=db.ext_bedpbed.c.BEDP_MENGE_BEDARF_VM,
).label("BEDP_MENGE_BEDARF_VM"),
# sql.case(
# (db.ext_bedpbed.c.BEDP_MENGE_BEDARF_VM <= -1, sql.null()),
# else_=db.ext_bedpbed.c.BEDP_MENGE_BEDARF_VM,
# ).label("BEDP_MENGE_BEDARF_VM"),
db.ext_titel_info.c.MELDENUMMER,
db.ext_titel_info.c.VERLAGSNR,
db.ext_titel_info.c.MENGE_VORMERKER,
@@ -227,11 +226,17 @@ print(f"Estimated size in memory: {df.estimated_size(unit='mb')} MB")
# %%
df.head()
# %%
temp = df.with_columns(
pl.col.BEDP_MENGE_BEDARF_VM.fill_null(0),
)
temp.filter(pl.col.BEDP_MENGE_BEDARF_VM < 0)
# %%
# // NO LIVE DATA NEEDED
# SAVING/LOADING
p_save = Path.cwd() / "raw_data_from_sql_query_20260115-altered_query.arrow"
# p_save = Path.cwd() / "raw_data_from_sql_query_20251203-2.arrow"
# p_save = Path.cwd() / "raw_data_from_sql_query_20260115-altered_query.arrow"
p_save = Path.cwd() / "raw_data_from_sql_query_20260116-1.arrow"
# df.write_ipc(p_save)
df = pl.read_ipc(p_save)
# %%
@@ -373,7 +378,6 @@ df_diff_VM_bedp_tinfo = df.filter(pl.col("BEDP_MENGE_BEDARF_VM") > pl.col("MENGE
p_save_diff_VM_bedp_tinfo = (
Path.cwd() / "diff_BEDP-MENGE-BEDARF-VM_TINF-MENGE-VORMERKER_20251211-1.xlsx"
)
from polars.datatypes import classes as pl_dtypes
df_diff_VM_bedp_tinfo.to_pandas().to_excel(p_save_diff_VM_bedp_tinfo, index=False)
# why are there entries where "BEDP_MENGE_BEDARF_VM" > "MENGE_VORMERKER"?
@@ -396,8 +400,12 @@ print(len(df.filter(pl.col("MELDENUMMER") == 18)))
# df.filter(pl.col("MELDENUMMER") == 18).filter((pl.col("BEDP_MENGE_BEDARF_VM").is_not_null()) & (pl.col("BEDP_MENGE_BEDARF_VM") > 0))
# %%
# VM_CRITERION = "MENGE_VORMERKER"
VM_CRITERION = "BEDP_MENGE_BEDARF_VM"
MANDANT_CRITERION = "BEDP_MAN"
VM_CRITERION: typing.Final[str] = "BEDP_MENGE_BEDARF_VM"
MANDANT_CRITERION: typing.Final[str] = "BEDP_MAN"
ORDER_QTY_CRIT: typing.Final[str] = "BEDP_MENGE_BEDARF_VM"
RESULT_COLUMN_ORDER: typing.Final[tuple[str, ...]] = tuple(
db.EXT_DOPT_ERGEBNIS.columns.keys()
)
def get_starting_date(
@@ -435,10 +443,55 @@ def get_raw_data() -> pl.DataFrame:
)
# def get_empty_pipeline_result(
# data: pl.DataFrame,
# ) -> PipelineResult:
# return PipelineResult(data)
def save_tmp_data(df: pl.DataFrame) -> None:
with engine.begin() as conn:
conn.execute(sql.delete(db.tmp_data))
with engine.begin() as conn:
conn.execute(sql.insert(db.tmp_data), df.to_dicts())
def get_tmp_data() -> pl.DataFrame:
return pl.read_database(
sql.select(db.tmp_data),
engine,
schema_overrides=db.tmp_data_schema_map,
)
def get_result_data() -> pl.DataFrame:
return pl.read_database(
sql.select(db.EXT_DOPT_ERGEBNIS),
engine,
schema_overrides=db.results_schema_map,
)
def save_result_data(results: pl.DataFrame) -> None:
with engine.begin() as conn:
conn.execute(sql.insert(db.EXT_DOPT_ERGEBNIS), results.to_dicts())
def clear_result_data() -> None:
with engine.begin() as conn:
conn.execute(sql.delete(db.EXT_DOPT_ERGEBNIS))
def save_result_data_native(results: pl.DataFrame) -> None:
results = results.with_columns(
[
pl.when(pl.col(c)).then(pl.lit("Y")).otherwise(pl.lit("N")).alias(c)
for c in results.select(cs.boolean()).columns
]
)
stmt = """
INSERT INTO "EXT_DOPT_ERGEBNIS" ("BEDARF_NR", "BEDARF_SEQUENZ", "VORLAGE", "WF_ID", "BEST_MENGE", "FREIGABE_AUTO")
VALUES (:1, :2, :3, :4, :5, :6)
"""
with engine.begin() as conn:
raw_conn = conn.connection.connection
with raw_conn.cursor() as cursor:
cursor.executemany(stmt, tmp.to_pandas(use_pyarrow_extension_array=True))
def _apply_several_filters(
@@ -469,7 +522,7 @@ class PipelineResult:
) -> None:
self._open = data
schema = db.results_schema_map.copy()
del schema["id"]
del schema["ID"]
self._results = pl.DataFrame(schema=schema)
schema = {}
@@ -511,7 +564,12 @@ class PipelineResult:
self,
data: pl.DataFrame,
) -> None:
self._results = pl.concat([self._results, data])
print(self._results)
res = pl.concat([self._results, data])
# self._results = res.with_columns(
# (pl.arange(0, res.height) + 1).alias("ID").cast(db.results_schema_map["ID"])
# )
self._results = res
def merge_pipeline(
self,
@@ -528,29 +586,28 @@ class PipelineResult:
freigabe_auto: types.Freigabe,
is_out: bool,
) -> None:
# TODO move to other position
ORDER_QTY_CRIT: typing.Final[str] = "BEDP_MENGE_BEDARF_VM"
results = data.rename(db.map_to_result)
results = data.rename(db.map_data_to_result)
# TODO rework because it is not WF-agnostic
order_qty_expr: pl.Expr
if is_out:
order_qty_expr = (
pl.lit(0)
.alias("ORDER_QTY_CRIT")
.alias("best_menge")
.cast(db.results_schema_map["best_menge"])
.alias("BEST_MENGE")
.cast(db.results_schema_map["BEST_MENGE"])
)
else:
order_qty_expr = pl.col(ORDER_QTY_CRIT).alias("best_menge")
results = results.with_columns(
[
pl.lit(vorlage).alias("vorlage").cast(db.results_schema_map["vorlage"]),
pl.lit(wf_id).alias("wf_id").cast(db.results_schema_map["wf_id"]),
# pl.lit(0).alias("ID").cast(db.results_schema_map["ID"]),
pl.lit(vorlage).alias("VORLAGE").cast(db.results_schema_map["VORLAGE"]),
pl.lit(wf_id).alias("WF_ID").cast(db.results_schema_map["WF_ID"]),
order_qty_expr,
pl.lit(freigabe_auto.value)
.alias("freigabe_auto")
.cast(db.results_schema_map["freigabe_auto"]),
.alias("FREIGABE_AUTO")
.cast(db.results_schema_map["FREIGABE_AUTO"]),
]
)
results = results.drop(
@@ -564,6 +621,9 @@ class PipelineResult:
"MANDFUEHR",
]
)
# results = results.select(RESULT_COLUMN_ORDER)
print(results)
print("####################")
self._subtract_data(data)
self._add_results(results)
@@ -854,7 +914,7 @@ def wf100_petersen_wdb_sub1(
# %%
# SAVING/LOADING
p_save = Path.cwd() / "raw_data_from_sql_query_20260114-1.arrow"
p_save = Path.cwd() / "raw_data_from_sql_query_20260116-1.arrow"
df = pl.read_ipc(p_save)
print(f"Number of entries: {len(df)}")
@@ -911,6 +971,61 @@ print(f"Number of entries open data: {len(pipe_res.open):>10}")
# %%
pipe_res.results
# %%
# // test result writing
res = pipe_res.results.clone()
res.height
# %%
# %%
# %%
save_result_data(res)
get_result_data()
# %%
clear_result_data()
# %%
get_result_data()
# %%
stmt = sql.text("TRUNCATE TABLE EXT_DOPT_ERGEBNIS")
with engine.begin() as conn:
conn.exec_driver_sql("TRUNCATE TABLE UMB.EXT_DOPT_ERGEBNIS")
# %%
stmt = sql.insert(db.EXT_DOPT_ERGEBNIS)
print(stmt.compile(engine))
# %%
engine.dispose()
# %%
# %%
t1 = time.perf_counter()
save_result_data(res)
t2 = time.perf_counter()
print(f"Elapsed: {t2 - t1}")
# %%
get_result_data()
# %%
clear_result_data()
get_result_data()
# %%
t1 = time.perf_counter()
raw_input(res)
t2 = time.perf_counter()
print(f"Elapsed: {t2 - t1}")
# %%
# %%
# raw_data.filter(pl.col("BEDARFNR") == 166982).filter(pl.col("BEDP_SEQUENZ") == 1)
# %%
# pipe_res.open.filter(pl.col("BEDP_MENGE_BEDARF_VM") > pl.col("MENGE_VORMERKER"))
@@ -946,9 +1061,6 @@ raw_data.filter(pl.col("BEDARFNR") == 922160).filter(pl.col("BEDP_SEQUENZ") == 3
# %%
raw_data.head()
# %%
filt_out
# %%
# ---------------------------------------------------------------------------- #

2
data_analysis/queries.sql
View File

@@ -259,4 +259,4 @@ ORDER BY view_name;
-- DESC all_views;
DESC EXT_DOPT_ERGEBNIS;
SELECT * FROM v$version;d-opt
SELECT * FROM v$version;