From 8879a2e1751c5a6f4fab01193f3a07cd95906ade Mon Sep 17 00:00:00 2001 From: foefl Date: Thu, 2 Jul 2026 11:11:26 +0200 Subject: [PATCH] add missing data --- prototypes/01-2_dataset_transform_whole.py | 34 ++++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/prototypes/01-2_dataset_transform_whole.py b/prototypes/01-2_dataset_transform_whole.py index 432c0ef..c415080 100644 --- a/prototypes/01-2_dataset_transform_whole.py +++ b/prototypes/01-2_dataset_transform_whole.py @@ -91,31 +91,31 @@ schema_read = { "ps": pl.UInt32, "ts": pl.String, "type_num": pl.UInt8, - "id": pl.UInt64, + "id_process": pl.UInt64, } schema = schema_read.copy() schema.update( { "ts": pl.Datetime, - "id": pl.UInt64, + "id_process": pl.UInt64, "ts_delta_step": pl.Duration, "ts_delta_cum": pl.Duration, } ) print(schema) -df = pl.DataFrame(schema=schema).with_columns(pl.col("ts").dt.replace_time_zone("UTC")) +df = pl.LazyFrame(schema=schema).with_columns(pl.col("ts").dt.replace_time_zone("UTC")) count = 0 for idx, file in enumerate(p_data_base.glob("**/*.json"), start=1): with open(file, "r") as f: data = json.load(f) type_num = data["initial"]["dsc_TypeNumber"]["value"] - df_file = pl.DataFrame(data["rows"], schema_overrides=schema_read) + df_file = pl.LazyFrame(data["rows"], schema_overrides=schema_read) df_file = df_file.with_columns( pl.col("ts").str.to_datetime(time_zone="UTC"), pl.lit(type_num).alias("type_num").cast(pl.UInt8), - pl.lit(idx).alias("id").cast(pl.UInt64), - ) + pl.lit(idx).alias("id_process").cast(pl.UInt64), + ).sort("ts", descending=False) df_file = df_file.with_columns( (pl.col.ts - pl.col.ts.shift(1)) .alias("ts_delta_step") @@ -130,17 +130,11 @@ for idx, file in enumerate(p_data_base.glob("**/*.json"), start=1): if BREAK_EARLY and idx == 3: break -# df = df.with_columns(pl.col("ts").str.to_datetime(time_zone="UTC")) -# df = df.select( -# ["id", "type_num", "ts", "ts_delta_step", "ts_delta_cum", "ps", "DU1260", "V1560"] -# ) -df.head() +df = df.with_row_index(offset=1) # %% -t = df.columns[0] - sensor_feats: list[str] = [] pattern = re.compile(r"^([A-Z]+[0-9]+)") -for feat in df.columns: +for feat in df.collect_schema().names(): matches = pattern.match(feat) if matches is None: continue @@ -149,9 +143,17 @@ for feat in df.columns: sensor_feats.sort() -feats_sorted = ["id", "type_num", "ts", "ts_delta_step", "ts_delta_cum", "ps"] + sensor_feats +feats_sorted = [ + "index", + "id_process", + "type_num", + "ts", + "ts_delta_step", + "ts_delta_cum", + "ps", +] + sensor_feats -df = df.select(feats_sorted) +df = df.select(feats_sorted).collect() df.head() # %% print(f"Files processed: {count}")