Kog-A-Pro/ag-duerr
1
0
Fork 0
generated from dopt-python/py311
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add missing data

Browse Source
This commit is contained in:
Florian Förster
2026-07-02 11:11:26 +02:00
parent a542ae858d
commit 8879a2e175
1 changed files with 18 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
prototypes/01-2_dataset_transform_whole.py
+18 -16
View File
@@ -91,31 +91,31 @@ schema_read = {
"ps": pl.UInt32, "ps": pl.UInt32,
"ts": pl.String, "ts": pl.String,
"type_num": pl.UInt8, "type_num": pl.UInt8,
"id": pl.UInt64, "id_process": pl.UInt64,
} }
schema = schema_read.copy() schema = schema_read.copy()
schema.update( schema.update(
{ {
"ts": pl.Datetime, "ts": pl.Datetime,
"id": pl.UInt64, "id_process": pl.UInt64,
"ts_delta_step": pl.Duration, "ts_delta_step": pl.Duration,
"ts_delta_cum": pl.Duration, "ts_delta_cum": pl.Duration,
} }
) )
print(schema) print(schema)
df = pl.DataFrame(schema=schema).with_columns(pl.col("ts").dt.replace_time_zone("UTC")) df = pl.LazyFrame(schema=schema).with_columns(pl.col("ts").dt.replace_time_zone("UTC"))
count = 0 count = 0
for idx, file in enumerate(p_data_base.glob("**/*.json"), start=1): for idx, file in enumerate(p_data_base.glob("**/*.json"), start=1):
with open(file, "r") as f: with open(file, "r") as f:
data = json.load(f) data = json.load(f)
type_num = data["initial"]["dsc_TypeNumber"]["value"] type_num = data["initial"]["dsc_TypeNumber"]["value"]
df_file = pl.DataFrame(data["rows"], schema_overrides=schema_read) df_file = pl.LazyFrame(data["rows"], schema_overrides=schema_read)
df_file = df_file.with_columns( df_file = df_file.with_columns(
pl.col("ts").str.to_datetime(time_zone="UTC"), pl.col("ts").str.to_datetime(time_zone="UTC"),
pl.lit(type_num).alias("type_num").cast(pl.UInt8), pl.lit(type_num).alias("type_num").cast(pl.UInt8),
pl.lit(idx).alias("id").cast(pl.UInt64), pl.lit(idx).alias("id_process").cast(pl.UInt64),
) ).sort("ts", descending=False)
df_file = df_file.with_columns( df_file = df_file.with_columns(
(pl.col.ts - pl.col.ts.shift(1)) (pl.col.ts - pl.col.ts.shift(1))
.alias("ts_delta_step") .alias("ts_delta_step")
@@ -130,17 +130,11 @@ for idx, file in enumerate(p_data_base.glob("**/*.json"), start=1):
if BREAK_EARLY and idx == 3: if BREAK_EARLY and idx == 3:
break break
# df = df.with_columns(pl.col("ts").str.to_datetime(time_zone="UTC")) df = df.with_row_index(offset=1)
# df = df.select(
# ["id", "type_num", "ts", "ts_delta_step", "ts_delta_cum", "ps", "DU1260", "V1560"]
# )
df.head()
# %% # %%
t = df.columns[0]
sensor_feats: list[str] = [] sensor_feats: list[str] = []
pattern = re.compile(r"^([A-Z]+[0-9]+)") pattern = re.compile(r"^([A-Z]+[0-9]+)")
for feat in df.columns: for feat in df.collect_schema().names():
matches = pattern.match(feat) matches = pattern.match(feat)
if matches is None: if matches is None:
continue continue
@@ -149,9 +143,17 @@ for feat in df.columns:
sensor_feats.sort() sensor_feats.sort()
feats_sorted = ["id", "type_num", "ts", "ts_delta_step", "ts_delta_cum", "ps"] + sensor_feats feats_sorted = [
"index",
"id_process",
"type_num",
"ts",
"ts_delta_step",
"ts_delta_cum",
"ps",
] + sensor_feats
df = df.select(feats_sorted) df = df.select(feats_sorted).collect()
df.head() df.head()
# %% # %%
print(f"Files processed: {count}") print(f"Files processed: {count}")