add missing data

prototypes/01-2_dataset_transform_whole.py

@@ -91,31 +91,31 @@ schema_read = {
     "ps": pl.UInt32,
     "ts": pl.String,
     "type_num": pl.UInt8,
-    "id": pl.UInt64,
+    "id_process": pl.UInt64,
 }
 schema = schema_read.copy()
 schema.update(
     {
         "ts": pl.Datetime,
-        "id": pl.UInt64,
+        "id_process": pl.UInt64,
         "ts_delta_step": pl.Duration,
         "ts_delta_cum": pl.Duration,
     }
 )
 print(schema)
-df = pl.DataFrame(schema=schema).with_columns(pl.col("ts").dt.replace_time_zone("UTC"))
+df = pl.LazyFrame(schema=schema).with_columns(pl.col("ts").dt.replace_time_zone("UTC"))
 count = 0
 for idx, file in enumerate(p_data_base.glob("**/*.json"), start=1):
     with open(file, "r") as f:
         data = json.load(f)
 
     type_num = data["initial"]["dsc_TypeNumber"]["value"]
-    df_file = pl.DataFrame(data["rows"], schema_overrides=schema_read)
+    df_file = pl.LazyFrame(data["rows"], schema_overrides=schema_read)
     df_file = df_file.with_columns(
         pl.col("ts").str.to_datetime(time_zone="UTC"),
         pl.lit(type_num).alias("type_num").cast(pl.UInt8),
-        pl.lit(idx).alias("id").cast(pl.UInt64),
-    )
+        pl.lit(idx).alias("id_process").cast(pl.UInt64),
+    ).sort("ts", descending=False)
     df_file = df_file.with_columns(
         (pl.col.ts - pl.col.ts.shift(1))
         .alias("ts_delta_step")
@@ -130,17 +130,11 @@ for idx, file in enumerate(p_data_base.glob("**/*.json"), start=1):
     if BREAK_EARLY and idx == 3:
         break
 
-# df = df.with_columns(pl.col("ts").str.to_datetime(time_zone="UTC"))
-# df = df.select(
-#     ["id", "type_num", "ts", "ts_delta_step", "ts_delta_cum", "ps", "DU1260", "V1560"]
-# )
-df.head()
+df = df.with_row_index(offset=1)
 # %%
-t = df.columns[0]
-
 sensor_feats: list[str] = []
 pattern = re.compile(r"^([A-Z]+[0-9]+)")
-for feat in df.columns:
+for feat in df.collect_schema().names():
     matches = pattern.match(feat)
     if matches is None:
         continue
@@ -149,9 +143,17 @@ for feat in df.columns:
 
 sensor_feats.sort()
 
-feats_sorted = ["id", "type_num", "ts", "ts_delta_step", "ts_delta_cum", "ps"] + sensor_feats
+feats_sorted = [
+    "index",
+    "id_process",
+    "type_num",
+    "ts",
+    "ts_delta_step",
+    "ts_delta_cum",
+    "ps",
+] + sensor_feats
 
-df = df.select(feats_sorted)
+df = df.select(feats_sorted).collect()
 df.head()
 # %%
 print(f"Files processed: {count}")