more robust graph filtering

scripts/analyse_dataset.py

@@ -42,8 +42,8 @@ from lang_main.types import (
 )
 
 # ** profiling
-ONLY_PROFILING_REPORT: Final[bool] = True
-USE_PROFILING: Final[bool] = True
+USE_PROFILING: Final[bool] = False
+ONLY_PROFILING_REPORT: Final[bool] = False
 PROFILE_REPORT_NAME: Final[str] = 'prof_report.profdata'
 
 # ** build pipelines

scripts/lang_main_config.toml

@@ -6,43 +6,40 @@ inputs = './inputs/'
 # dataset = '../data/Dummy_Dataset_N_1000.csv'
 results = './results/test_20240807/'
 dataset = '../data/02_202307/Export4.csv'
-#results = './results/Export7/'
-#dataset = './01_03_Rohdaten_202403/Export7_59499_Zeilen.csv'
-#results = './results/Export7_trunc/'
-#dataset = './01_03_Rohdaten_202403/Export7_trunc.csv'
 
 # only debugging features, production-ready pipelines should always
 # be fully executed
 [control]
-preprocessing_skip = false
-token_analysis_skip = false
+preprocessing_skip = true
+token_analysis_skip = true
 graph_postprocessing_skip = false
 graph_rescaling_skip = false
 graph_static_rendering_skip = false
-time_analysis_skip = false
-
-#[export_filenames]
-#filename_cossim_filter_candidates = 'CosSim-FilterCandidates'
+time_analysis_skip = true
 
 [preprocess]
-filename_cossim_filter_candidates = 'CosSim-FilterCandidates'
 date_cols = [
-    "VorgangsDatum", 
-    "ErledigungsDatum", 
-    "Arbeitsbeginn", 
+    "VorgangsDatum",
+    "ErledigungsDatum",
+    "Arbeitsbeginn",
     "ErstellungsDatum",
 ]
 threshold_amount_characters = 5
 threshold_similarity = 0.8
 
 [graph_postprocessing]
-threshold_edge_weight = 150
+threshold_edge_number = 330
+# threshold_edge_weight = 150
 
 [time_analysis.uniqueness]
 threshold_unique_texts = 4
 criterion_feature = 'HObjektText'
 feature_name_obj_id = 'ObjektID'
 
+[time_analysis.preparation]
+name_delta_feat_to_repair = 'Zeitspanne bis zur Behebung [Tage]'
+name_delta_feat_to_next_failure = 'Zeitspanne bis zum nächsten Ereignis [Tage]'
+
 [time_analysis.model_input]
 # input_features = [
 #     'VorgangsTypName',