diff --git a/notebooks/lang_main_config.toml b/notebooks/lang_main_config.toml
index e83d210..ad3247a 100644
--- a/notebooks/lang_main_config.toml
+++ b/notebooks/lang_main_config.toml
@@ -2,12 +2,10 @@
[paths]
inputs = './inputs/'
-results = '../scripts/results/test_20240619/'
+# results = './results/dummy_N_1000/'
+# dataset = '../data/Dummy_Dataset_N_1000.csv'
+results = './results/'
dataset = '../data/02_202307/Export4.csv'
-#results = './results/Export7/'
-#dataset = './01_03_Rohdaten_202403/Export7_59499_Zeilen.csv'
-#results = './results/Export7_trunc/'
-#dataset = './01_03_Rohdaten_202403/Export7_trunc.csv'
# only debugging features, production-ready pipelines should always
# be fully executed
@@ -19,21 +17,18 @@ graph_rescaling_skip = false
graph_static_rendering_skip = false
time_analysis_skip = true
-#[export_filenames]
-#filename_cossim_filter_candidates = 'CosSim-FilterCandidates'
-
[preprocess]
-filename_cossim_filter_candidates = 'CosSim-FilterCandidates'
date_cols = [
- "VorgangsDatum",
- "ErledigungsDatum",
- "Arbeitsbeginn",
+ "VorgangsDatum",
+ "ErledigungsDatum",
+ "Arbeitsbeginn",
"ErstellungsDatum",
]
threshold_amount_characters = 5
threshold_similarity = 0.8
[graph_postprocessing]
+threshold_edge_number = 300
threshold_edge_weight = 150
[time_analysis.uniqueness]
@@ -41,6 +36,10 @@ threshold_unique_texts = 4
criterion_feature = 'HObjektText'
feature_name_obj_id = 'ObjektID'
+[time_analysis.preparation]
+name_delta_feat_to_repair = 'Zeitspanne bis zur Behebung [Tage]'
+name_delta_feat_to_next_failure = 'Zeitspanne bis zum nächsten Ereignis [Tage]'
+
[time_analysis.model_input]
# input_features = [
# 'VorgangsTypName',
diff --git a/notebooks/misc.ipynb b/notebooks/misc.ipynb
index 93f47dc..1a3e8dd 100644
--- a/notebooks/misc.ipynb
+++ b/notebooks/misc.ipynb
@@ -104,7 +104,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 9,
"id": "af118d77-d87a-4687-be5b-e810a24c403e",
"metadata": {
"scrolled": true
@@ -122,7 +122,21 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "2024-08-07 07:36:54 +0000 | io:INFO | Loaded TOML config file successfully.\n"
+ "2024-09-12 12:20:04 +0000 | lang_main:io:INFO | Loaded TOML config file successfully.\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "A:\\Arbeitsaufgaben\\lang-main\\.venv\\Lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
+ " warnings.warn(\n",
+ "A:\\Arbeitsaufgaben\\lang-main\\.venv\\Lib\\site-packages\\transformers\\utils\\generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
+ " _torch_pytree._register_pytree_node(\n",
+ "A:\\Arbeitsaufgaben\\lang-main\\.venv\\Lib\\site-packages\\transformers\\utils\\generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
+ " _torch_pytree._register_pytree_node(\n",
+ "A:\\Arbeitsaufgaben\\lang-main\\.venv\\Lib\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
+ " warnings.warn(\n"
]
}
],
@@ -132,12 +146,16 @@
"from lang_main import model_loader\n",
"from lang_main.types import LanguageModels\n",
"from lang_main.constants import MODEL_LOADER_MAP\n",
+ "import lang_main.pipelines.predefined as pipes\n",
+ "import lang_main.render.cytoscape as cyto\n",
+ "from lang_main.analysis import graphs\n",
"\n",
"from pathlib import Path\n",
"import pickle\n",
"import base64\n",
"import os\n",
"from logging import NullHandler\n",
+ "import time\n",
"\n",
"import numpy as np\n",
"import networkx as nx\n",
@@ -154,45 +172,868 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 25,
"id": "59cbcf38-6fe1-403b-9c10-f107e28185f0",
"metadata": {},
"outputs": [],
+ "source": [
+ "results = r'A:\\Arbeitsaufgaben\\lang-main\\scripts\\results\\test_20240807'\n",
+ "p_res = Path(results)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "id": "a34650ae-286f-4539-97d1-98dbecd0dafc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "prefilter = 'TK-GRAPH_POSTPROCESSING.pkl'\n",
+ "postfilter = 'TK-GRAPH_ANALYSIS.pkl'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "id": "313bd956-fc32-4425-9c47-daad3622528b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "p_pre = p_res / prefilter\n",
+ "p_post = p_res / postfilter"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "id": "b745044b-6152-4c69-8fab-016731d57532",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "assert p_pre.exists()\n",
+ "assert p_post.exists()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "id": "7bd52a09-0114-4514-905f-d01d357c6a60",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2024-09-12 12:33:10 +0000 | lang_main:io:INFO | Loaded file successfully.\n",
+ "2024-09-12 12:33:10 +0000 | lang_main:io:INFO | Loaded file successfully.\n"
+ ]
+ }
+ ],
+ "source": [
+ "res_pre = io.load_pickle(p_pre)\n",
+ "res_post = io.load_pickle(p_post)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "id": "a2a9be43-9ca6-4410-93bf-1e528a8ce8b6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tkg_pre = res_pre[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "id": "7255a466-b421-465b-92a6-27a5e135e083",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tkg_post = res_post[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "id": "dc259aab-ca18-4ab0-bcd0-0d9779c17f2c",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "TokenGraph(name: TokenGraph, number of nodes: 7024, number of edges: 25600)\n",
+ "TokenGraph(name: TokenGraph, number of nodes: 227, number of edges: 330)\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(tkg_pre)\n",
+ "print(tkg_post)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "id": "2faca328-80bb-4e49-be79-6c480c6ef0c6",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "TokenGraph(name: TokenGraph, number of nodes: 351, number of edges: 603)"
+ ]
+ },
+ "execution_count": 21,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "BOUND = 30\n",
+ "tkg_filt = graphs.filter_graph_by_edge_weight(tkg_pre, bound_lower=BOUND, bound_upper=None)\n",
+ "tkg_filt = graphs.filter_graph_by_node_degree(tkg_filt, bound_lower=1, bound_upper=None)\n",
+ "tkg_filt"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "id": "2fc9c0d6-06d6-4716-821c-1e213d72eb38",
+ "metadata": {
+ "jupyter": {
+ "source_hidden": true
+ },
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "You are connected to Cytoscape!\n",
+ "Applying default style...\n",
+ "Applying preferred layout\n",
+ "Exec time whole was 9.541292 sec\n",
+ "Exec time import was 1.265542 sec\n",
+ "Exec time layout was 3.962870 sec\n",
+ "Exec time style was 4.312881 sec\n"
+ ]
+ }
+ ],
+ "source": [
+ "t1 = time.perf_counter_ns()\n",
+ "cyto.import_to_cytoscape(tkg_filt)\n",
+ "t2 = time.perf_counter_ns()\n",
+ "cyto.layout_network()\n",
+ "t3 = time.perf_counter_ns()\n",
+ "cyto.apply_style_to_network()\n",
+ "t4 = time.perf_counter_ns()\n",
+ "exec_time_whole = (t4 - t1) * 1e-9\n",
+ "exec_time_import = (t2 - t1) * 1e-9\n",
+ "exec_time_layout = (t3 - t2) * 1e-9\n",
+ "exec_time_style = (t4 - t3) * 1e-9\n",
+ "print(f'Exec time whole was {exec_time_whole:.6f} sec')\n",
+ "print(f'Exec time import was {exec_time_import:.6f} sec')\n",
+ "print(f'Exec time layout was {exec_time_layout:.6f} sec')\n",
+ "print(f'Exec time style was {exec_time_style:.6f} sec')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "id": "a68a0fb9-8cfe-4b27-9eb8-6c4895ae3f40",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "render_pipe = pipes.build_tk_graph_render_pipe(with_subgraphs=True, export_folder=Path.cwd())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 23,
+ "id": "f7fc4e3b-7470-4050-92d9-57cbc0f27f69",
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2024-09-12 12:21:28 +0000 | lang_main:base:INFO | Starting pipeline >>Graph_Static-Rendering<<...\n",
+ "You are connected to Cytoscape!\n",
+ "Applying default style...\n",
+ "Applying preferred layout\n",
+ "No nodes selected.\n",
+ "No nodes selected.\n",
+ "No nodes selected.\n",
+ "No nodes selected.\n",
+ "2024-09-12 12:21:42 +0000 | lang_main:io:INFO | Saved file successfully under A:\\Arbeitsaufgaben\\lang-main\\notebooks\\results\\Pipe-Graph_Static-Rendering_Step-6_build_subnetworks.pkl\n",
+ "2024-09-12 12:21:42 +0000 | lang_main:base:INFO | Processing pipeline >>Graph_Static-Rendering<< successfully ended after 6 steps.\n",
+ "Exec time whole was 13.943395 sec\n"
+ ]
+ }
+ ],
+ "source": [
+ "t1 = time.perf_counter_ns()\n",
+ "render_pipe.run((tkg_filt,))\n",
+ "t2 = time.perf_counter_ns()\n",
+ "exec_time_whole = (t2 - t1) * 1e-9\n",
+ "print(f'Exec time whole was {exec_time_whole:.6f} sec')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "46c18cbc-b7b7-427e-8001-f1db5a9679d3",
+ "metadata": {},
+ "outputs": [],
"source": []
},
{
"cell_type": "code",
- "execution_count": 1,
- "id": "a33cb410-f774-4cc9-b972-bf05df36d3d7",
+ "execution_count": 2,
+ "id": "f042d3d9-ea84-4b7e-baba-1d9f6e261277",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import seaborn as sns\n",
+ "sns.set()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "f2e8f841-d577-4b98-8f72-dee9c908264b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "record1 = [\n",
+ " 168, 203, 262, 351, 465, 726, 1122,\n",
+ " 1368, 1709, 2928\n",
+ "]\n",
+ "record2 = [\n",
+ " 225, 293, 405, 603, 857,\n",
+ " 1317, 2191, 2802, 3765, 7149\n",
+ "]\n",
+ "record3 = [\n",
+ " 12.94, 12.96, 13.55, 14.45, 15.39,\n",
+ " 17.64, 22.90, 26.95, 33.82, 61.75\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "ab400322-4d3e-4502-88fe-d3c68a97d8a4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data = {\n",
+ " 'nodes_num': record1,\n",
+ " 'edges_num': record2,\n",
+ " 'time_sec': record3,\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "459f5450-167c-430c-bded-e47c33073584",
"metadata": {},
"outputs": [
{
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "A:\\Arbeitsaufgaben\\lang-main\\.venv\\Lib\\site-packages\\transformers\\utils\\generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n",
- " _torch_pytree._register_pytree_node(\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "spacy imports\n",
- "end\n",
- "Nothing\n",
- "2024-08-07 07:51:22 +0000 | io:INFO | Loaded TOML config file successfully.\n"
- ]
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " nodes_num | \n",
+ " edges_num | \n",
+ " time_sec | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 168 | \n",
+ " 225 | \n",
+ " 12.94 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 203 | \n",
+ " 293 | \n",
+ " 12.96 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 262 | \n",
+ " 405 | \n",
+ " 13.55 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 351 | \n",
+ " 603 | \n",
+ " 14.45 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 465 | \n",
+ " 857 | \n",
+ " 15.39 | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " 726 | \n",
+ " 1317 | \n",
+ " 17.64 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " 1122 | \n",
+ " 2191 | \n",
+ " 22.90 | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " 1368 | \n",
+ " 2802 | \n",
+ " 26.95 | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " 1709 | \n",
+ " 3765 | \n",
+ " 33.82 | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " 2928 | \n",
+ " 7149 | \n",
+ " 61.75 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "