using dash-cytoscape

scripts/analyse_dataset.py

@@ -28,6 +28,8 @@ from lang_main.pipelines.predefined import (
 )
 from lang_main.types import (
     ObjectID,
+    PandasIndex,
+    SpacyDoc,
     TimelineCandidates,
 )
 from pandas import DataFrame, Series
@@ -37,7 +39,7 @@ from pandas import DataFrame, Series
 def run_preprocessing() -> DataFrame:
     create_saving_folder(
         saving_path_folder=SAVE_PATH_FOLDER,
-        overwrite_existing=True,
+        overwrite_existing=False,
     )
     # run pipelines
     ret = typing.cast(
@@ -56,15 +58,16 @@ def run_preprocessing() -> DataFrame:
 
 def run_token_analysis(
     preprocessed_data: DataFrame,
-) -> TokenGraph:
+) -> tuple[TokenGraph, dict[PandasIndex, SpacyDoc]]:
     # build token graph
-    (tk_graph,) = typing.cast(
-        tuple[TokenGraph], pipe_token_analysis.run(starting_values=(preprocessed_data,))
+    (tk_graph, docs_mapping) = typing.cast(
+        tuple[TokenGraph, dict[PandasIndex, SpacyDoc]],
+        pipe_token_analysis.run(starting_values=(preprocessed_data,)),
     )
     tk_graph.save_graph(SAVE_PATH_FOLDER, directed=False)
     tk_graph.to_pickle(SAVE_PATH_FOLDER, filename=f'{pipe_token_analysis.name}-TokenGraph')
 
-    return tk_graph
+    return tk_graph, docs_mapping
 
 
 def run_graph_postprocessing(
@@ -127,9 +130,9 @@ def main() -> None:
                 'Preprocessing step skipped. Token analysis cannot be performed.'
             )
         preprocessed_data_trunc = typing.cast(
-            DataFrame, preprocessed_data[['entry', 'num_occur']].copy()
+            DataFrame, preprocessed_data[['batched_idxs', 'entry', 'num_occur']].copy()
         )  # type: ignore
-        tk_graph = run_token_analysis(preprocessed_data_trunc)
+        tk_graph, docs_mapping = run_token_analysis(preprocessed_data_trunc)
     elif not SKIP_TOKEN_ANALYSIS:
         # !! hardcoded result filenames
         # whole graph

scripts/dashboard/app.py

@@ -0,0 +1,190 @@
+import time
+import webbrowser
+from pathlib import Path
+from threading import Thread
+from typing import cast
+
+import pandas as pd
+import plotly.express as px
+from dash import (
+    Dash,
+    Input,
+    Output,
+    State,
+    callback,
+    dash_table,
+    dcc,
+    html,
+)
+from lang_main.io import load_pickle
+from lang_main.types import ObjectID, TimelineCandidates
+from pandas import DataFrame
+
+# df = pd.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/gapminder_unfiltered.csv')
+
+# ** data
+p_df = Path(r'./Pipe-TargetFeature_Step-3_remove_NA.pkl').resolve()
+p_tl = Path(r'/Pipe-Timeline_Analysis_Step-4_get_timeline_candidates.pkl').resolve()
+ret = cast(DataFrame, load_pickle(p_df))
+data = ret[0]
+ret = cast(tuple[TimelineCandidates, dict[ObjectID, str]], load_pickle(p_tl))
+cands = ret[0]
+texts = ret[1]
+
+# p_df = Path(r'.\test-notebooks\dashboard\data.pkl')
+# p_cands = Path(r'.\test-notebooks\dashboard\map_candidates.pkl')
+# p_map = Path(r'.\test-notebooks\dashboard\map_texts.pkl')
+# data = cast(DataFrame, load_pickle(p_df))
+# cands = cast(TimelineCandidates, load_pickle(p_cands))
+# texts = cast(dict[ObjectID, str], load_pickle(p_map))
+
+table_feats = [
+    'ErstellungsDatum',
+    'ErledigungsDatum',
+    'VorgangsTypName',
+    'VorgangsBeschreibung',
+]
+table_feats_dates = [
+    'ErstellungsDatum',
+    'ErledigungsDatum',
+]
+
+# ** graph config
+markers = {
+    'size': 12,
+    'color': 'yellow',
+    'line': {
+        'width': 2,
+        'color': 'red',
+    },
+}
+hover_data = {
+    'ErstellungsDatum': '|%d.%m.%Y',
+    'VorgangsBeschreibung': True,
+}
+
+
+app = Dash(prevent_initial_callbacks=True)
+
+app.layout = [
+    html.H1(children='Demo Zeitreihenanalyse', style={'textAlign': 'center'}),
+    html.Div(
+        children=[
+            html.H2('Wählen Sie ein Objekt aus (ObjektID):'),
+            dcc.Dropdown(
+                list(cands.keys()),
+                id='dropdown-selection',
+                placeholder='ObjektID auswählen...',
+            ),
+        ]
+    ),
+    html.Div(
+        children=[
+            html.H3(id='object_text'),
+            dcc.Dropdown(id='choice-candidates'),
+            dcc.Graph(id='graph-output'),
+        ]
+    ),
+    html.Div(children=[dash_table.DataTable(id='table-candidates')]),
+]
+
+
+@callback(
+    Output('object_text', 'children'),
+    Input('dropdown-selection', 'value'),
+    prevent_initial_call=True,
+)
+def update_obj_text(obj_id):
+    obj_id = int(obj_id)
+    obj_text = texts[obj_id]
+    headline = f'HObjektText: {obj_text}'
+    return headline
+
+
+@callback(
+    Output('choice-candidates', 'options'),
+    Input('dropdown-selection', 'value'),
+    prevent_initial_call=True,
+)
+def update_choice_candidates(obj_id):
+    obj_id = int(obj_id)
+    cands_obj_id = cands[obj_id]
+    choices = list(range(1, len(cands_obj_id) + 1))
+    return choices
+
+
+@callback(
+    Output('graph-output', 'figure'),
+    Input('choice-candidates', 'value'),
+    State('dropdown-selection', 'value'),
+    prevent_initial_call=True,
+)
+def update_timeline(index, obj_id):
+    obj_id = int(obj_id)
+    # title
+    obj_text = texts[obj_id]
+    title = f'HObjektText: {obj_text}'
+    # cands
+    cands_obj_id = cands[obj_id]
+    cands_choice = cands_obj_id[int(index) - 1]
+    # data
+    df = data.loc[list(cands_choice)].sort_index()  # type: ignore
+    # figure
+    fig = px.line(
+        data_frame=df,
+        x='ErstellungsDatum',
+        y='ObjektID',
+        title=title,
+        hover_data=hover_data,
+    )
+    fig.update_traces(mode='markers+lines', marker=markers, marker_symbol='diamond')
+    fig.update_xaxes(
+        tickformat='%B\n%Y',
+        rangeslider_visible=True,
+    )
+    fig.update_yaxes(type='category')
+    fig.update_layout(hovermode='x unified')
+    return fig
+
+
+@callback(
+    [Output('table-candidates', 'data'), Output('table-candidates', 'columns')],
+    Input('choice-candidates', 'value'),
+    State('dropdown-selection', 'value'),
+    prevent_initial_call=True,
+)
+def update_table_candidates(index, obj_id):
+    obj_id = int(obj_id)
+    # cands
+    cands_obj_id = cands[obj_id]
+    cands_choice = cands_obj_id[int(index) - 1]
+    # data
+    df = data.loc[list(cands_choice)].sort_index()  # type: ignore
+    df = df.filter(items=table_feats, axis=1).sort_values(
+        by='ErstellungsDatum', ascending=True
+    )
+    cols = [{'name': i, 'id': i} for i in df.columns]
+    # convert dates to strings
+    for col in table_feats_dates:
+        df[col] = df[col].dt.strftime(r'%Y-%m-%d')
+
+    table_data = df.to_dict('records')
+    return table_data, cols
+
+
+def _start_webbrowser():
+    host = '127.0.0.1'
+    port = '8050'
+    adress = f'http://{host}:{port}/'
+    time.sleep(2)
+    webbrowser.open_new(adress)
+
+
+def main():
+    webbrowser_thread = Thread(target=_start_webbrowser, daemon=True)
+    webbrowser_thread.start()
+    app.run(debug=True)
+
+
+if __name__ == '__main__':
+    main()

scripts/dashboard/cyto.py

@@ -0,0 +1,203 @@
+import time
+import webbrowser
+from pathlib import Path
+from threading import Thread
+from typing import cast
+
+import dash_cytoscape as cyto
+import lang_main.io
+from dash import Dash, Input, Output, State, dcc, html
+from lang_main.analysis import graphs
+
+target = '../results/test_20240529/Pipe-Token_Analysis_Step-1_build_token_graph.pkl'
+p = Path(target).resolve()
+ret = lang_main.io.load_pickle(p)
+tk_graph = cast(graphs.TokenGraph, ret[0])
+tk_graph_filtered = tk_graph.filter_by_edge_weight(150)
+tk_graph_filtered = tk_graph_filtered.filter_by_node_degree(1)
+cyto_data, weight_data = graphs.convert_graph_to_cytoscape(tk_graph_filtered)
+
+MIN_WEIGHT = weight_data['min']
+MAX_WEIGHT = weight_data['max']
+
+
+cyto.load_extra_layouts()
+app = Dash(__name__)
+
+my_stylesheet = [
+    # Group selectors
+    {
+        'selector': 'node',
+        'style': {
+            'shape': 'circle',
+            'content': 'data(label)',
+            'background-color': '#B10DC9',
+            'border-width': 2,
+            'border-color': 'black',
+            'border-opacity': 1,
+            'opacity': 1,
+            'color': 'black',
+            'text-opacity': 1,
+            'font-size': 12,
+            'z-index': 9999,
+        },
+    },
+    {
+        'selector': 'edge',
+        'style': {
+            'width': 2,
+            'curve-style': 'bezier',
+            'line-color': 'grey',
+            'line-style': 'solid',
+            'line-opacity': 1,
+        },
+    },
+    # Class selectors
+    # {'selector': '.red', 'style': {'background-color': 'red', 'line-color': 'red'}},
+    # {'selector': '.triangle', 'style': {'shape': 'triangle'}},
+]
+
+app.layout = html.Div(
+    [
+        html.Button('Reset', id='bt-reset'),
+        dcc.Dropdown(
+            id='layout_choice_internal',
+            options=[
+                'random',
+                'grid',
+                'circle',
+                'concentric',
+                'breadthfirst',
+                'cose',
+            ],
+            value='cose',
+            clearable=False,
+        ),
+        dcc.Dropdown(
+            id='layout_choice_external',
+            options=[
+                'cose-bilkent',
+                'cola',
+                'euler',
+                'spread',
+                'dagre',
+                'klay',
+            ],
+            clearable=False,
+        ),
+        dcc.RangeSlider(
+            id='weight_slider',
+            min=MIN_WEIGHT,
+            max=MAX_WEIGHT,
+            step=1000,
+        ),
+        cyto.Cytoscape(
+            id='cytoscape-graph',
+            layout={'name': 'cose'},
+            style={'width': '100%', 'height': '600px'},
+            stylesheet=my_stylesheet,
+            elements=cyto_data,
+            zoom=1,
+        ),
+    ]
+)
+
+
+@app.callback(
+    Output('cytoscape-graph', 'layout', allow_duplicate=True),
+    Input('layout_choice_internal', 'value'),
+    prevent_initial_call=True,
+)
+def update_layout_internal(layout_choice):
+    return {'name': layout_choice}
+
+
+@app.callback(
+    Output('cytoscape-graph', 'layout', allow_duplicate=True),
+    Input('layout_choice_external', 'value'),
+    prevent_initial_call=True,
+)
+def update_layout_external(layout_choice):
+    return {'name': layout_choice}
+
+
+@app.callback(
+    Output('cytoscape-graph', 'zoom'),
+    Output('cytoscape-graph', 'elements'),
+    Input('bt-reset', 'n_clicks'),
+    prevent_initial_call=True,
+)
+def reset_layout(n_clicks):
+    return (1, cyto_data)
+
+
+# @app.callback(
+#     Output('cytoscape-graph', 'stylesheet'),
+#     Input('weight_slider', 'value'),
+#     State('cytoscape-graph', 'stylesheet'),
+#     prevent_initial_call=True,
+# )
+# def select_weight(range_chosen, stylesheet):
+#     min_weight, max_weight = range_chosen
+#     new_stylesheet = stylesheet.copy()
+#     new_stylesheet.append(
+#         {
+#             'selector': f'[weight >= {min_weight}]',
+#             'style': {'line-color': 'blue', 'line-style': 'dashed'},
+#         }
+#     )
+#     new_stylesheet.append(
+#         {
+#             'selector': f'[weight <= {max_weight}]',
+#             'style': {'line-color': 'blue', 'line-style': 'dashed'},
+#         }
+#     )
+#     return new_stylesheet
+
+
+# app.layout = html.Div(
+#     [
+#         cyto.Cytoscape(
+#             id='cytoscape-two-nodes',
+#             layout={'name': 'preset'},
+#             style={'width': '100%', 'height': '400px'},
+#             stylesheet=my_stylesheet,
+#             elements=[
+#                 {
+#                     'data': {
+#                         'id': 'one',
+#                         'label': 'Titel 1',
+#                     },
+#                     'position': {'x': 75, 'y': 75},
+#                     'grabbable': False,
+#                     #'locked': True,
+#                     'classes': 'red',
+#                 },
+#                 {
+#                     'data': {'id': 'two', 'label': 'Title 2'},
+#                     'position': {'x': 200, 'y': 200},
+#                     'classes': 'triangle',
+#                 },
+#                 {'data': {'source': 'one', 'target': 'two', 'weight': 2000}},
+#             ],
+#         )
+#     ]
+# )
+
+
+def _start_webbrowser():
+    host = '127.0.0.1'
+    port = '8050'
+    adress = f'http://{host}:{port}/'
+    time.sleep(2)
+    webbrowser.open_new(adress)
+
+
+def main():
+    webbrowser_thread = Thread(target=_start_webbrowser, daemon=True)
+    webbrowser_thread.start()
+    app.run(debug=True)
+
+
+if __name__ == '__main__':
+    main()

scripts/dashboard/cyto_2.py

@@ -0,0 +1,368 @@
+import json
+import os
+
+import dash
+import dash_cytoscape as cyto
+from dash import Input, Output, State, callback, dcc, html
+
+# Load extra layouts
+cyto.load_extra_layouts()
+
+
+# Display utility functions
+def _merge(a, b):
+    return dict(a, **b)
+
+
+def _omit(omitted_keys, d):
+    return {k: v for k, v in d.items() if k not in omitted_keys}
+
+
+# Custom Display Components
+def Card(children, **kwargs):
+    return html.Section(
+        children,
+        style=_merge(
+            {
+                'padding': 20,
+                'margin': 5,
+                'borderRadius': 5,
+                'border': 'thin lightgrey solid',
+                'background-color': 'white',
+                # Remove possibility to select the text for better UX
+                'user-select': 'none',
+                '-moz-user-select': 'none',
+                '-webkit-user-select': 'none',
+                '-ms-user-select': 'none',
+            },
+            kwargs.get('style', {}),
+        ),
+        **_omit(['style'], kwargs),
+    )
+
+
+def SectionTitle(title, size, align='center', color='#222'):
+    return html.Div(
+        style={'text-align': align, 'color': color},
+        children=dcc.Markdown('#' * size + ' ' + title),
+    )
+
+
+def NamedCard(title, size, children, **kwargs):
+    size = min(size, 6)
+    size = max(size, 1)
+
+    return html.Div([Card([SectionTitle(title, size, align='left')] + children, **kwargs)])
+
+
+def NamedSlider(name, **kwargs):
+    return html.Div(
+        style={'padding': '20px 10px 25px 4px'},
+        children=[
+            html.P(f'{name}:'),
+            html.Div(style={'margin-left': '6px'}, children=dcc.Slider(**kwargs)),
+        ],
+    )
+
+
+def NamedDropdown(name, **kwargs):
+    return html.Div(
+        style={'margin': '10px 0px'},
+        children=[
+            html.P(children=f'{name}:', style={'margin-left': '3px'}),
+            dcc.Dropdown(**kwargs),
+        ],
+    )
+
+
+def NamedRadioItems(name, **kwargs):
+    return html.Div(
+        style={'padding': '20px 10px 25px 4px'},
+        children=[html.P(children=f'{name}:'), dcc.RadioItems(**kwargs)],
+    )
+
+
+def NamedInput(name, **kwargs):
+    return html.Div(children=[html.P(children=f'{name}:'), dcc.Input(**kwargs)])
+
+
+# Utils
+def DropdownOptionsList(*args):
+    return [{'label': val.capitalize(), 'value': val} for val in args]
+
+
+asset_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'assets')
+
+app = dash.Dash(__name__, assets_folder=asset_path)
+server = app.server
+
+
+# ###################### DATA PREPROCESSING ######################
+# Load data
+with open('sample_network.txt', 'r', encoding='utf-8') as f:
+    network_data = f.read().split('\n')
+
+# We select the first 750 edges and associated nodes for an easier visualization
+edges = network_data[:750]
+nodes = set()
+
+following_node_di = {}  # user id -> list of users they are following
+following_edges_di = {}  # user id -> list of cy edges starting from user id
+
+followers_node_di = {}  # user id -> list of followers (cy_node format)
+followers_edges_di = {}  # user id -> list of cy edges ending at user id
+
+cy_edges = []
+cy_nodes = []
+
+for edge in edges:
+    if ' ' not in edge:
+        continue
+
+    source, target = edge.split(' ')
+
+    cy_edge = {'data': {'id': source + target, 'source': source, 'target': target}}
+    cy_target = {'data': {'id': target, 'label': 'User #' + str(target[-5:])}}
+    cy_source = {'data': {'id': source, 'label': 'User #' + str(source[-5:])}}
+
+    if source not in nodes:
+        nodes.add(source)
+        cy_nodes.append(cy_source)
+    if target not in nodes:
+        nodes.add(target)
+        cy_nodes.append(cy_target)
+
+    # Process dictionary of following
+    if not following_node_di.get(source):
+        following_node_di[source] = []
+    if not following_edges_di.get(source):
+        following_edges_di[source] = []
+
+    following_node_di[source].append(cy_target)
+    following_edges_di[source].append(cy_edge)
+
+    # Process dictionary of followers
+    if not followers_node_di.get(target):
+        followers_node_di[target] = []
+    if not followers_edges_di.get(target):
+        followers_edges_di[target] = []
+
+    followers_node_di[target].append(cy_source)
+    followers_edges_di[target].append(cy_edge)
+
+genesis_node = cy_nodes[0]
+genesis_node['classes'] = 'genesis'
+default_elements = [genesis_node]
+
+default_stylesheet = [
+    {'selector': 'node', 'style': {'opacity': 0.65, 'z-index': 9999}},
+    {
+        'selector': 'edge',
+        'style': {'curve-style': 'bezier', 'opacity': 0.45, 'z-index': 5000},
+    },
+    {'selector': '.followerNode', 'style': {'background-color': '#0074D9'}},
+    {
+        'selector': '.followerEdge',
+        'style': {
+            'mid-target-arrow-color': 'blue',
+            'mid-target-arrow-shape': 'vee',
+            'line-color': '#0074D9',
+        },
+    },
+    {'selector': '.followingNode', 'style': {'background-color': '#FF4136'}},
+    {
+        'selector': '.followingEdge',
+        'style': {
+            'mid-target-arrow-color': 'red',
+            'mid-target-arrow-shape': 'vee',
+            'line-color': '#FF4136',
+        },
+    },
+    {
+        'selector': '.genesis',
+        'style': {
+            'background-color': '#B10DC9',
+            'border-width': 2,
+            'border-color': 'purple',
+            'border-opacity': 1,
+            'opacity': 1,
+            'label': 'data(label)',
+            'color': '#B10DC9',
+            'text-opacity': 1,
+            'font-size': 12,
+            'z-index': 9999,
+        },
+    },
+    {
+        'selector': ':selected',
+        'style': {
+            'border-width': 2,
+            'border-color': 'black',
+            'border-opacity': 1,
+            'opacity': 1,
+            'label': 'data(label)',
+            'color': 'black',
+            'font-size': 12,
+            'z-index': 9999,
+        },
+    },
+]
+
+# ################################# APP LAYOUT ################################
+styles = {
+    'json-output': {
+        'overflow-y': 'scroll',
+        'height': 'calc(50% - 25px)',
+        'border': 'thin lightgrey solid',
+    },
+    'tab': {'height': 'calc(98vh - 80px)'},
+}
+
+app.layout = html.Div(
+    [
+        html.Div(
+            className='eight columns',
+            children=[
+                cyto.Cytoscape(
+                    id='cytoscape',
+                    elements=default_elements,
+                    stylesheet=default_stylesheet,
+                    style={'height': '95vh', 'width': '100%'},
+                )
+            ],
+        ),
+        html.Div(
+            className='four columns',
+            children=[
+                dcc.Tabs(
+                    id='tabs',
+                    children=[
+                        dcc.Tab(
+                            label='Control Panel',
+                            children=[
+                                NamedDropdown(
+                                    name='Layout',
+                                    id='dropdown-layout',
+                                    options=DropdownOptionsList(
+                                        'random',
+                                        'grid',
+                                        'circle',
+                                        'concentric',
+                                        'breadthfirst',
+                                        'cose',
+                                        'cose-bilkent',
+                                        'dagre',
+                                        'cola',
+                                        'klay',
+                                        'spread',
+                                        'euler',
+                                    ),
+                                    value='grid',
+                                    clearable=False,
+                                ),
+                                NamedRadioItems(
+                                    name='Expand',
+                                    id='radio-expand',
+                                    options=DropdownOptionsList('followers', 'following'),
+                                    value='followers',
+                                ),
+                            ],
+                        ),
+                        dcc.Tab(
+                            label='JSON',
+                            children=[
+                                html.Div(
+                                    style=styles['tab'],
+                                    children=[
+                                        html.P('Node Object JSON:'),
+                                        html.Pre(
+                                            id='tap-node-json-output',
+                                            style=styles['json-output'],
+                                        ),
+                                        html.P('Edge Object JSON:'),
+                                        html.Pre(
+                                            id='tap-edge-json-output',
+                                            style=styles['json-output'],
+                                        ),
+                                    ],
+                                )
+                            ],
+                        ),
+                    ],
+                ),
+            ],
+        ),
+    ]
+)
+
+
+# ############################## CALLBACKS ####################################
+@callback(Output('tap-node-json-output', 'children'), Input('cytoscape', 'tapNode'))
+def display_tap_node(data):
+    return json.dumps(data, indent=2)
+
+
+@callback(Output('tap-edge-json-output', 'children'), Input('cytoscape', 'tapEdge'))
+def display_tap_edge(data):
+    return json.dumps(data, indent=2)
+
+
+@callback(Output('cytoscape', 'layout'), Input('dropdown-layout', 'value'))
+def update_cytoscape_layout(layout):
+    return {'name': layout}
+
+
+@callback(
+    Output('cytoscape', 'elements'),
+    Input('cytoscape', 'tapNodeData'),
+    State('cytoscape', 'elements'),
+    State('radio-expand', 'value'),
+)
+def generate_elements(nodeData, elements, expansion_mode):
+    if not nodeData:
+        return default_elements
+
+    # If the node has already been expanded, we don't expand it again
+    if nodeData.get('expanded'):
+        return elements
+
+    # This retrieves the currently selected element, and tag it as expanded
+    for element in elements:
+        if nodeData['id'] == element.get('data').get('id'):
+            element['data']['expanded'] = True
+            break
+
+    if expansion_mode == 'followers':
+        followers_nodes = followers_node_di.get(nodeData['id'])
+        followers_edges = followers_edges_di.get(nodeData['id'])
+
+        if followers_nodes:
+            for node in followers_nodes:
+                node['classes'] = 'followerNode'
+            elements.extend(followers_nodes)
+
+        if followers_edges:
+            for follower_edge in followers_edges:
+                follower_edge['classes'] = 'followerEdge'
+            elements.extend(followers_edges)
+
+    elif expansion_mode == 'following':
+        following_nodes = following_node_di.get(nodeData['id'])
+        following_edges = following_edges_di.get(nodeData['id'])
+
+        if following_nodes:
+            for node in following_nodes:
+                if node['data']['id'] != genesis_node['data']['id']:
+                    node['classes'] = 'followingNode'
+                    elements.append(node)
+
+        if following_edges:
+            for follower_edge in following_edges:
+                follower_edge['classes'] = 'followingEdge'
+            elements.extend(following_edges)
+
+    return elements
+
+
+if __name__ == '__main__':
+    app.run_server(debug=True)

scripts/dashboard/lang_main_config.toml

@@ -0,0 +1,56 @@
+# lang_main: Config file
+
+[paths]
+inputs = './inputs/'
+results = './results/test_new2/'
+dataset = './01_2_Rohdaten_neu/Export4.csv'
+#results = './results/Export7/'
+#dataset = './01_03_Rohdaten_202403/Export7_59499_Zeilen.csv'
+#results = './results/Export7_trunc/'
+#dataset = './01_03_Rohdaten_202403/Export7_trunc.csv'
+
+[control]
+preprocessing = true
+preprocessing_skip = false
+token_analysis = false
+token_analysis_skip = false
+graph_postprocessing = false
+graph_postprocessing_skip = false
+time_analysis = false
+time_analysis_skip = false
+
+#[export_filenames]
+#filename_cossim_filter_candidates = 'CosSim-FilterCandidates'
+
+[preprocess]
+filename_cossim_filter_candidates = 'CosSim-FilterCandidates'
+date_cols = [
+    "VorgangsDatum", 
+    "ErledigungsDatum", 
+    "Arbeitsbeginn", 
+    "ErstellungsDatum",
+]
+threshold_amount_characters = 5
+threshold_similarity = 0.8
+
+[graph_postprocessing]
+threshold_edge_weight = 150
+
+[time_analysis.uniqueness]
+threshold_unique_texts = 4
+criterion_feature = 'HObjektText'
+feature_name_obj_id = 'ObjektID'
+
+[time_analysis.model_input]
+input_features = [
+    'VorgangsTypName',
+    'VorgangsArtText',
+    'VorgangsBeschreibung',
+]
+activity_feature = 'VorgangsTypName'
+activity_types = [
+    'Reparaturauftrag (Portal)',
+    'Störungsmeldung',
+]
+threshold_num_acitivities = 1
+threshold_similarity = 0.8

scripts/lang_main_config.toml

@@ -10,14 +10,14 @@ dataset = '../data/02_202307/Export4.csv'
 #dataset = './01_03_Rohdaten_202403/Export7_trunc.csv'
 
 [control]
-preprocessing = true
-preprocessing_skip = true
-token_analysis = false
-token_analysis_skip = true
+preprocessing = false
+preprocessing_skip = false
+token_analysis = true
+token_analysis_skip = false
 graph_postprocessing = false
 graph_postprocessing_skip = true
-time_analysis = true
-time_analysis_skip = false
+time_analysis = false
+time_analysis_skip = true
 
 #[export_filenames]
 #filename_cossim_filter_candidates = 'CosSim-FilterCandidates'

scripts/pre_test_examples.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from lang_main.constants import (
+    INPUT_PATH_FOLDER,
+    PATH_TO_DATASET,
+    SAVE_PATH_FOLDER,
+    input_path_conf,
+)
+
+print(SAVE_PATH_FOLDER, '\n')
+print(INPUT_PATH_FOLDER, '\n')
+print(PATH_TO_DATASET, '\n')
+
+print('------------------------')
+print(Path.cwd(), '\n', input_path_conf)