mop-tom/lang-main
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 3 Releases Wiki Activity

more robust graph filtering

Browse Source
This commit is contained in:
Florian Förster
2024-09-12 15:19:24 +02:00
parent e85334ab06
commit 27d40d5c99
13 changed files with 1332 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/lang_main/analysis/graphs.py
View File

@@ -5,7 +5,7 @@ import sys
import typing
from collections.abc import Hashable, Iterable
from pathlib import Path
from typing import Any, Final, Literal, Self, cast, overload
from typing import Any, Literal, Self, cast, overload
import networkx as nx
import numpy as np
@@ -15,9 +15,14 @@ from pandas import DataFrame
from lang_main.constants import (
EDGE_WEIGHT_DECIMALS,
LOGGING_DEFAULT_GRAPHS,
PROPERTY_NAME_DEGREE_WEIGHTED,
)
from lang_main.errors import EdgePropertyNotContainedError, EmptyEdgesError, EmptyGraphError
from lang_main.errors import (
EdgePropertyNotContainedError,
EmptyEdgesError,
EmptyGraphError,
)
from lang_main.io import load_pickle, save_pickle
from lang_main.loggers import logger_graphs as logger
from lang_main.types import (
@@ -27,9 +32,6 @@ from lang_main.types import (
WeightData,
)
# TODO change logging behaviour, add logging to file
LOGGING_DEFAULT: Final[bool] = False
def save_to_GraphML(
graph: DiGraph | Graph,
@@ -45,7 +47,7 @@ def save_to_GraphML(
def get_graph_metadata(
graph: Graph | DiGraph,
logging: bool = LOGGING_DEFAULT,
logging: bool = LOGGING_DEFAULT_GRAPHS,
) -> dict[str, int]:
# info about graph
graph_info: dict[str, int] = {}
@@ -121,7 +123,7 @@ def update_graph(
# build undirected adjacency matrix
def convert_graph_to_undirected(
graph: DiGraph,
logging: bool = LOGGING_DEFAULT,
logging: bool = LOGGING_DEFAULT_GRAPHS,
cast_int: bool = False,
) -> Graph:
dtype = np.float32
@@ -282,6 +284,23 @@ def filter_graph_by_node_degree(
return filtered_graph
def filter_graph_by_number_edges(
graph: TokenGraph,
limit: int,
property: str = 'weight',
descending: bool = True,
) -> TokenGraph:
graph = graph.copy()
# edges
original = set(graph.edges(data=property)) # type: ignore
original_sorted = sorted(original, key=lambda tup: tup[2], reverse=descending)
chosen = set(original_sorted[:limit])
edges_to_drop = original.difference(chosen)
graph.remove_edges_from(edges_to_drop)
return graph
def add_weighted_degree(
graph: DiGraph | Graph,
edge_weight_property: str = 'weight',

38
src/lang_main/constants.py
View File

@@ -1,10 +1,6 @@
from pathlib import Path
from typing import Final
# TODO check removal
# import spacy
# from sentence_transformers import SentenceTransformer
# from spacy.language import Language as GermanSpacyModel
from lang_main import CONFIG, CYTO_PATH_STYLESHEET
from lang_main import model_loader as m_load
from lang_main.types import (
@@ -20,19 +16,20 @@ __all__ = [
'CYTO_PATH_STYLESHEET',
]
# ** logging
# graphs
LOGGING_DEFAULT_GRAPHS: Final[bool] = False
# ** paths
input_path_conf = Path.cwd() / Path(CONFIG['paths']['inputs'])
INPUT_PATH_FOLDER: Final[Path] = input_path_conf.resolve()
# INPUT_PATH_FOLDER: Final[Path] = (CALLER_PATH / input_path_conf).resolve()
# TODO reactivate later
# if not INPUT_PATH_FOLDER.exists():
# raise FileNotFoundError(f'Input path >>{INPUT_PATH_FOLDER}<< does not exist.')
save_path_conf = Path.cwd() / Path(CONFIG['paths']['results'])
SAVE_PATH_FOLDER: Final[Path] = save_path_conf.resolve()
# SAVE_PATH_FOLDER: Final[Path] = (CALLER_PATH / save_path_conf).resolve()
path_dataset_conf = Path.cwd() / Path(CONFIG['paths']['dataset'])
PATH_TO_DATASET: Final[Path] = path_dataset_conf.resolve()
# PATH_TO_DATASET: Final[Path] = (CALLER_PATH / path_dataset_conf).resolve()
# if not PATH_TO_DATASET.exists():
# raise FileNotFoundError(f'Dataset path >>{PATH_TO_DATASET}<< does not exist.')
# ** control
@@ -64,20 +61,9 @@ MODEL_LOADER_MAP: Final[ModelLoaderMap] = {
},
},
}
# ** sentence_transformers
# STFR_MODEL: Final[SentenceTransformer] = SentenceTransformer(
# 'sentence-transformers/all-mpnet-base-v2', device=STFR_DEVICE
# )
# ** spacy
# SPCY_MODEL: Final[GermanSpacyModel] = spacy.load('de_dep_news_trf')
# ** export
# ** preprocessing
FILENAME_COSSIM_FILTER_CANDIDATES: Final[str] = CONFIG['preprocess'][
'filename_cossim_filter_candidates'
]
DATE_COLS: Final[list[str]] = CONFIG['preprocess']['date_cols']
THRESHOLD_AMOUNT_CHARACTERS: Final[float] = CONFIG['preprocess'][
'threshold_amount_characters'
@@ -87,10 +73,13 @@ THRESHOLD_SIMILARITY: Final[float] = CONFIG['preprocess']['threshold_similarity'
# ** graph postprocessing
EDGE_WEIGHT_DECIMALS: Final[int] = 4
THRESHOLD_EDGE_WEIGHT: Final[int] = CONFIG['graph_postprocessing']['threshold_edge_weight']
THRESHOLD_EDGE_NUMBER: Final[int] = CONFIG['graph_postprocessing']['threshold_edge_number']
# THRESHOLD_EDGE_WEIGHT: Final[int] = CONFIG['graph_postprocessing']['threshold_edge_weight']
PROPERTY_NAME_DEGREE_WEIGHTED: Final[str] = 'degree_weighted'
# ** graph exports (Cytoscape)
CYTO_MAX_NODE_COUNT: Final[int] = 500
CYTO_MAX_EDGE_COUNT: Final[int] = 800
CYTO_COLLECTION_NAME: Final[str] = 'lang_main'
CYTO_BASE_NETWORK_NAME: Final[str] = 'token_graph'
CYTO_LAYOUT_NAME: Final[CytoLayouts] = 'force-directed'
@@ -119,9 +108,14 @@ UNIQUE_CRITERION_FEATURE: Final[str] = CONFIG['time_analysis']['uniqueness'][
]
FEATURE_NAME_OBJ_ID: Final[str] = CONFIG['time_analysis']['uniqueness']['feature_name_obj_id']
# ** time_analysis.preparation
NAME_DELTA_FEAT_TO_REPAIR: Final[str] = 'delta_to_repair'
# NAME_DELTA_FEAT_TO_REPAIR: Final[str] = 'Zeitspanne bis zur Behebung [Tage]'
NAME_DELTA_FEAT_TO_NEXT_FAILURE: Final[str] = 'Zeitspanne bis zum nächsten Ereignis [Tage]'
# NAME_DELTA_FEAT_TO_REPAIR: Final[str] = 'delta_to_repair'
CONFIG['time_analysis']['preparation']['name_delta_feat_to_repair']
NAME_DELTA_FEAT_TO_REPAIR: Final[str] = CONFIG['time_analysis']['preparation'][
'name_delta_feat_to_repair'
]
NAME_DELTA_FEAT_TO_NEXT_FAILURE: Final[str] = CONFIG['time_analysis']['preparation'][
'name_delta_feat_to_next_failure'
]
# ** time_analysis.model_input
MODEL_INPUT_FEATURES: Final[tuple[str, ...]] = tuple(
CONFIG['time_analysis']['model_input']['input_features']

4
src/lang_main/errors.py
View File

@@ -10,3 +10,7 @@ class EmptyGraphError(Exception):
class EmptyEdgesError(EmptyGraphError):
"""Error raised if action should be performed on a graph's edges, but
it does not contain any"""
class GraphRenderError(Exception):
"""Error raised if a graph object can not be rendered"""

25
src/lang_main/lang_main_config.toml
View File

@@ -2,12 +2,10 @@
[paths]
inputs = './inputs/'
results = './results/test_20240619/'
# results = './results/dummy_N_1000/'
# dataset = '../data/Dummy_Dataset_N_1000.csv'
results = './results/test_20240807/'
dataset = '../data/02_202307/Export4.csv'
#results = './results/Export7/'
#dataset = './01_03_Rohdaten_202403/Export7_59499_Zeilen.csv'
#results = './results/Export7_trunc/'
#dataset = './01_03_Rohdaten_202403/Export7_trunc.csv'
# only debugging features, production-ready pipelines should always
# be fully executed
@@ -19,28 +17,29 @@ graph_rescaling_skip = false
graph_static_rendering_skip = false
time_analysis_skip = true
#[export_filenames]
#filename_cossim_filter_candidates = 'CosSim-FilterCandidates'
[preprocess]
filename_cossim_filter_candidates = 'CosSim-FilterCandidates'
date_cols = [
"VorgangsDatum",
"ErledigungsDatum",
"Arbeitsbeginn",
"VorgangsDatum",
"ErledigungsDatum",
"Arbeitsbeginn",
"ErstellungsDatum",
]
threshold_amount_characters = 5
threshold_similarity = 0.8
[graph_postprocessing]
threshold_edge_weight = 150
threshold_edge_number = 330
# threshold_edge_weight = 150
[time_analysis.uniqueness]
threshold_unique_texts = 4
criterion_feature = 'HObjektText'
feature_name_obj_id = 'ObjektID'
[time_analysis.preparation]
name_delta_feat_to_repair = 'Zeitspanne bis zur Behebung [Tage]'
name_delta_feat_to_next_failure = 'Zeitspanne bis zum nächsten Ereignis [Tage]'
[time_analysis.model_input]
# input_features = [
# 'VorgangsTypName',

40
src/lang_main/pipelines/predefined.py
View File

@@ -34,7 +34,7 @@ from lang_main.constants import (
NAME_DELTA_FEAT_TO_REPAIR,
SAVE_PATH_FOLDER,
THRESHOLD_AMOUNT_CHARACTERS,
THRESHOLD_EDGE_WEIGHT,
THRESHOLD_EDGE_NUMBER,
THRESHOLD_NUM_ACTIVITIES,
THRESHOLD_SIMILARITY,
THRESHOLD_TIMELINE_SIMILARITY,
@@ -89,31 +89,6 @@ def build_base_target_feature_pipe() -> Pipeline:
return pipe_target_feat
# output: DataFrame containing target feature with
# number of occurrences and associated ObjectIDs
# ** embedding pipe
# ?? still needed?
# using similarity between entries to catch duplicates with typo or similar content
# pipe_embds = BasePipeline(name='Embedding1', working_dir=SAVE_PATH_FOLDER)
# pipe_embds.add(build_cosSim_matrix, {'model': model_stfr}, save_result=True)
# pipe_embds.add(
# filt_thresh_cosSim_matrix, {'threshold': THRESHOLD_SIMILARITY}, save_result=True
# )
# pipe_embds.add(
# list_cosSim_dupl_candidates,
# {
# 'save_candidates': True,
# 'saving_path': SAVE_PATH_FOLDER,
# 'filename': FILENAME_COSSIM_FILTER_CANDIDATES,
# 'pipeline': pipe_embds,
# },
# save_result=True,
# )
# ** Merge duplicates
def build_merge_duplicates_pipe() -> Pipeline:
pipe_merge = Pipeline(name='Merge_Duplicates', working_dir=SAVE_PATH_FOLDER)
@@ -162,11 +137,18 @@ def build_tk_graph_post_pipe() -> Pipeline:
pipe_graph_postprocessing = Pipeline(
name='Graph_Postprocessing', working_dir=SAVE_PATH_FOLDER
)
# pipe_graph_postprocessing.add(
# graphs.filter_graph_by_edge_weight,
# {
# 'bound_lower': THRESHOLD_EDGE_WEIGHT,
# 'bound_upper': None,
# },
# )
pipe_graph_postprocessing.add(
graphs.filter_graph_by_edge_weight,
graphs.filter_graph_by_number_edges,
{
'bound_lower': THRESHOLD_EDGE_WEIGHT,
'bound_upper': None,
'limit': THRESHOLD_EDGE_NUMBER,
'property': 'weight',
},
)
pipe_graph_postprocessing.add(

73
src/lang_main/render/cytoscape.py
View File

@@ -5,6 +5,7 @@ from typing import Literal, cast
import py4cytoscape as p4c
from networkx import DiGraph, Graph
from py4cytoscape import network_selection as p4c_network_selection
from py4cytoscape.exceptions import CyError
from requests.exceptions import RequestException
@@ -14,6 +15,8 @@ from lang_main.constants import (
CYTO_ITER_NEIGHBOUR_DEPTH,
CYTO_LAYOUT_NAME,
CYTO_LAYOUT_PROPERTIES,
CYTO_MAX_EDGE_COUNT,
CYTO_MAX_NODE_COUNT,
CYTO_NETWORK_ZOOM_FACTOR,
CYTO_NUMBER_SUBGRAPHS,
CYTO_PATH_STYLESHEET,
@@ -23,7 +26,9 @@ from lang_main.constants import (
PROPERTY_NAME_DEGREE_WEIGHTED,
SAVE_PATH_FOLDER,
)
from lang_main.errors import GraphRenderError
from lang_main.loggers import logger_rendering as logger
from lang_main.render import cytoscape_monkeypatch as cs_monkeypatch
from lang_main.types import (
CytoExportFileTypes,
CytoExportPageSizes,
@@ -32,9 +37,17 @@ from lang_main.types import (
CytoNodeID,
)
# monkeypatch non-stable py4cytoscape function
p4c_network_selection.select_edges_connecting_selected_nodes = (
cs_monkeypatch.select_edges_connecting_selected_nodes
)
p4c.select_edges_connecting_selected_nodes = (
cs_monkeypatch.select_edges_connecting_selected_nodes
)
# ** Cytoscape API related, using py4cytoscape
def verify_connection():
def verify_connection() -> None:
"""Cytoscape: checks if CyREST and Cytoscape versions are compatible nad
if Cytoscape API endpoint is reachable
@@ -55,6 +68,60 @@ def verify_connection():
raise error
def verify_graph_render_size(
graph: Graph | DiGraph,
max_node_count: int | None = CYTO_MAX_NODE_COUNT,
max_edge_count: int | None = CYTO_MAX_EDGE_COUNT,
) -> None:
"""verify that the graph size can still be handled within an acceptable time
frame for rendering in Cytoscape
Parameters
----------
graph : Graph | DiGraph
graph to verify
max_node_count : int | None, optional
maximum allowed number of nodes, by default CYTO_MAX_NODE_COUNT
max_edge_count : int | None, optional
maximum allowed number of edges, by default CYTO_MAX_EDGE_COUNT
Raises
------
GraphRenderError
if any of the provided limits is exceeded
"""
num_nodes = len(graph.nodes)
num_edges = len(graph.edges)
if max_node_count is not None and num_nodes > max_node_count:
raise GraphRenderError(
f'Maximum number of nodes for rendering exceeded. '
f'Limit {max_node_count}, Counted: {num_nodes}'
)
if max_edge_count is not None and num_edges > max_edge_count:
raise GraphRenderError(
f'Maximum number of edges for rendering exceeded. '
f'Limit {max_edge_count}, Counted: {num_edges}'
)
def change_default_layout() -> None:
"""Cytoscape: resets the default layout to `grid` to accelerate the import process
(grid layout one of the fastest)
Raises
------
RequestException
API endpoint not reachable or CyREST operation not successful
"""
body: dict[str, str] = {'value': 'grid', 'key': 'layout.default'}
try:
p4c.cyrest_put('properties/cytoscape3.props/layout.default', body=body)
except RequestException as error:
logger.error('[CytoAPIConnection] Property change of default layout not successful.')
raise error
def import_to_cytoscape(
graph: DiGraph | Graph,
network_name: str = CYTO_BASE_NETWORK_NAME,
@@ -70,6 +137,10 @@ def import_to_cytoscape(
"""
logger.debug('Checking Cytoscape connection...')
verify_connection()
logger.debug('Checking graph size for rendering...')
verify_graph_render_size(graph)
logger.debug('Setting default layout to improve import speed...')
change_default_layout()
logger.debug('Setting Cytoscape sandbox...')
p4c.sandbox_set(
sandbox_name=sandbox_name,

73
src/lang_main/render/cytoscape_monkeypatch.py Normal file
View File

@@ -0,0 +1,73 @@
import re
from py4cytoscape import networks
from py4cytoscape.network_selection import get_selected_nodes, select_edges
from py4cytoscape.py4cytoscape_logger import cy_log
from py4cytoscape.py4cytoscape_utils import * # type: ignore # noqa: F403
re_parenthesis_1 = re.compile(r'[(]+')
re_parenthesis_2 = re.compile(r'[)]+')
@cy_log
def select_edges_connecting_selected_nodes(network=None, base_url=DEFAULT_BASE_URL): # noqa: F405
"""Select edges in a Cytoscape Network connecting the selected nodes, including self loops connecting single nodes.
Any edges selected beforehand are deselected before any new edges are selected
Args:
network (SUID or str or None): Name or SUID of a network. Default is the
"current" network active in Cytoscape.
base_url (str): Ignore unless you need to specify a custom domain,
port or version to connect to the CyREST API. Default is http://127.0.0.1:1234
and the latest version of the CyREST API supported by this version of py4cytoscape.
Returns:
dict: {'nodes': [node list], 'edges': [edge list]} or None if no selected nodes
Raises:
CyError: if network name or SUID doesn't exist
requests.exceptions.RequestException: if can't connect to Cytoscape or Cytoscape returns an error
Examples:
>>> select_edges_connecting_selected_nodes()
None
>>> select_edges_connecting_selected_nodes(network='My Network')
{'nodes': [103990, 103991, ...], 'edges': [104432, 104431, ...]}
>>> select_edges_connecting_selected_nodes(network=52)
{'nodes': [103990, 103991, ...], 'edges': [104432, 104431, ...]}
Note:
In the return value node list is list of all selected nodes, and
edge list is the SUIDs of selected edges -- dict is None if no nodes were selected or there were no newly
created edges
"""
net_suid = networks.get_network_suid(network, base_url=base_url)
selected_nodes = get_selected_nodes(network=net_suid, base_url=base_url)
# TODO: In R version, NA test is after len() test ... shouldn't it be before?
if not selected_nodes:
return None
all_edges = networks.get_all_edges(net_suid, base_url=base_url)
selected_sources = set()
selected_targets = set()
for n in selected_nodes:
n = re_parenthesis_1.sub('\(', n) # type: ignore
n = re_parenthesis_2.sub('\)', n) # type: ignore
selected_sources |= set(filter(re.compile('^' + n).search, all_edges)) # type: ignore
selected_targets |= set(filter(re.compile(n + '$').search, all_edges)) # type: ignore
selected_edges = list(selected_sources.intersection(selected_targets))
if len(selected_edges) == 0:
return None
res = select_edges(
selected_edges,
by_col='name',
preserve_current_selection=False,
network=net_suid,
base_url=base_url,
)
return res
# TODO: isn't the pattern match a bit cheesy ... shouldn't it be ^+n+' (' and ') '+n+$ ???