756 lines
32 KiB
HTML
756 lines
32 KiB
HTML
<!doctype html>
|
||
<html lang="en">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1">
|
||
<meta name="generator" content="pdoc3 0.11.5">
|
||
<title>lang_main.pipelines.base API documentation</title>
|
||
<meta name="description" content="">
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/13.0.0/sanitize.min.css" integrity="sha512-y1dtMcuvtTMJc1yPgEqF0ZjQbhnc/bFhyvIyVNb9Zk5mIGtqVaAB1Ttl28su8AvFMOY0EwRbAe+HCLqj6W7/KA==" crossorigin>
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/10up-sanitize.css/13.0.0/typography.min.css" integrity="sha512-Y1DYSb995BAfxobCkKepB1BqJJTPrOp3zPL74AWFugHHmmdcvO+C48WLrUOlhGMc0QG7AE3f7gmvvcrmX2fDoA==" crossorigin>
|
||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css" crossorigin>
|
||
<style>:root{--highlight-color:#fe9}.flex{display:flex !important}body{line-height:1.5em}#content{padding:20px}#sidebar{padding:1.5em;overflow:hidden}#sidebar > *:last-child{margin-bottom:2cm}.http-server-breadcrumbs{font-size:130%;margin:0 0 15px 0}#footer{font-size:.75em;padding:5px 30px;border-top:1px solid #ddd;text-align:right}#footer p{margin:0 0 0 1em;display:inline-block}#footer p:last-child{margin-right:30px}h1,h2,h3,h4,h5{font-weight:300}h1{font-size:2.5em;line-height:1.1em}h2{font-size:1.75em;margin:2em 0 .50em 0}h3{font-size:1.4em;margin:1.6em 0 .7em 0}h4{margin:0;font-size:105%}h1:target,h2:target,h3:target,h4:target,h5:target,h6:target{background:var(--highlight-color);padding:.2em 0}a{color:#058;text-decoration:none;transition:color .2s ease-in-out}a:visited{color:#503}a:hover{color:#b62}.title code{font-weight:bold}h2[id^="header-"]{margin-top:2em}.ident{color:#900;font-weight:bold}pre code{font-size:.8em;line-height:1.4em;padding:1em;display:block}code{background:#f3f3f3;font-family:"DejaVu Sans Mono",monospace;padding:1px 4px;overflow-wrap:break-word}h1 code{background:transparent}pre{border-top:1px solid #ccc;border-bottom:1px solid #ccc;margin:1em 0}#http-server-module-list{display:flex;flex-flow:column}#http-server-module-list div{display:flex}#http-server-module-list dt{min-width:10%}#http-server-module-list p{margin-top:0}.toc ul,#index{list-style-type:none;margin:0;padding:0}#index code{background:transparent}#index h3{border-bottom:1px solid #ddd}#index ul{padding:0}#index h4{margin-top:.6em;font-weight:bold}@media (min-width:200ex){#index .two-column{column-count:2}}@media (min-width:300ex){#index .two-column{column-count:3}}dl{margin-bottom:2em}dl dl:last-child{margin-bottom:4em}dd{margin:0 0 1em 3em}#header-classes + dl > dd{margin-bottom:3em}dd dd{margin-left:2em}dd p{margin:10px 0}.name{background:#eee;font-size:.85em;padding:5px 10px;display:inline-block;min-width:40%}.name:hover{background:#e0e0e0}dt:target .name{background:var(--highlight-color)}.name > span:first-child{white-space:nowrap}.name.class > span:nth-child(2){margin-left:.4em}.inherited{color:#999;border-left:5px solid #eee;padding-left:1em}.inheritance em{font-style:normal;font-weight:bold}.desc h2{font-weight:400;font-size:1.25em}.desc h3{font-size:1em}.desc dt code{background:inherit}.source > summary,.git-link-div{color:#666;text-align:right;font-weight:400;font-size:.8em;text-transform:uppercase}.source summary > *{white-space:nowrap;cursor:pointer}.git-link{color:inherit;margin-left:1em}.source pre{max-height:500px;overflow:auto;margin:0}.source pre code{font-size:12px;overflow:visible;min-width:max-content}.hlist{list-style:none}.hlist li{display:inline}.hlist li:after{content:',\2002'}.hlist li:last-child:after{content:none}.hlist .hlist{display:inline;padding-left:1em}img{max-width:100%}td{padding:0 .5em}.admonition{padding:.1em 1em;margin:1em 0}.admonition-title{font-weight:bold}.admonition.note,.admonition.info,.admonition.important{background:#aef}.admonition.todo,.admonition.versionadded,.admonition.tip,.admonition.hint{background:#dfd}.admonition.warning,.admonition.versionchanged,.admonition.deprecated{background:#fd4}.admonition.error,.admonition.danger,.admonition.caution{background:lightpink}</style>
|
||
<style media="screen and (min-width: 700px)">@media screen and (min-width:700px){#sidebar{width:30%;height:100vh;overflow:auto;position:sticky;top:0}#content{width:70%;max-width:100ch;padding:3em 4em;border-left:1px solid #ddd}pre code{font-size:1em}.name{font-size:1em}main{display:flex;flex-direction:row-reverse;justify-content:flex-end}.toc ul ul,#index ul ul{padding-left:1em}.toc > ul > li{margin-top:.5em}}</style>
|
||
<style media="print">@media print{#sidebar h1{page-break-before:always}.source{display:none}}@media print{*{background:transparent !important;color:#000 !important;box-shadow:none !important;text-shadow:none !important}a[href]:after{content:" (" attr(href) ")";font-size:90%}a[href][title]:after{content:none}abbr[title]:after{content:" (" attr(title) ")"}.ir a:after,a[href^="javascript:"]:after,a[href^="#"]:after{content:""}pre,blockquote{border:1px solid #999;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}img{max-width:100% !important}@page{margin:0.5cm}p,h2,h3{orphans:3;widows:3}h1,h2,h3,h4,h5,h6{page-break-after:avoid}}</style>
|
||
<script defer src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js" integrity="sha512-D9gUyxqja7hBtkWpPWGt9wfbfaMGVt9gnyCvYa+jojwwPHLCzUm5i8rpk7vD7wNee9bA35eYIjobYPaQuKS1MQ==" crossorigin></script>
|
||
<script>window.addEventListener('DOMContentLoaded', () => {
|
||
hljs.configure({languages: ['bash', 'css', 'diff', 'graphql', 'ini', 'javascript', 'json', 'plaintext', 'python', 'python-repl', 'rust', 'shell', 'sql', 'typescript', 'xml', 'yaml']});
|
||
hljs.highlightAll();
|
||
/* Collapse source docstrings */
|
||
setTimeout(() => {
|
||
[...document.querySelectorAll('.hljs.language-python > .hljs-string')]
|
||
.filter(el => el.innerHTML.length > 200 && ['"""', "'''"].includes(el.innerHTML.substring(0, 3)))
|
||
.forEach(el => {
|
||
let d = document.createElement('details');
|
||
d.classList.add('hljs-string');
|
||
d.innerHTML = '<summary>"""</summary>' + el.innerHTML.substring(3);
|
||
el.replaceWith(d);
|
||
});
|
||
}, 100);
|
||
})</script>
|
||
</head>
|
||
<body>
|
||
<main>
|
||
<article id="content">
|
||
<header>
|
||
<h1 class="title">Module <code>lang_main.pipelines.base</code></h1>
|
||
</header>
|
||
<section id="section-intro">
|
||
</section>
|
||
<section>
|
||
</section>
|
||
<section>
|
||
</section>
|
||
<section>
|
||
</section>
|
||
<section>
|
||
<h2 class="section-title" id="header-classes">Classes</h2>
|
||
<dl>
|
||
<dt id="lang_main.pipelines.base.BasePipeline"><code class="flex name class">
|
||
<span>class <span class="ident">BasePipeline</span></span>
|
||
<span>(</span><span>name: str, working_dir: Path)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class BasePipeline(ABC):
|
||
def __init__(
|
||
self,
|
||
name: str,
|
||
working_dir: Path,
|
||
) -> None:
|
||
# init base class
|
||
super().__init__()
|
||
|
||
# name of pipeline
|
||
self.name = name
|
||
# working directory for pipeline == output path
|
||
self.working_dir = working_dir
|
||
|
||
# container for actions to perform during pass
|
||
self.actions: list[Callable] = []
|
||
self.action_names: list[str] = []
|
||
self.action_skip: list[bool] = []
|
||
# progress tracking, start at 1
|
||
self.curr_proc_idx: int = 1
|
||
|
||
def __repr__(self) -> str:
|
||
return (
|
||
f'{self.__class__.__name__}(name: {self.name}, '
|
||
f'working dir: {self.working_dir}, contents: {self.action_names})'
|
||
)
|
||
|
||
def panic_wrong_action_type(
|
||
self,
|
||
action: Any,
|
||
compatible_type: str,
|
||
) -> Never:
|
||
raise WrongActionTypeError(
|
||
(
|
||
f'Action must be of type {compatible_type}, '
|
||
f'but is of type >>{type(action)}<<.'
|
||
)
|
||
)
|
||
|
||
def prep_run(self) -> None:
|
||
logger.info('Starting pipeline >>%s<<...', self.name)
|
||
# progress tracking
|
||
self.curr_proc_idx = 1
|
||
# check if performable actions available
|
||
if len(self.actions) == 0:
|
||
raise NoPerformableActionError(
|
||
'The pipeline does not contain any performable actions.'
|
||
)
|
||
|
||
def post_run(self) -> None:
|
||
logger.info(
|
||
'Processing pipeline >>%s<< successfully ended after %d steps.',
|
||
self.name,
|
||
(self.curr_proc_idx - 1),
|
||
)
|
||
|
||
@abstractmethod
|
||
def add(self) -> None: ...
|
||
|
||
@abstractmethod
|
||
def logic(self) -> None: ...
|
||
|
||
def run(self, *args, **kwargs) -> Any:
|
||
self.prep_run()
|
||
ret = self.logic(*args, **kwargs)
|
||
self.post_run()
|
||
return ret</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Helper class that provides a standard way to create an ABC using
|
||
inheritance.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Subclasses</h3>
|
||
<ul class="hlist">
|
||
<li><a title="lang_main.pipelines.base.Pipeline" href="#lang_main.pipelines.base.Pipeline">Pipeline</a></li>
|
||
<li><a title="lang_main.pipelines.base.PipelineContainer" href="#lang_main.pipelines.base.PipelineContainer">PipelineContainer</a></li>
|
||
</ul>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="lang_main.pipelines.base.BasePipeline.add"><code class="name flex">
|
||
<span>def <span class="ident">add</span></span>(<span>self) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@abstractmethod
|
||
def add(self) -> None: ...</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.BasePipeline.logic"><code class="name flex">
|
||
<span>def <span class="ident">logic</span></span>(<span>self) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@abstractmethod
|
||
def logic(self) -> None: ...</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.BasePipeline.panic_wrong_action_type"><code class="name flex">
|
||
<span>def <span class="ident">panic_wrong_action_type</span></span>(<span>self, action: Any, compatible_type: str) ‑> Never</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def panic_wrong_action_type(
|
||
self,
|
||
action: Any,
|
||
compatible_type: str,
|
||
) -> Never:
|
||
raise WrongActionTypeError(
|
||
(
|
||
f'Action must be of type {compatible_type}, '
|
||
f'but is of type >>{type(action)}<<.'
|
||
)
|
||
)</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.BasePipeline.post_run"><code class="name flex">
|
||
<span>def <span class="ident">post_run</span></span>(<span>self) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def post_run(self) -> None:
|
||
logger.info(
|
||
'Processing pipeline >>%s<< successfully ended after %d steps.',
|
||
self.name,
|
||
(self.curr_proc_idx - 1),
|
||
)</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.BasePipeline.prep_run"><code class="name flex">
|
||
<span>def <span class="ident">prep_run</span></span>(<span>self) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def prep_run(self) -> None:
|
||
logger.info('Starting pipeline >>%s<<...', self.name)
|
||
# progress tracking
|
||
self.curr_proc_idx = 1
|
||
# check if performable actions available
|
||
if len(self.actions) == 0:
|
||
raise NoPerformableActionError(
|
||
'The pipeline does not contain any performable actions.'
|
||
)</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.BasePipeline.run"><code class="name flex">
|
||
<span>def <span class="ident">run</span></span>(<span>self, *args, **kwargs) ‑> Any</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def run(self, *args, **kwargs) -> Any:
|
||
self.prep_run()
|
||
ret = self.logic(*args, **kwargs)
|
||
self.post_run()
|
||
return ret</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.Pipeline"><code class="flex name class">
|
||
<span>class <span class="ident">Pipeline</span></span>
|
||
<span>(</span><span>name: str, working_dir: Path)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class Pipeline(BasePipeline):
|
||
def __init__(
|
||
self,
|
||
name: str,
|
||
working_dir: Path,
|
||
) -> None:
|
||
# init base class
|
||
super().__init__(name=name, working_dir=working_dir)
|
||
# name of pipeline
|
||
self.name = name
|
||
# working directory for pipeline == output path
|
||
self.working_dir = working_dir
|
||
# container for actions to perform during pass
|
||
self.actions_kwargs: list[dict[str, Any]] = []
|
||
self.save_results: ResultHandling = []
|
||
self.load_results: ResultHandling = []
|
||
# intermediate result
|
||
self._intermediate_result: tuple[Any, ...] | None = None
|
||
|
||
def __repr__(self) -> str:
|
||
return (
|
||
f'{self.__class__.__name__}(name: {self.name}, '
|
||
f'working dir: {self.working_dir}, contents: {self.action_names})'
|
||
)
|
||
|
||
@override
|
||
def add(
|
||
self,
|
||
action: Callable,
|
||
action_kwargs: dict[str, Any] | None = None,
|
||
skip: bool = False,
|
||
save_result: bool = False,
|
||
load_result: bool = False,
|
||
filename: str | None = None,
|
||
) -> None:
|
||
# check explicitly for function type
|
||
# if isinstance(action, FunctionType):
|
||
if action_kwargs is None:
|
||
action_kwargs = {}
|
||
|
||
if isinstance(action, Callable):
|
||
self.actions.append(action)
|
||
self.action_names.append(action.__name__)
|
||
self.actions_kwargs.append(action_kwargs.copy())
|
||
self.action_skip.append(skip)
|
||
self.save_results.append((save_result, filename))
|
||
self.load_results.append((load_result, filename))
|
||
else:
|
||
self.panic_wrong_action_type(action=action, compatible_type=Callable.__name__)
|
||
|
||
def get_result_path(
|
||
self,
|
||
action_idx: int,
|
||
filename: str | None,
|
||
) -> tuple[Path, str]:
|
||
action_name = self.action_names[action_idx]
|
||
if filename is None:
|
||
target_filename = f'Pipe-{self.name}_Step-{self.curr_proc_idx}_{action_name}'
|
||
else:
|
||
target_filename = filename
|
||
target_path = self.working_dir.joinpath(target_filename).with_suffix('.pkl')
|
||
return target_path, action_name
|
||
|
||
def load_step(
|
||
self,
|
||
action_idx: int,
|
||
filename: str | None,
|
||
) -> tuple[Any, ...]:
|
||
target_path, action_name = self.get_result_path(action_idx, filename)
|
||
|
||
if not target_path.exists():
|
||
raise FileNotFoundError(
|
||
(
|
||
f'No intermediate results for action >>{action_name}<< '
|
||
f'under >>{target_path}<< found'
|
||
)
|
||
)
|
||
# results should be tuple, but that is not guaranteed
|
||
result_loaded = cast(tuple[Any, ...], load_pickle(target_path))
|
||
if not isinstance(result_loaded, tuple):
|
||
raise TypeError(f'Loaded results must be tuple, not {type(result_loaded)}')
|
||
|
||
return result_loaded
|
||
|
||
def save_step(
|
||
self,
|
||
action_idx: int,
|
||
filename: str | None,
|
||
) -> None:
|
||
target_path, _ = self.get_result_path(action_idx, filename)
|
||
save_pickle(obj=self._intermediate_result, path=target_path)
|
||
|
||
@override
|
||
def logic(
|
||
self,
|
||
starting_values: tuple[Any, ...] | None = None,
|
||
) -> tuple[Any, ...]:
|
||
first_performed: bool = False
|
||
|
||
for idx, (action, action_kwargs) in enumerate(zip(self.actions, self.actions_kwargs)):
|
||
if self.action_skip[idx]:
|
||
self.curr_proc_idx += 1
|
||
continue
|
||
|
||
# loading
|
||
if self.load_results[idx][0]:
|
||
filename = self.load_results[idx][1]
|
||
ret = self.load_step(action_idx=idx, filename=filename)
|
||
self._intermediate_result = ret
|
||
logger.info(
|
||
'[No Calculation] Loaded result for action >>%s<< successfully',
|
||
self.action_names[idx],
|
||
)
|
||
self.curr_proc_idx += 1
|
||
continue
|
||
# calculation
|
||
if not first_performed:
|
||
args = starting_values
|
||
first_performed = True
|
||
else:
|
||
args = ret
|
||
|
||
if args is not None:
|
||
ret = action(*args, **action_kwargs)
|
||
else:
|
||
ret = action(**action_kwargs)
|
||
|
||
if ret is not None and not isinstance(ret, tuple):
|
||
ret = (ret,)
|
||
ret = cast(tuple[Any, ...], ret)
|
||
# save intermediate result
|
||
self._intermediate_result = ret
|
||
# saving result locally, always save last action
|
||
if self.save_results[idx][0] or idx == (len(self.actions) - 1):
|
||
filename = self.save_results[idx][1]
|
||
self.save_step(action_idx=idx, filename=filename)
|
||
# processing tracking
|
||
self.curr_proc_idx += 1
|
||
|
||
return ret</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Helper class that provides a standard way to create an ABC using
|
||
inheritance.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="lang_main.pipelines.base.BasePipeline" href="#lang_main.pipelines.base.BasePipeline">BasePipeline</a></li>
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="lang_main.pipelines.base.Pipeline.add"><code class="name flex">
|
||
<span>def <span class="ident">add</span></span>(<span>self,<br>action: Callable,<br>action_kwargs: dict[str, Any] | None = None,<br>skip: bool = False,<br>save_result: bool = False,<br>load_result: bool = False,<br>filename: str | None = None) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@override
|
||
def add(
|
||
self,
|
||
action: Callable,
|
||
action_kwargs: dict[str, Any] | None = None,
|
||
skip: bool = False,
|
||
save_result: bool = False,
|
||
load_result: bool = False,
|
||
filename: str | None = None,
|
||
) -> None:
|
||
# check explicitly for function type
|
||
# if isinstance(action, FunctionType):
|
||
if action_kwargs is None:
|
||
action_kwargs = {}
|
||
|
||
if isinstance(action, Callable):
|
||
self.actions.append(action)
|
||
self.action_names.append(action.__name__)
|
||
self.actions_kwargs.append(action_kwargs.copy())
|
||
self.action_skip.append(skip)
|
||
self.save_results.append((save_result, filename))
|
||
self.load_results.append((load_result, filename))
|
||
else:
|
||
self.panic_wrong_action_type(action=action, compatible_type=Callable.__name__)</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.Pipeline.get_result_path"><code class="name flex">
|
||
<span>def <span class="ident">get_result_path</span></span>(<span>self, action_idx: int, filename: str | None) ‑> tuple[pathlib.Path, str]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def get_result_path(
|
||
self,
|
||
action_idx: int,
|
||
filename: str | None,
|
||
) -> tuple[Path, str]:
|
||
action_name = self.action_names[action_idx]
|
||
if filename is None:
|
||
target_filename = f'Pipe-{self.name}_Step-{self.curr_proc_idx}_{action_name}'
|
||
else:
|
||
target_filename = filename
|
||
target_path = self.working_dir.joinpath(target_filename).with_suffix('.pkl')
|
||
return target_path, action_name</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.Pipeline.load_step"><code class="name flex">
|
||
<span>def <span class="ident">load_step</span></span>(<span>self, action_idx: int, filename: str | None) ‑> tuple[typing.Any, ...]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def load_step(
|
||
self,
|
||
action_idx: int,
|
||
filename: str | None,
|
||
) -> tuple[Any, ...]:
|
||
target_path, action_name = self.get_result_path(action_idx, filename)
|
||
|
||
if not target_path.exists():
|
||
raise FileNotFoundError(
|
||
(
|
||
f'No intermediate results for action >>{action_name}<< '
|
||
f'under >>{target_path}<< found'
|
||
)
|
||
)
|
||
# results should be tuple, but that is not guaranteed
|
||
result_loaded = cast(tuple[Any, ...], load_pickle(target_path))
|
||
if not isinstance(result_loaded, tuple):
|
||
raise TypeError(f'Loaded results must be tuple, not {type(result_loaded)}')
|
||
|
||
return result_loaded</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.Pipeline.logic"><code class="name flex">
|
||
<span>def <span class="ident">logic</span></span>(<span>self, starting_values: tuple[Any, ...] | None = None) ‑> tuple[typing.Any, ...]</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@override
|
||
def logic(
|
||
self,
|
||
starting_values: tuple[Any, ...] | None = None,
|
||
) -> tuple[Any, ...]:
|
||
first_performed: bool = False
|
||
|
||
for idx, (action, action_kwargs) in enumerate(zip(self.actions, self.actions_kwargs)):
|
||
if self.action_skip[idx]:
|
||
self.curr_proc_idx += 1
|
||
continue
|
||
|
||
# loading
|
||
if self.load_results[idx][0]:
|
||
filename = self.load_results[idx][1]
|
||
ret = self.load_step(action_idx=idx, filename=filename)
|
||
self._intermediate_result = ret
|
||
logger.info(
|
||
'[No Calculation] Loaded result for action >>%s<< successfully',
|
||
self.action_names[idx],
|
||
)
|
||
self.curr_proc_idx += 1
|
||
continue
|
||
# calculation
|
||
if not first_performed:
|
||
args = starting_values
|
||
first_performed = True
|
||
else:
|
||
args = ret
|
||
|
||
if args is not None:
|
||
ret = action(*args, **action_kwargs)
|
||
else:
|
||
ret = action(**action_kwargs)
|
||
|
||
if ret is not None and not isinstance(ret, tuple):
|
||
ret = (ret,)
|
||
ret = cast(tuple[Any, ...], ret)
|
||
# save intermediate result
|
||
self._intermediate_result = ret
|
||
# saving result locally, always save last action
|
||
if self.save_results[idx][0] or idx == (len(self.actions) - 1):
|
||
filename = self.save_results[idx][1]
|
||
self.save_step(action_idx=idx, filename=filename)
|
||
# processing tracking
|
||
self.curr_proc_idx += 1
|
||
|
||
return ret</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.Pipeline.save_step"><code class="name flex">
|
||
<span>def <span class="ident">save_step</span></span>(<span>self, action_idx: int, filename: str | None) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">def save_step(
|
||
self,
|
||
action_idx: int,
|
||
filename: str | None,
|
||
) -> None:
|
||
target_path, _ = self.get_result_path(action_idx, filename)
|
||
save_pickle(obj=self._intermediate_result, path=target_path)</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.PipelineContainer"><code class="flex name class">
|
||
<span>class <span class="ident">PipelineContainer</span></span>
|
||
<span>(</span><span>name: str, working_dir: Path)</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">class PipelineContainer(BasePipeline):
|
||
"""Container class for basic actions.
|
||
Basic actions are usually functions, which do not take any parameters
|
||
and return nothing. Indeed, if an action returns any values after its
|
||
procedure is finished, an error is raised. Therefore, PipelineContainers
|
||
can be seen as a concatenation of many (independent) simple procedures
|
||
which are executed in the order in which they were added to the pipe.
|
||
With a simple call of the ``run`` method the actions are performed.
|
||
Additionally, there is an option to skip actions which can be set in
|
||
the ``add`` method. This allows for easily configurable pipelines,
|
||
e.g., via a user configuration.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
name: str,
|
||
working_dir: Path,
|
||
) -> None:
|
||
super().__init__(name=name, working_dir=working_dir)
|
||
|
||
@override
|
||
def add(
|
||
self,
|
||
action: Callable,
|
||
skip: bool = False,
|
||
) -> None:
|
||
if isinstance(action, Callable):
|
||
self.actions.append(action)
|
||
self.action_names.append(action.__name__)
|
||
self.action_skip.append(skip)
|
||
else:
|
||
self.panic_wrong_action_type(action=action, compatible_type=Callable.__name__)
|
||
|
||
@override
|
||
def logic(self) -> None:
|
||
for idx, (action, action_name) in enumerate(zip(self.actions, self.action_names)):
|
||
# loading
|
||
if self.action_skip[idx]:
|
||
logger.info('[No Calculation] Skipping >>%s<<...', action_name)
|
||
self.curr_proc_idx += 1
|
||
continue
|
||
# calculation
|
||
ret = action()
|
||
if ret is not None:
|
||
raise OutputInPipelineContainerError(
|
||
(
|
||
f'Output in PipelineContainers not allowed. Action {action_name} '
|
||
f'returned values in Container {self.name}.'
|
||
)
|
||
)
|
||
# processing tracking
|
||
self.curr_proc_idx += 1</code></pre>
|
||
</details>
|
||
<div class="desc"><p>Container class for basic actions.
|
||
Basic actions are usually functions, which do not take any parameters
|
||
and return nothing. Indeed, if an action returns any values after its
|
||
procedure is finished, an error is raised. Therefore, PipelineContainers
|
||
can be seen as a concatenation of many (independent) simple procedures
|
||
which are executed in the order in which they were added to the pipe.
|
||
With a simple call of the <code>run</code> method the actions are performed.
|
||
Additionally, there is an option to skip actions which can be set in
|
||
the <code>add</code> method. This allows for easily configurable pipelines,
|
||
e.g., via a user configuration.</p></div>
|
||
<h3>Ancestors</h3>
|
||
<ul class="hlist">
|
||
<li><a title="lang_main.pipelines.base.BasePipeline" href="#lang_main.pipelines.base.BasePipeline">BasePipeline</a></li>
|
||
<li>abc.ABC</li>
|
||
</ul>
|
||
<h3>Methods</h3>
|
||
<dl>
|
||
<dt id="lang_main.pipelines.base.PipelineContainer.add"><code class="name flex">
|
||
<span>def <span class="ident">add</span></span>(<span>self, action: Callable, skip: bool = False) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@override
|
||
def add(
|
||
self,
|
||
action: Callable,
|
||
skip: bool = False,
|
||
) -> None:
|
||
if isinstance(action, Callable):
|
||
self.actions.append(action)
|
||
self.action_names.append(action.__name__)
|
||
self.action_skip.append(skip)
|
||
else:
|
||
self.panic_wrong_action_type(action=action, compatible_type=Callable.__name__)</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
<dt id="lang_main.pipelines.base.PipelineContainer.logic"><code class="name flex">
|
||
<span>def <span class="ident">logic</span></span>(<span>self) ‑> None</span>
|
||
</code></dt>
|
||
<dd>
|
||
<details class="source">
|
||
<summary>
|
||
<span>Expand source code</span>
|
||
</summary>
|
||
<pre><code class="python">@override
|
||
def logic(self) -> None:
|
||
for idx, (action, action_name) in enumerate(zip(self.actions, self.action_names)):
|
||
# loading
|
||
if self.action_skip[idx]:
|
||
logger.info('[No Calculation] Skipping >>%s<<...', action_name)
|
||
self.curr_proc_idx += 1
|
||
continue
|
||
# calculation
|
||
ret = action()
|
||
if ret is not None:
|
||
raise OutputInPipelineContainerError(
|
||
(
|
||
f'Output in PipelineContainers not allowed. Action {action_name} '
|
||
f'returned values in Container {self.name}.'
|
||
)
|
||
)
|
||
# processing tracking
|
||
self.curr_proc_idx += 1</code></pre>
|
||
</details>
|
||
<div class="desc"></div>
|
||
</dd>
|
||
</dl>
|
||
</dd>
|
||
</dl>
|
||
</section>
|
||
</article>
|
||
<nav id="sidebar">
|
||
<div class="toc">
|
||
<ul></ul>
|
||
</div>
|
||
<ul id="index">
|
||
<li><h3>Super-module</h3>
|
||
<ul>
|
||
<li><code><a title="lang_main.pipelines" href="index.html">lang_main.pipelines</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li><h3><a href="#header-classes">Classes</a></h3>
|
||
<ul>
|
||
<li>
|
||
<h4><code><a title="lang_main.pipelines.base.BasePipeline" href="#lang_main.pipelines.base.BasePipeline">BasePipeline</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="lang_main.pipelines.base.BasePipeline.add" href="#lang_main.pipelines.base.BasePipeline.add">add</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.BasePipeline.logic" href="#lang_main.pipelines.base.BasePipeline.logic">logic</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.BasePipeline.panic_wrong_action_type" href="#lang_main.pipelines.base.BasePipeline.panic_wrong_action_type">panic_wrong_action_type</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.BasePipeline.post_run" href="#lang_main.pipelines.base.BasePipeline.post_run">post_run</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.BasePipeline.prep_run" href="#lang_main.pipelines.base.BasePipeline.prep_run">prep_run</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.BasePipeline.run" href="#lang_main.pipelines.base.BasePipeline.run">run</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="lang_main.pipelines.base.Pipeline" href="#lang_main.pipelines.base.Pipeline">Pipeline</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="lang_main.pipelines.base.Pipeline.add" href="#lang_main.pipelines.base.Pipeline.add">add</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.Pipeline.get_result_path" href="#lang_main.pipelines.base.Pipeline.get_result_path">get_result_path</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.Pipeline.load_step" href="#lang_main.pipelines.base.Pipeline.load_step">load_step</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.Pipeline.logic" href="#lang_main.pipelines.base.Pipeline.logic">logic</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.Pipeline.save_step" href="#lang_main.pipelines.base.Pipeline.save_step">save_step</a></code></li>
|
||
</ul>
|
||
</li>
|
||
<li>
|
||
<h4><code><a title="lang_main.pipelines.base.PipelineContainer" href="#lang_main.pipelines.base.PipelineContainer">PipelineContainer</a></code></h4>
|
||
<ul class="">
|
||
<li><code><a title="lang_main.pipelines.base.PipelineContainer.add" href="#lang_main.pipelines.base.PipelineContainer.add">add</a></code></li>
|
||
<li><code><a title="lang_main.pipelines.base.PipelineContainer.logic" href="#lang_main.pipelines.base.PipelineContainer.logic">logic</a></code></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</nav>
|
||
</main>
|
||
<footer id="footer">
|
||
<p>Generated by <a href="https://pdoc3.github.io/pdoc" title="pdoc: Python API documentation generator"><cite>pdoc</cite> 0.11.5</a>.</p>
|
||
</footer>
|
||
</body>
|
||
</html>
|