generated from dopt-python/py311
basic database interaction
This commit is contained in:
@@ -9,6 +9,7 @@ import polars as pl
|
||||
import sqlalchemy as sql
|
||||
from sqlalchemy import Column, String, Table, TypeDecorator
|
||||
|
||||
from wce_crm import constants
|
||||
from wce_crm import types as t
|
||||
|
||||
|
||||
@@ -32,84 +33,109 @@ class SafeDateTime(TypeDecorator):
|
||||
return None
|
||||
|
||||
|
||||
class UTCDateTime(TypeDecorator):
|
||||
"""Safely coerces naive datetimes from SQLite into timezone-aware UTC."""
|
||||
|
||||
impl = sql.DateTime
|
||||
cache_ok = True
|
||||
|
||||
def process_bind_param(self, value, dialect):
|
||||
"""Runs when saving to the database."""
|
||||
if value is not None:
|
||||
# Ensure it's converted to UTC before saving
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=datetime.timezone.utc)
|
||||
else:
|
||||
value = value.astimezone(datetime.timezone.utc)
|
||||
return value
|
||||
|
||||
def process_result_value(self, value, dialect):
|
||||
"""Runs when fetching from the database."""
|
||||
if value is not None and value.tzinfo is None:
|
||||
# Explicitly tell Python this data *is* UTC
|
||||
value = value.replace(tzinfo=datetime.timezone.utc)
|
||||
return value
|
||||
|
||||
|
||||
md_crm = sql.MetaData()
|
||||
md_main = sql.MetaData()
|
||||
ENGINE = sql.create_engine(f"sqlite:///{constants.Config.DB_PATH_MAIN}")
|
||||
|
||||
# ---------- OLD "Kontaktliste" ----------
|
||||
|
||||
md_kontaktliste = sql.MetaData()
|
||||
# md_kontaktliste = sql.MetaData()
|
||||
|
||||
ext_kl_unternehmen: sql.Table = Table(
|
||||
"Unternehmen",
|
||||
md_kontaktliste,
|
||||
Column("u_id", sql.Integer, nullable=False, unique=True),
|
||||
Column("u_zeitstempel_eintrag", sql.DateTime, nullable=False),
|
||||
Column("u_rechtsform", sql.Text, nullable=False),
|
||||
Column("u_firmenname", sql.Text, nullable=False),
|
||||
Column("u_strasse", sql.Text, nullable=False),
|
||||
Column("u_hausnummer", sql.Text, nullable=False),
|
||||
Column("u_adresszusatz", sql.Text, nullable=True),
|
||||
Column("u_plz", sql.Text, nullable=False),
|
||||
Column("u_ort", sql.Text, nullable=False),
|
||||
Column("u_postfach", sql.Text, nullable=True),
|
||||
Column("u_website", sql.Text, nullable=True),
|
||||
Column("u_anrede", sql.Text, nullable=False),
|
||||
Column("u_titel", sql.Text, nullable=True),
|
||||
Column("u_vorname", sql.Text, nullable=False),
|
||||
Column("u_nachname", sql.Text, nullable=False),
|
||||
Column("u_funktion", sql.Text, nullable=False),
|
||||
Column("u_mail", sql.Text, nullable=False),
|
||||
Column("u_telefon", sql.Text, nullable=False),
|
||||
Column("u_plz_postfach", sql.Text, nullable=True),
|
||||
Column("u_einwilligung_inhaber", sql.Boolean, nullable=True),
|
||||
Column("u_einwilligung_ansprechpartner", sql.Boolean, nullable=True),
|
||||
Column("u_aktiv", sql.Boolean, nullable=False, default=1),
|
||||
)
|
||||
# ext_kl_unternehmen: sql.Table = Table(
|
||||
# "Unternehmen",
|
||||
# md_kontaktliste,
|
||||
# Column("u_id", sql.Integer, nullable=False, unique=True),
|
||||
# Column("u_zeitstempel_eintrag", sql.DateTime, nullable=False),
|
||||
# Column("u_rechtsform", sql.Text, nullable=False),
|
||||
# Column("u_firmenname", sql.Text, nullable=False),
|
||||
# Column("u_strasse", sql.Text, nullable=False),
|
||||
# Column("u_hausnummer", sql.Text, nullable=False),
|
||||
# Column("u_adresszusatz", sql.Text, nullable=True),
|
||||
# Column("u_plz", sql.Text, nullable=False),
|
||||
# Column("u_ort", sql.Text, nullable=False),
|
||||
# Column("u_postfach", sql.Text, nullable=True),
|
||||
# Column("u_website", sql.Text, nullable=True),
|
||||
# Column("u_anrede", sql.Text, nullable=False),
|
||||
# Column("u_titel", sql.Text, nullable=True),
|
||||
# Column("u_vorname", sql.Text, nullable=False),
|
||||
# Column("u_nachname", sql.Text, nullable=False),
|
||||
# Column("u_funktion", sql.Text, nullable=False),
|
||||
# Column("u_mail", sql.Text, nullable=False),
|
||||
# Column("u_telefon", sql.Text, nullable=False),
|
||||
# Column("u_plz_postfach", sql.Text, nullable=True),
|
||||
# Column("u_einwilligung_inhaber", sql.Boolean, nullable=True),
|
||||
# Column("u_einwilligung_ansprechpartner", sql.Boolean, nullable=True),
|
||||
# Column("u_aktiv", sql.Boolean, nullable=False, default=1),
|
||||
# )
|
||||
|
||||
ext_kl_unternehmen_schema: t.PolarsSchema = {
|
||||
"u_id": pl.UInt64,
|
||||
"u_zeitstempel_eintrag": pl.Datetime,
|
||||
"u_rechtsform": pl.String,
|
||||
"u_firmenname": pl.String,
|
||||
"u_strasse": pl.String,
|
||||
"u_hausnummer": pl.String,
|
||||
"u_adresszusatz": pl.String,
|
||||
"u_plz": pl.String,
|
||||
"u_ort": pl.String,
|
||||
"u_postfach": pl.String,
|
||||
"u_website": pl.String,
|
||||
"u_anrede": pl.String,
|
||||
"u_titel": pl.String,
|
||||
"u_vorname": pl.String,
|
||||
"u_nachname": pl.String,
|
||||
"u_funktion": pl.String,
|
||||
"u_mail": pl.String,
|
||||
"u_telefon": pl.String,
|
||||
"u_plz_postfach": pl.String,
|
||||
"u_einwilligung_inhaber": pl.Boolean,
|
||||
"u_einwilligung_ansprechpartner": pl.Boolean,
|
||||
"u_aktiv": pl.Boolean,
|
||||
}
|
||||
# ext_kl_unternehmen_schema: t.PolarsSchema = {
|
||||
# "u_id": pl.UInt64,
|
||||
# "u_zeitstempel_eintrag": pl.Datetime,
|
||||
# "u_rechtsform": pl.String,
|
||||
# "u_firmenname": pl.String,
|
||||
# "u_strasse": pl.String,
|
||||
# "u_hausnummer": pl.String,
|
||||
# "u_adresszusatz": pl.String,
|
||||
# "u_plz": pl.String,
|
||||
# "u_ort": pl.String,
|
||||
# "u_postfach": pl.String,
|
||||
# "u_website": pl.String,
|
||||
# "u_anrede": pl.String,
|
||||
# "u_titel": pl.String,
|
||||
# "u_vorname": pl.String,
|
||||
# "u_nachname": pl.String,
|
||||
# "u_funktion": pl.String,
|
||||
# "u_mail": pl.String,
|
||||
# "u_telefon": pl.String,
|
||||
# "u_plz_postfach": pl.String,
|
||||
# "u_einwilligung_inhaber": pl.Boolean,
|
||||
# "u_einwilligung_ansprechpartner": pl.Boolean,
|
||||
# "u_aktiv": pl.Boolean,
|
||||
# }
|
||||
|
||||
|
||||
def get_ext_kontaktliste(
|
||||
db_path: Path | None,
|
||||
) -> pl.DataFrame:
|
||||
if db_path is None:
|
||||
ENV_PTH = os.environ.get("DOPT_DB_KONTAKTLISTE", None)
|
||||
if ENV_PTH is None:
|
||||
raise ValueError("No database path provided or found as ENV var.")
|
||||
db_path = Path(ENV_PTH)
|
||||
# def get_ext_kontaktliste(
|
||||
# db_path: Path | None,
|
||||
# ) -> pl.DataFrame:
|
||||
# if db_path is None:
|
||||
# ENV_PTH = os.environ.get("DOPT_DB_KONTAKTLISTE", None)
|
||||
# if ENV_PTH is None:
|
||||
# raise ValueError("No database path provided or found as ENV var.")
|
||||
# db_path = Path(ENV_PTH)
|
||||
|
||||
if not db_path.exists():
|
||||
raise FileNotFoundError(f"Database not found under >{db_path}<")
|
||||
# if not db_path.exists():
|
||||
# raise FileNotFoundError(f"Database not found under >{db_path}<")
|
||||
|
||||
engine = sql.create_engine(f"sqlite:///{db_path}")
|
||||
stmt = sql.select(ext_kl_unternehmen)
|
||||
return pl.read_database(stmt, engine, schema_overrides=ext_kl_unternehmen_schema)
|
||||
# engine = sql.create_engine(f"sqlite:///{db_path}")
|
||||
# stmt = sql.select(ext_kl_unternehmen)
|
||||
# return pl.read_database(stmt, engine, schema_overrides=ext_kl_unternehmen_schema)
|
||||
|
||||
|
||||
df_kontaktliste = get_ext_kontaktliste(None)
|
||||
# df_kontaktliste = get_ext_kontaktliste(None)
|
||||
|
||||
# ----------------------------------------------------
|
||||
|
||||
@@ -188,7 +214,7 @@ def get_ext_crm_master(
|
||||
return pl.read_database(stmt, engine, schema_overrides=ext_crm_master_schema)
|
||||
|
||||
|
||||
df_crm_master = get_ext_crm_master(None)
|
||||
DF_CRM_MASTER = get_ext_crm_master(constants.Config.DB_PATH_CRM)
|
||||
|
||||
ext_crm_nutzer: sql.Table = Table(
|
||||
"Nutzer",
|
||||
@@ -287,7 +313,6 @@ def get_ext_crm_contact_person(
|
||||
engine = sql.create_engine(f"sqlite:///{db_path}")
|
||||
stmt = sql.select(ext_crm_contact_person)
|
||||
df = pl.read_database(stmt, engine, schema_overrides=ext_crm_contact_person_schema)
|
||||
# TODO Database seems to contain entries with invalid characters like \t or \n
|
||||
df = df.with_columns(
|
||||
pl.col(pl.String).str.replace_all(r"[\r\t\n]", " ").str.strip_chars(" ")
|
||||
)
|
||||
@@ -295,22 +320,27 @@ def get_ext_crm_contact_person(
|
||||
return df
|
||||
|
||||
|
||||
df_contact_person = get_ext_crm_contact_person(None)
|
||||
|
||||
# df_contact_person = get_ext_crm_contact_person(None)
|
||||
DF_CONTACT_PERSON = get_ext_crm_contact_person(constants.Config.DB_PATH_CRM)
|
||||
|
||||
grunderfassung_unternehmen: sql.Table = Table(
|
||||
"grunderfassung_unternehmen",
|
||||
md_main,
|
||||
Column("erfassung_id", sql.Integer, nullable=False, unique=True, autoincrement=True),
|
||||
Column(
|
||||
"erfassung_id",
|
||||
sql.Integer,
|
||||
primary_key=True,
|
||||
autoincrement=True,
|
||||
),
|
||||
Column(
|
||||
"Metadaten_erstellung",
|
||||
sql.DateTime(timezone=True),
|
||||
UTCDateTime,
|
||||
nullable=True,
|
||||
default=lambda: datetime.datetime.now(datetime.UTC),
|
||||
),
|
||||
Column(
|
||||
"Metadaten_aktualisierung",
|
||||
sql.DateTime(timezone=True),
|
||||
UTCDateTime,
|
||||
nullable=True,
|
||||
default=lambda: datetime.datetime.now(datetime.UTC),
|
||||
onupdate=lambda: datetime.datetime.now(datetime.UTC),
|
||||
@@ -344,6 +374,7 @@ grunderfassung_unternehmen: sql.Table = Table(
|
||||
Column("Stammdaten__anzahl_kinder__anzahl", sql.Text, nullable=True),
|
||||
Column("Stammdaten__aufenthaltsort", sql.Text, nullable=True),
|
||||
Column("Stammdaten__bundesland", sql.Text, nullable=True),
|
||||
Column("Stammdaten__land", sql.Text, nullable=True),
|
||||
Column("Stammdaten__email", sql.Text, nullable=True),
|
||||
Column("Stammdaten__familienstand", sql.Text, nullable=True),
|
||||
Column("Stammdaten__festnetznummer", sql.Text, nullable=True),
|
||||
@@ -364,3 +395,5 @@ grunderfassung_unternehmen: sql.Table = Table(
|
||||
Column("WeitereInfos__WI_gueltigkeit_aufenthaltstitel", sql.Date, nullable=True),
|
||||
Column("WeitereInfos__WI_meldung_institution", sql.Text, nullable=True),
|
||||
)
|
||||
|
||||
md_main.create_all(ENGINE)
|
||||
|
||||
Reference in New Issue
Block a user