     1|from __future__ import annotations
     2|
     3|import hashlib
     4|import json
     5|import re
     6|import uuid
     7|from dataclasses import dataclass
     8|from datetime import datetime, timezone
     9|from pathlib import Path
    10|from sqlite3 import Connection
    11|from typing import Any
    12|from collections import Counter
    13|
    14|from jarvis_finance.audit.log import record_audit_event
    15|
    16|SOURCE_PLATFORMS = {"PostFinance", "True Wealth", "Raiffeisen", "Manuell / Sonstige"}
    17|ACCOUNT_TYPES = {"brokerage", "cash", "robo_portfolio", "reserve", "other"}
    18|ASSET_CLASSES = {"equity", "etf", "cash", "other"}
    19|REVIEW_STATUSES = {"needs_manual_review", "missing_isin", "missing_ticker", "ambiguous"}
    20|
    21|
    22|def utc_now() -> str:
    23|    return datetime.now(timezone.utc).isoformat()
    24|
    25|
    26|def normalize_name(value: str | None) -> str:
    27|    text = (value or "").strip().lower()
    28|    text = re.sub(r"[^a-z0-9äöüéèàç]+", " ", text, flags=re.IGNORECASE)
    29|    return re.sub(r"\s+", " ", text).strip()
    30|
    31|
    32|def _json(data: dict[str, Any] | list[Any] | None) -> str:
    33|    return json.dumps(data or {}, ensure_ascii=False, sort_keys=True)
    34|
    35|
    36|def _hash_label(value: str | bytes | Path) -> str:
    37|    if isinstance(value, Path):
    38|        return hashlib.sha256(value.read_bytes()).hexdigest()
    39|    if isinstance(value, bytes):
    40|        return hashlib.sha256(value).hexdigest()
    41|    return hashlib.sha256(str(value).encode("utf-8")).hexdigest()
    42|
    43|
    44|def _first_matching_instrument(conn: Connection, *, isin: str | None, ticker: str | None, exchange: str | None, currency: str | None) -> str | None:
    45|    if isin:
    46|        row = conn.execute("SELECT instrument_id FROM instruments WHERE upper(isin)=upper(?) AND is_active=1 ORDER BY created_at LIMIT 1", (isin,)).fetchone()
    47|        if row:
    48|            return row["instrument_id"]
    49|    if ticker and exchange:
    50|        row = conn.execute(
    51|            """
    52|            SELECT instrument_id FROM instruments
    53|            WHERE upper(ticker)=upper(?) AND upper(COALESCE(exchange,''))=upper(?)
    54|              AND (? IS NULL OR upper(currency)=upper(?)) AND is_active=1
    55|            ORDER BY created_at LIMIT 1
    56|            """,
    57|            (ticker, exchange, currency, currency),
    58|        ).fetchone()
    59|        if row:
    60|            return row["instrument_id"]
    61|    return None
    62|
    63|
    64|def classify_instrument_mapping(
    65|    conn: Connection,
    66|    *,
    67|    asset_class: str,
    68|    source_label: str,
    69|    isin: str | None = None,
    70|    ticker: str | None = None,
    71|    exchange: str | None = None,
    72|    currency: str | None = None,
    73|) -> tuple[str, str, str | None, list[str]]:
    74|    flags: list[str] = []
    75|    asset = asset_class if asset_class in ASSET_CLASSES else "other"
    76|    instrument_id = _first_matching_instrument(conn, isin=isin, ticker=ticker, exchange=exchange, currency=currency)
    77|    if asset == "cash":
    78|        return "mapped", "0.90", instrument_id, flags
    79|    if asset in {"equity", "etf"}:
    80|        if isin and instrument_id:
    81|            return "mapped", "0.98", instrument_id, flags
    82|        if isin and not instrument_id:
    83|            flags.append("needs_manual_review")
    84|            return "needs_manual_review", "0.60", None, flags
    85|        flags.append("missing_isin")
    86|        if ticker and exchange and instrument_id:
    87|            flags.append("ticker_exchange_fallback_requires_review")
    88|            return "needs_manual_review", "0.75", instrument_id, flags
    89|        if not ticker:
    90|            flags.append("missing_ticker")
    91|        if source_label and not ticker:
    92|            flags.append("name_only_not_auto_mapped")
    93|        return "needs_manual_review", "0.20", None, flags
    94|    if not isin and not ticker:
    95|        flags.append("name_only_not_auto_mapped")
    96|        return "needs_manual_review", "0.10", None, flags
    97|    return "needs_manual_review", "0.40", instrument_id, flags
    98|
    99|
   100|def create_instrument_mapping(
   101|    conn: Connection,
   102|    *,
   103|    source_platform: str,
   104|    source_label: str,
   105|    asset_class: str = "other",
   106|    source_account: str | None = None,
   107|    isin: str | None = None,
   108|    ticker: str | None = None,
   109|    exchange: str | None = None,
   110|    currency: str | None = None,
   111|    notes: str | None = None,
   112|) -> dict[str, Any]:
   113|    normalized = normalize_name(source_label)
   114|    status, confidence, instrument_id, flags = classify_instrument_mapping(
   115|        conn,
   116|        asset_class=asset_class,
   117|        source_label=source_label,
   118|        isin=isin,
   119|        ticker=ticker,
   120|        exchange=exchange,
   121|        currency=currency,
   122|    )
   123|    mapping_id = str(uuid.uuid4())
   124|    now = utc_now()
   125|    conn.execute(
   126|        """
   127|        INSERT INTO instrument_mappings(
   128|            mapping_id, source_name, source_platform, source_account, source_label, normalized_name,
   129|            isin, ticker, exchange, currency, asset_class, source_instrument_name,
   130|            source_isin, source_ticker, source_exchange, source_currency, instrument_id,
   131|            mapping_status, confidence, quality_flags_json, notes, created_at, updated_at
   132|        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   133|        """,
   134|        (
   135|            mapping_id, source_platform, source_platform, source_account, source_label, normalized,
   136|            isin, ticker, exchange, currency, asset_class, source_label,
   137|            isin, ticker, exchange, currency, instrument_id,
   138|            status, confidence, _json(flags), notes, now, now,
   139|        ),
   140|    )
   141|    return {"mapping_id": mapping_id, "mapping_status": status, "confidence": confidence, "instrument_id": instrument_id, "quality_flags": flags}
   142|
   143|
   144|def confirm_instrument_mapping(
   145|    conn: Connection,
   146|    *,
   147|    mapping_id: str,
   148|    instrument_id: str,
   149|    note: str,
   150|    created_by: str = "manual_review",
   151|) -> str:
   152|    if not note or not note.strip():
   153|        raise ValueError("Manual mapping confirmation requires a note")
   154|    row = conn.execute("SELECT * FROM instrument_mappings WHERE mapping_id=?", (mapping_id,)).fetchone()
   155|    if not row:
   156|        raise ValueError("instrument mapping not found")
   157|    old = dict(row)
   158|    now = utc_now()
   159|    conn.execute(
   160|        "UPDATE instrument_mappings SET instrument_id=?, mapping_status='mapped', confidence='1.00', notes=?, updated_at=? WHERE mapping_id=?",
   161|        (instrument_id, note, now, mapping_id),
   162|    )
   163|    audit_id = record_audit_event(
   164|        conn,
   165|        source="manual_review_queue",
   166|        action="confirm_instrument_mapping",
   167|        entity_type="instrument_mapping",
   168|        entity_id=mapping_id,
   169|        old_values={"mapping_status": old.get("mapping_status"), "instrument_id": old.get("instrument_id")},
   170|        new_values={"mapping_status": "mapped", "instrument_id": instrument_id},
   171|        user_text_note=note,
   172|        confirmed=True,
   173|        created_by=created_by,
   174|    )
   175|    return audit_id
   176|
   177|
   178|def create_platform_account_mapping(
   179|    conn: Connection,
   180|    *,
   181|    source_platform: str,
   182|    source_account_label: str,
   183|    account_type: str = "other",
   184|    currency: str | None = None,
   185|    internal_platform_id: str | None = None,
   186|    internal_account_id: str | None = None,
   187|    notes: str | None = None,
   188|) -> dict[str, Any]:
   189|    if source_platform not in SOURCE_PLATFORMS:
   190|        raise ValueError("unsupported source_platform")
   191|    acct_type = account_type if account_type in ACCOUNT_TYPES else "other"
   192|    flags: list[str] = []
   193|    if source_platform == "Raiffeisen" and acct_type not in {"cash", "reserve", "other"}:
   194|        flags.append("raiffeisen_instrument_detail_missing")
   195|        acct_type = "cash"
   196|    status = "mapped" if internal_platform_id and internal_account_id else "needs_manual_review"
   197|    mapping_id = str(uuid.uuid4())
   198|    now = utc_now()
   199|    conn.execute(
   200|        """
   201|        INSERT INTO platform_account_mappings(
   202|            mapping_id, source_name, source_platform, source_platform_label, source_account_label,
   203|            normalized_platform, normalized_account_name, source_account_type, account_type,
   204|            source_currency, currency, platform_id, account_id, internal_platform_id,
   205|            internal_account_id, mapping_status, quality_flags_json, notes, created_at, updated_at
   206|        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   207|        """,
   208|        (
   209|            mapping_id, source_platform, source_platform, source_platform, source_account_label,
   210|            normalize_name(source_platform), normalize_name(source_account_label), acct_type, acct_type,
   211|            currency, currency, internal_platform_id, internal_account_id, internal_platform_id,
   212|            internal_account_id, status, _json(flags), notes, now, now,
   213|        ),
   214|    )
   215|    return {"mapping_id": mapping_id, "mapping_status": status, "account_type": acct_type, "quality_flags": flags}
   216|
   217|
   218|@dataclass(frozen=True)
   219|class DryRunInput:
   220|    source_platform: str
   221|    source_file_type: str
   222|    source_filename: str
   223|    detected_snapshot_date: str | None = None
   224|    snapshot_date_status: str = "missing"
   225|    rows_total: int = 0
   226|    candidate_positions: int = 0
   227|    candidate_cash_rows: int = 0
   228|    mapped_positions: int = 0
   229|    blocked_positions: int = 0
   230|    warnings_count: int = 0
   231|    errors_count: int = 0
   232|    quality_flags: dict[str, int] | None = None
   233|    summary: dict[str, Any] | None = None
   234|    detected_sections: list[str] | None = None
   235|
   236|
   237|def create_broker_import_dry_run(conn: Connection, item: DryRunInput) -> str:
   238|    dry_run_id = str(uuid.uuid4())
   239|    now = utc_now()
   240|    source_hash = _hash_label(item.source_filename)
   241|    summary = dict(item.summary or {})
   242|    summary.pop("rows", None)  # explicit guard: no raw rows in persisted summary
   243|    summary.pop("values", None)
   244|    conn.execute(
   245|        """
   246|        INSERT INTO broker_import_dry_runs(
   247|            dry_run_id, source_name, source_platform, source_file_label, source_file_type,
   248|            source_hash, source_filename_hash, detected_snapshot_date, snapshot_date_status,
   249|            detected_sections_json, field_coverage_json, rows_total, rows_position_candidates,
   250|            rows_cash_candidates, candidate_positions, candidate_cash_rows, mapped_positions,
   251|            blocked_positions, warnings_count, errors_count, quality_flags_json, summary_json,
   252|            created_at, notes
   253|        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   254|        """,
   255|        (
   256|            dry_run_id, item.source_platform, item.source_platform, "[runtime-file-redacted]", item.source_file_type,
   257|            source_hash, source_hash, item.detected_snapshot_date, item.snapshot_date_status,
   258|            _json(item.detected_sections or []), _json({}), item.rows_total, item.candidate_positions,
   259|            item.candidate_cash_rows, item.candidate_positions, item.candidate_cash_rows, item.mapped_positions,
   260|            item.blocked_positions, item.warnings_count, item.errors_count, _json(item.quality_flags or {}),
   261|            _json(summary), now, "dry-run summary only; no productive ledger writes",
   262|        ),
   263|    )
   264|    return dry_run_id
   265|
   266|
   267|
   268|def _row_hash(item: dict[str, Any]) -> str:
   269|    return hashlib.sha256(json.dumps(item, ensure_ascii=False, sort_keys=True, default=str).encode("utf-8")).hexdigest()
   270|
   271|
   272|def create_broker_import_review_item(
   273|    conn: Connection,
   274|    *,
   275|    dry_run_id: str,
   276|    source_platform: str,
   277|    source_file_type: str,
   278|    source_row_ref: str,
   279|    source_label: str,
   280|    detected_asset_class: str,
   281|    detected_currency: str | None = None,
   282|    detected_quantity_present: bool = False,
   283|    detected_market_value_present: bool = False,
   284|    isin: str | None = None,
   285|    ticker: str | None = None,
   286|    exchange: str | None = None,
   287|    mapped_instrument_id: str | None = None,
   288|    mapped_account_id: str | None = None,
   289|    quality_flags: list[str] | tuple[str, ...] | None = None,
   290|    review_status: str = "open",
   291|) -> str:
   292|    review_item_id = str(uuid.uuid4())
   293|    now = utc_now()
   294|    normalized = normalize_name(source_label)
   295|    row_hash = _row_hash({
   296|        "source_platform": source_platform, "source_row_ref": source_row_ref,
   297|        "source_label": source_label, "asset_class": detected_asset_class,
   298|        "currency": detected_currency, "isin": isin, "ticker": ticker, "exchange": exchange,
   299|    })
   300|    conn.execute(
   301|        """
   302|        INSERT INTO broker_import_review_items(
   303|            review_item_id, dry_run_id, source_platform, source_file_type, source_row_ref,
   304|            row_hash, source_label, normalized_name, detected_asset_class, detected_currency,
   305|            detected_quantity_present, detected_market_value_present, isin, ticker, exchange,
   306|            mapped_instrument_id, mapped_account_id, quality_flags_json, review_status,
   307|            reviewer_note, created_at, updated_at
   308|        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   309|        """,
   310|        (
   311|            review_item_id, dry_run_id, source_platform, source_file_type, source_row_ref,
   312|            row_hash, source_label, normalized, detected_asset_class, detected_currency,
   313|            1 if detected_quantity_present else 0, 1 if detected_market_value_present else 0,
   314|            isin, ticker, exchange, mapped_instrument_id, mapped_account_id,
   315|            _json(list(quality_flags or [])), review_status, None, now, now,
   316|        ),
   317|    )
   318|    return review_item_id
   319|
   320|
   321|def get_broker_review_items(conn: Connection, *, status: str | None = "open", limit: int = 100) -> list[dict[str, str]]:
   322|    params: list[Any] = []
   323|    where = ""
   324|    if status:
   325|        where = "WHERE review_status=?"
   326|        params.append(status)
   327|    params.append(limit)
   328|    rows = conn.execute(
   329|        f"""
   330|        SELECT review_item_id AS item_id, dry_run_id, source_platform, source_file_type,
   331|               source_row_ref, source_label, normalized_name, detected_asset_class,
   332|               detected_currency, detected_quantity_present, detected_market_value_present,
   333|               isin, ticker, exchange, mapped_instrument_id, mapped_account_id,
   334|               quality_flags_json, review_status, reviewer_note, created_at
   335|        FROM broker_import_review_items {where}
   336|        ORDER BY created_at DESC LIMIT ?
   337|        """,
   338|        tuple(params),
   339|    ).fetchall()
   340|    return [{k: "" if row[k] is None else str(row[k]) for k in row.keys()} for row in rows]
   341|
   342|
   343|def aggregate_review_quality_flags(conn: Connection) -> dict[str, int]:
   344|    flags: Counter[str] = Counter()
   345|    for row in conn.execute("SELECT quality_flags_json FROM broker_import_review_items WHERE review_status IN ('open','blocked')").fetchall():
   346|        try:
   347|            for flag in json.loads(row["quality_flags_json"] or "[]"):
   348|                flags[str(flag)] += 1
   349|        except json.JSONDecodeError:
   350|            flags["invalid_quality_flags"] += 1
   351|    return dict(flags)
   352|
   353|
   354|def run_broker_import_dry_run(conn: Connection, parsed, *, source_filename: str) -> dict[str, Any]:
   355|    mapped_positions = 0
   356|    blocked_positions = 0
   357|    review_items = 0
   358|    for candidate in parsed.candidates:
   359|        if candidate.detected_asset_class == "cash":
   360|            continue
   361|        status, _confidence, instrument_id, map_flags = classify_instrument_mapping(
   362|            conn,
   363|            asset_class=candidate.detected_asset_class,
   364|            source_label=candidate.source_label,
   365|            isin=candidate.isin,
   366|            ticker=candidate.ticker,
   367|            exchange=candidate.exchange,
   368|            currency=candidate.detected_currency,
   369|        )
   370|        if status == "mapped" and instrument_id:
   371|            mapped_positions += 1
   372|        else:
   373|            blocked_positions += 1
   374|    quality_summary = parsed.quality_flags_summary
   375|    dry_run_id = create_broker_import_dry_run(conn, DryRunInput(
   376|        source_platform=parsed.source_platform,
   377|        source_file_type=parsed.source_file_type,
   378|        source_filename=source_filename,
   379|        detected_snapshot_date=parsed.detected_snapshot_date,
   380|        snapshot_date_status=parsed.snapshot_date_status,
   381|        rows_total=parsed.rows_total,
   382|        candidate_positions=parsed.candidate_positions,
   383|        candidate_cash_rows=parsed.candidate_cash_rows,
   384|        mapped_positions=mapped_positions,
   385|        blocked_positions=blocked_positions,
   386|        warnings_count=sum(quality_summary.values()),
   387|        errors_count=len(parsed.errors),
   388|        quality_flags=quality_summary,
   389|        summary={
   390|            "source_platform": parsed.source_platform,
   391|            "source_file_type": parsed.source_file_type,
   392|            "rows_total": parsed.rows_total,
   393|            "candidate_positions": parsed.candidate_positions,
   394|            "candidate_cash_rows": parsed.candidate_cash_rows,
   395|            "mapped_positions": mapped_positions,
   396|            "blocked_positions": blocked_positions,
   397|        },
   398|        detected_sections=list(parsed.detected_sections),
   399|    ))
   400|    for candidate in parsed.candidates:
   401|        mapped_instrument_id = _first_matching_instrument(
   402|            conn, isin=candidate.isin, ticker=candidate.ticker,
   403|            exchange=candidate.exchange, currency=candidate.detected_currency,
   404|        )
   405|        create_broker_import_review_item(
   406|            conn,
   407|            dry_run_id=dry_run_id,
   408|            source_platform=candidate.source_platform,
   409|            source_file_type=candidate.source_file_type,
   410|            source_row_ref=candidate.source_row_ref,
   411|            source_label=candidate.source_label,
   412|            detected_asset_class=candidate.detected_asset_class,
   413|            detected_currency=candidate.detected_currency,
   414|            detected_quantity_present=candidate.detected_quantity_present,
   415|            detected_market_value_present=candidate.detected_market_value_present,
   416|            isin=candidate.isin,
   417|            ticker=candidate.ticker,
   418|            exchange=candidate.exchange,
   419|            mapped_instrument_id=mapped_instrument_id,
   420|            mapped_account_id=None,
   421|            quality_flags=candidate.quality_flags,
   422|            review_status="blocked" if "aggregate_only" in candidate.quality_flags else "open",
   423|        )
   424|        review_items += 1
   425|    return {
   426|        "dry_run_id": dry_run_id,
   427|        "rows_total": parsed.rows_total,
   428|        "candidate_positions": parsed.candidate_positions,
   429|        "candidate_cash_rows": parsed.candidate_cash_rows,
   430|        "mapped_positions": mapped_positions,
   431|        "blocked_positions": blocked_positions,
   432|        "review_items_created": review_items,
   433|        "warnings_count": sum(quality_summary.values()),
   434|        "errors_count": len(parsed.errors),
   435|        "quality_flags_summary": quality_summary,
   436|    }
   437|
   438|
   439|def confirm_review_item_mapping(conn: Connection, *, review_item_id: str, instrument_id: str | None = None, account_id: str | None = None, note: str, created_by: str = "manual_review") -> str:
   440|    if not note or not note.strip():
   441|        raise ValueError("Review item change requires a note")
   442|    row = conn.execute("SELECT * FROM broker_import_review_items WHERE review_item_id=?", (review_item_id,)).fetchone()
   443|    if not row:
   444|        raise ValueError("review item not found")
   445|    now = utc_now()
   446|    conn.execute(
   447|        "UPDATE broker_import_review_items SET mapped_instrument_id=COALESCE(?, mapped_instrument_id), mapped_account_id=COALESCE(?, mapped_account_id), review_status='mapped', reviewer_note=?, updated_at=? WHERE review_item_id=?",
   448|        (instrument_id, account_id, note, now, review_item_id),
   449|    )
   450|    return record_audit_event(
   451|        conn,
   452|        source="manual_review_queue",
   453|        action="confirm_review_item_mapping",
   454|        entity_type="broker_import_review_item",
   455|        entity_id=review_item_id,
   456|        old_values={"review_status": row["review_status"], "mapped_instrument_id": row["mapped_instrument_id"], "mapped_account_id": row["mapped_account_id"]},
   457|        new_values={"review_status": "mapped", "mapped_instrument_id": instrument_id, "mapped_account_id": account_id},
   458|        user_text_note=note,
   459|        confirmed=True,
   460|        created_by=created_by,
   461|    )
   462|
   463|def get_manual_review_queue(conn: Connection) -> list[dict[str, str]]:
   464|    rows: list[dict[str, str]] = []
   465|    for row in conn.execute(
   466|        """
   467|        SELECT 'broker_import_review_item' AS item_type, review_item_id AS item_id, source_platform,
   468|               mapped_account_id AS account_label, source_label AS label, review_status AS mapping_status,
   469|               quality_flags_json, reviewer_note AS notes, created_at
   470|        FROM broker_import_review_items
   471|        WHERE review_status IN ('open','blocked')
   472|        UNION ALL
   473|        SELECT 'instrument_mapping' AS item_type, mapping_id AS item_id, source_platform,
   474|               source_account AS account_label, source_label AS label, mapping_status,
   475|               quality_flags_json, notes, created_at
   476|        FROM instrument_mappings
   477|        WHERE mapping_status IN ('needs_manual_review','missing_isin','missing_ticker','ambiguous')
   478|        UNION ALL
   479|        SELECT 'platform_account_mapping' AS item_type, mapping_id AS item_id, source_platform,
   480|               source_account_label AS account_label, source_account_label AS label, mapping_status,
   481|               quality_flags_json, notes, created_at
   482|        FROM platform_account_mappings
   483|        WHERE mapping_status IN ('needs_manual_review','missing_isin','missing_ticker','ambiguous')
   484|        ORDER BY created_at DESC
   485|        """
   486|    ).fetchall():
   487|        rows.append({k: "" if row[k] is None else str(row[k]) for k in row.keys()})
   488|    return rows
   489|
   490|
   491|def build_import_wizard_summary(conn: Connection) -> dict[str, str]:
   492|    last = conn.execute("SELECT * FROM broker_import_dry_runs ORDER BY created_at DESC LIMIT 1").fetchone()
   493|    pending = len(get_manual_review_queue(conn))
   494|    return {
   495|        "mode": "read-only",
   496|        "productive_import_enabled": "false",
   497|        "pending_review_items": str(pending),
   498|        "open_runtime_review_items": str(conn.execute("SELECT COUNT(*) AS n FROM broker_import_review_items WHERE review_status IN ('open','blocked')").fetchone()["n"]),
   499|        "last_dry_run_source": last["source_platform"] if last else "",
   500|        "last_dry_run_status": last["snapshot_date_status"] if last else "none",
   501|

Use offset=501 to continue reading (showing 1-500 of 506 lines)


     1|from __future__ import annotations
     2|
     3|import hashlib
     4|from datetime import datetime, timezone
     5|from sqlite3 import Connection
     6|
     7|from .schema import INITIAL_SCHEMA_SQL
     8|
     9|MIGRATION_VERSION = 5
    10|MIGRATION_NAME = "005_broker_import_review_items"
    11|
    12|INSTRUMENT_OPTIONAL_COLUMNS = {
    13|    "position_category": "TEXT",
    14|    "ter": "TEXT",
    15|    "distribution_policy": "TEXT",
    16|    "index_name": "TEXT",
    17|    "fund_domicile": "TEXT",
    18|    "benchmark": "TEXT",
    19|}
    20|
    21|TEXT_AFFINITY_COLUMNS = {
    22|    "transactions": {"quantity", "price_original", "gross_amount_original", "fee_original", "tax_original", "net_amount_original", "fx_rate_to_chf", "gross_amount_chf", "fee_chf", "tax_chf", "net_amount_chf"},
    23|    "crypto_holdings": {"quantity", "legacy_snapshot_value_original", "legacy_snapshot_value_chf"},
    24|    "crypto_transactions": {"quantity", "price_original", "gross_amount_original", "fee_quantity", "fee_original", "fx_rate_to_chf", "amount_chf"},
    25|    "crypto_prices": {"price", "market_cap", "volume_24h", "change_24h_pct"},
    26|}
    27|
    28|
    29|def utc_now() -> str:
    30|    return datetime.now(timezone.utc).isoformat()
    31|
    32|
    33|def checksum_sql(sql: str) -> str:
    34|    return hashlib.sha256(sql.encode("utf-8")).hexdigest()
    35|
    36|
    37|def get_schema_version(conn: Connection) -> int:
    38|    row = conn.execute("SELECT MAX(version) AS version FROM schema_migrations").fetchone()
    39|    return int(row["version"] or 0) if row else 0
    40|
    41|
    42|def _table_columns(conn: Connection, table: str) -> dict[str, str]:
    43|    return {row["name"]: (row["type"] or "") for row in conn.execute(f"PRAGMA table_info({table})").fetchall()}
    44|
    45|
    46|def _add_missing_instrument_columns(conn: Connection) -> None:
    47|    existing = _table_columns(conn, "instruments")
    48|    for name, col_type in INSTRUMENT_OPTIONAL_COLUMNS.items():
    49|        if name not in existing:
    50|            conn.execute(f"ALTER TABLE instruments ADD COLUMN {name} {col_type}")
    51|
    52|
    53|def _rebuild_table_with_text_columns(conn: Connection, table: str, text_columns: set[str]) -> None:
    54|    cols = conn.execute(f"PRAGMA table_info({table})").fetchall()
    55|    if not cols:
    56|        return
    57|    needs_rebuild = any(row["name"] in text_columns and (row["type"] or "").upper() != "TEXT" for row in cols)
    58|    if not needs_rebuild:
    59|        return
    60|    tmp = f"{table}__text_migration"
    61|    col_defs: list[str] = []
    62|    pk_cols = [row["name"] for row in cols if row["pk"]]
    63|    for row in cols:
    64|        name = row["name"]
    65|        col_type = "TEXT" if name in text_columns else (row["type"] or "TEXT")
    66|        parts = [name, col_type]
    67|        if row["pk"] and len(pk_cols) == 1:
    68|            parts.append("PRIMARY KEY")
    69|        if row["notnull"]:
    70|            parts.append("NOT NULL")
    71|        if row["dflt_value"] is not None:
    72|            parts.append(f"DEFAULT {row['dflt_value']}")
    73|        col_defs.append(" ".join(parts))
    74|    if len(pk_cols) > 1:
    75|        col_defs.append("PRIMARY KEY (" + ", ".join(pk_cols) + ")")
    76|    conn.execute(f"DROP TABLE IF EXISTS {tmp}")
    77|    conn.execute(f"CREATE TABLE {tmp} (" + ", ".join(col_defs) + ")")
    78|    names = [row["name"] for row in cols]
    79|    select_exprs = [f"CAST({name} AS TEXT)" if name in text_columns else name for name in names]
    80|    conn.execute(f"INSERT INTO {tmp} ({', '.join(names)}) SELECT {', '.join(select_exprs)} FROM {table}")
    81|    conn.execute(f"DROP TABLE {table}")
    82|    conn.execute(f"ALTER TABLE {tmp} RENAME TO {table}")
    83|    if table == "crypto_holdings":
    84|        conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_crypto_holdings_asset_wallet ON crypto_holdings(asset_id, wallet_id)")
    85|
    86|
    87|def _add_missing_columns(conn: Connection, table: str, columns: dict[str, str]) -> None:
    88|    existing = _table_columns(conn, table)
    89|    for name, col_type in columns.items():
    90|        if name not in existing:
    91|            conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {col_type}")
    92|
    93|
    94|def _create_broker_bank_mapping_tables(conn: Connection) -> None:
    95|    conn.executescript(
    96|        """
    97|        CREATE TABLE IF NOT EXISTS instrument_mappings (
    98|            mapping_id TEXT PRIMARY KEY,
    99|            source_name TEXT,
   100|            source_platform TEXT NOT NULL,
   101|            source_account TEXT,
   102|            source_label TEXT NOT NULL,
   103|            normalized_name TEXT NOT NULL,
   104|            isin TEXT,
   105|            ticker TEXT,
   106|            exchange TEXT,
   107|            currency TEXT,
   108|            asset_class TEXT NOT NULL DEFAULT 'other',
   109|            source_instrument_name TEXT,
   110|            source_isin TEXT,
   111|            source_ticker TEXT,
   112|            source_exchange TEXT,
   113|            source_currency TEXT,
   114|            instrument_id TEXT REFERENCES instruments(instrument_id),
   115|            mapping_status TEXT NOT NULL DEFAULT 'needs_manual_review',
   116|            confidence TEXT NOT NULL DEFAULT '0',
   117|            quality_flags_json TEXT,
   118|            notes TEXT,
   119|            created_at TEXT NOT NULL,
   120|            updated_at TEXT
   121|        );
   122|
   123|        CREATE TABLE IF NOT EXISTS platform_account_mappings (
   124|            mapping_id TEXT PRIMARY KEY,
   125|            source_name TEXT,
   126|            source_platform TEXT NOT NULL,
   127|            source_platform_label TEXT,
   128|            source_account_label TEXT NOT NULL,
   129|            normalized_platform TEXT NOT NULL,
   130|            normalized_account_name TEXT NOT NULL,
   131|            source_account_type TEXT,
   132|            account_type TEXT NOT NULL DEFAULT 'other',
   133|            source_currency TEXT,
   134|            currency TEXT,
   135|            platform_id TEXT REFERENCES platforms(platform_id),
   136|            account_id TEXT REFERENCES accounts(account_id),
   137|            internal_platform_id TEXT REFERENCES platforms(platform_id),
   138|            internal_account_id TEXT REFERENCES accounts(account_id),
   139|            mapping_status TEXT NOT NULL DEFAULT 'needs_manual_review',
   140|            quality_flags_json TEXT,
   141|            notes TEXT,
   142|            created_at TEXT NOT NULL,
   143|            updated_at TEXT
   144|        );
   145|
   146|        CREATE TABLE IF NOT EXISTS broker_import_dry_runs (
   147|            dry_run_id TEXT PRIMARY KEY,
   148|            source_name TEXT,
   149|            source_platform TEXT NOT NULL,
   150|            source_file_label TEXT,
   151|            source_file_type TEXT NOT NULL,
   152|            source_hash TEXT,
   153|            source_filename_hash TEXT,
   154|            detected_snapshot_date TEXT,
   155|            snapshot_date_status TEXT NOT NULL DEFAULT 'missing',
   156|            detected_sections_json TEXT NOT NULL,
   157|            field_coverage_json TEXT,
   158|            rows_total INTEGER NOT NULL DEFAULT 0,
   159|            rows_position_candidates INTEGER NOT NULL DEFAULT 0,
   160|            rows_cash_candidates INTEGER NOT NULL DEFAULT 0,
   161|            candidate_positions INTEGER NOT NULL DEFAULT 0,
   162|            candidate_cash_rows INTEGER NOT NULL DEFAULT 0,
   163|            mapped_positions INTEGER NOT NULL DEFAULT 0,
   164|            blocked_positions INTEGER NOT NULL DEFAULT 0,
   165|            warnings_count INTEGER NOT NULL DEFAULT 0,
   166|            errors_count INTEGER NOT NULL DEFAULT 0,
   167|            quality_flags_json TEXT NOT NULL,
   168|            summary_json TEXT NOT NULL DEFAULT '{}',
   169|            created_at TEXT NOT NULL,
   170|            notes TEXT
   171|        );
   172|        """
   173|    )
   174|    _add_missing_columns(conn, "instrument_mappings", {
   175|        "source_name": "TEXT",
   176|        "source_platform": "TEXT",
   177|        "source_account": "TEXT",
   178|        "source_label": "TEXT",
   179|        "normalized_name": "TEXT",
   180|        "isin": "TEXT",
   181|        "ticker": "TEXT",
   182|        "exchange": "TEXT",
   183|        "currency": "TEXT",
   184|        "asset_class": "TEXT DEFAULT 'other'",
   185|        "confidence": "TEXT DEFAULT '0'",
   186|    })
   187|    _add_missing_columns(conn, "platform_account_mappings", {
   188|        "source_platform": "TEXT",
   189|        "normalized_platform": "TEXT",
   190|        "normalized_account_name": "TEXT",
   191|        "account_type": "TEXT DEFAULT 'other'",
   192|        "currency": "TEXT",
   193|        "internal_platform_id": "TEXT",
   194|        "internal_account_id": "TEXT",
   195|    })
   196|    _add_missing_columns(conn, "broker_import_dry_runs", {
   197|        "source_platform": "TEXT",
   198|        "source_filename_hash": "TEXT",
   199|        "detected_snapshot_date": "TEXT",
   200|        "snapshot_date_status": "TEXT DEFAULT 'missing'",
   201|        "candidate_positions": "INTEGER DEFAULT 0",
   202|        "candidate_cash_rows": "INTEGER DEFAULT 0",
   203|        "mapped_positions": "INTEGER DEFAULT 0",
   204|        "blocked_positions": "INTEGER DEFAULT 0",
   205|        "warnings_count": "INTEGER DEFAULT 0",
   206|        "errors_count": "INTEGER DEFAULT 0",
   207|        "summary_json": "TEXT DEFAULT '{}'",
   208|    })
   209|
   210|
   211|def _create_broker_import_review_items(conn: Connection) -> None:
   212|    conn.executescript(
   213|        """
   214|        CREATE TABLE IF NOT EXISTS broker_import_review_items (
   215|            review_item_id TEXT PRIMARY KEY,
   216|            dry_run_id TEXT NOT NULL REFERENCES broker_import_dry_runs(dry_run_id),
   217|            source_platform TEXT NOT NULL,
   218|            source_file_type TEXT NOT NULL,
   219|            source_row_ref TEXT NOT NULL,
   220|            row_hash TEXT NOT NULL,
   221|            source_label TEXT,
   222|            normalized_name TEXT,
   223|            detected_asset_class TEXT,
   224|            detected_currency TEXT,
   225|            detected_quantity_present INTEGER NOT NULL DEFAULT 0,
   226|            detected_market_value_present INTEGER NOT NULL DEFAULT 0,
   227|            isin TEXT,
   228|            ticker TEXT,
   229|            exchange TEXT,
   230|            mapped_instrument_id TEXT REFERENCES instruments(instrument_id),
   231|            mapped_account_id TEXT REFERENCES accounts(account_id),
   232|            quality_flags_json TEXT NOT NULL,
   233|            review_status TEXT NOT NULL DEFAULT 'open',
   234|            reviewer_note TEXT,
   235|            created_at TEXT NOT NULL,
   236|            updated_at TEXT
   237|        );
   238|        CREATE INDEX IF NOT EXISTS idx_broker_review_items_dry_run ON broker_import_review_items(dry_run_id);
   239|        CREATE INDEX IF NOT EXISTS idx_broker_review_items_status ON broker_import_review_items(review_status);
   240|        """
   241|    )
   242|
   243|def _apply_compat_migrations(conn: Connection) -> None:
   244|    _add_missing_instrument_columns(conn)
   245|    for table, text_columns in TEXT_AFFINITY_COLUMNS.items():
   246|        _rebuild_table_with_text_columns(conn, table, text_columns)
   247|    _create_broker_bank_mapping_tables(conn)
   248|    _create_broker_import_review_items(conn)
   249|
   250|
   251|def apply_migrations(conn: Connection) -> None:
   252|    conn.executescript(INITIAL_SCHEMA_SQL)
   253|    existing_initial = conn.execute("SELECT 1 FROM schema_migrations WHERE version = 1").fetchone()
   254|    if not existing_initial:
   255|        conn.execute(
   256|            "INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)",
   257|            (1, "001_initial_schema", utc_now(), checksum_sql(INITIAL_SCHEMA_SQL)),
   258|        )
   259|    _apply_compat_migrations(conn)
   260|    existing = conn.execute("SELECT 1 FROM schema_migrations WHERE version = ?", (MIGRATION_VERSION,)).fetchone()
   261|    if not existing:
   262|        conn.execute(
   263|            "INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)",
   264|            (MIGRATION_VERSION, MIGRATION_NAME, utc_now(), checksum_sql(MIGRATION_NAME)),
   265|        )
   266|    conn.commit()
   267|


     1|from __future__ import annotations
     2|
     3|import json
     4|from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
     5|from pathlib import Path
     6|from sqlite3 import Connection
     7|
     8|from jarvis_finance.config.settings import Settings
     9|from jarvis_finance.crypto.holdings import calculate_crypto_holdings
    10|from jarvis_finance.ledger.cash import calculate_cash_balances
    11|from jarvis_finance.ledger.positions import calculate_positions
    12|from jarvis_finance.market.providers import is_price_stale
    13|
    14|MONEY = Decimal("0.01")
    15|
    16|
    17|def d(value: object, default: str = "0") -> Decimal:
    18|    if value is None or str(value).strip() == "":
    19|        return Decimal(default)
    20|    try:
    21|        return Decimal(str(value))
    22|    except InvalidOperation:
    23|        return Decimal(default)
    24|
    25|
    26|def dec_text(value: Decimal | int | str | None, places: str | None = None) -> str:
    27|    if value is None:
    28|        return ""
    29|    value = d(value)
    30|    if places:
    31|        value = value.quantize(Decimal(places), rounding=ROUND_HALF_UP)
    32|    return format(value, "f")
    33|
    34|
    35|def money_text(value: Decimal | None, currency: str = "CHF") -> str:
    36|    if value is None:
    37|        return f"n/a {currency}"
    38|    return f"{dec_text(value, '0.01')} {currency}"
    39|
    40|
    41|def rowdicts(rows) -> list[dict[str, str]]:
    42|    return [{k: ("" if row[k] is None else str(row[k])) for k in row.keys()} for row in rows]
    43|
    44|
    45|def db_exists(path: str | Path) -> bool:
    46|    return Path(path).expanduser().exists()
    47|
    48|
    49|def runtime_outside_repo(settings: Settings, repo_root: Path) -> bool:
    50|    try:
    51|        settings.runtime_paths.base_dir.resolve().relative_to(repo_root.resolve())
    52|        return False
    53|    except ValueError:
    54|        return True
    55|
    56|
    57|def latest_crypto_price(conn: Connection, asset_id: str, currency: str = "CHF", *, max_age_seconds: int = 86_400) -> dict[str, str] | None:
    58|    row = conn.execute(
    59|        """
    60|        SELECT price, price_currency, fetched_at, provider_timestamp, quality_status, provider
    61|        FROM crypto_prices
    62|        WHERE asset_id=? AND price_currency=?
    63|        ORDER BY COALESCE(provider_timestamp, fetched_at, '') DESC, fetched_at DESC LIMIT 1
    64|        """,
    65|        (asset_id, currency.upper()),
    66|    ).fetchone()
    67|    if not row:
    68|        return None
    69|    item = dict(row)
    70|    ts = item.get("provider_timestamp") or item.get("fetched_at")
    71|    if item.get("quality_status") == "fresh" and is_price_stale(ts, max_age_seconds=max_age_seconds):
    72|        item["quality_status"] = "stale"
    73|    return item
    74|
    75|
    76|def get_crypto_overview(conn: Connection, *, price_max_age_seconds: int = 86_400) -> list[dict[str, str]]:
    77|    holdings = calculate_crypto_holdings(conn)
    78|    rows: list[dict[str, str]] = []
    79|    for asset_id, holding in sorted(holdings.total_by_asset.items()):
    80|        qty = holding.quantity
    81|        asset = conn.execute("SELECT * FROM crypto_assets WHERE asset_id=?", (asset_id,)).fetchone()
    82|        if not asset:
    83|            continue
    84|        price = latest_crypto_price(conn, asset_id, "CHF", max_age_seconds=price_max_age_seconds)
    85|        price_dec = d(price["price"]) if price and price["price"] else None
    86|        value = qty * price_dec if price_dec is not None else None
    87|        rows.append({
    88|            "asset_id": asset_id,
    89|            "coin_name": asset["coin_name"],
    90|            "symbol": asset["symbol"],
    91|            "coingecko_id": asset["coingecko_id"] or "",
    92|            "quantity": dec_text(qty),
    93|            "latest_price_chf": dec_text(price_dec, "0.01") if price_dec is not None else "",
    94|            "market_value_chf": dec_text(value, "0.01") if value is not None else "",
    95|            "price_quality": price["quality_status"] if price else "missing",
    96|            "fetched_at": price["fetched_at"] if price else "",
    97|        })
    98|    return rows
    99|
   100|
   101|def get_crypto_by_wallet(conn: Connection, *, price_max_age_seconds: int = 86_400) -> list[dict[str, str]]:
   102|    holdings = calculate_crypto_holdings(conn)
   103|    rows: list[dict[str, str]] = []
   104|    for (wallet_id, asset_id), holding in sorted(holdings.wallet_holdings.items()):
   105|        wallet = conn.execute("SELECT wallet_name, wallet_type, platform_provider FROM crypto_wallets WHERE wallet_id=?", (wallet_id,)).fetchone()
   106|        asset = conn.execute("SELECT coin_name, symbol, coingecko_id FROM crypto_assets WHERE asset_id=?", (asset_id,)).fetchone()
   107|        price = latest_crypto_price(conn, asset_id, "CHF", max_age_seconds=price_max_age_seconds)
   108|        price_dec = d(price["price"]) if price and price["price"] else None
   109|        value = holding.quantity * price_dec if price_dec is not None else None
   110|        rows.append({
   111|            "wallet_id": wallet_id,
   112|            "wallet_name": wallet["wallet_name"] if wallet else wallet_id,
   113|            "wallet_type": wallet["wallet_type"] if wallet else "",
   114|            "provider": wallet["platform_provider"] if wallet else "",
   115|            "coin_name": asset["coin_name"] if asset else asset_id,
   116|            "symbol": asset["symbol"] if asset else "",
   117|            "coingecko_id": asset["coingecko_id"] if asset else "",
   118|            "quantity": dec_text(holding.quantity),
   119|            "market_value_chf": dec_text(value, "0.01") if value is not None else "",
   120|            "wallet_value_chf": dec_text(value, "0.01") if value is not None else "",
   121|            "price_quality": price["quality_status"] if price else "missing",
   122|            "price_fetched_at": price["fetched_at"] if price else "",
   123|            "verification_status": holding.verification_status,
   124|            "last_verified_at": holding.last_verified_at or "",
   125|        })
   126|    return rows
   127|
   128|
   129|def get_wallets(conn: Connection) -> list[dict[str, str]]:
   130|    return rowdicts(conn.execute(
   131|        """
   132|        SELECT wallet_id, wallet_name, wallet_type, platform_provider AS provider,
   133|               network_chain, last_verified_at, is_active, notes
   134|        FROM crypto_wallets ORDER BY wallet_name
   135|        """
   136|    ).fetchall())
   137|
   138|
   139|def get_latest_crypto_transactions(conn: Connection, limit: int = 20) -> list[dict[str, str]]:
   140|    return rowdicts(conn.execute(
   141|        """
   142|        SELECT ct.transaction_datetime, ct.transaction_type, ca.symbol, ct.quantity, ct.fee_quantity,
   143|               fw.wallet_name AS from_wallet, tw.wallet_name AS to_wallet, ct.currency_original,
   144|               ct.amount_chf, ct.confirmation_status, ct.source
   145|        FROM crypto_transactions ct
   146|        LEFT JOIN crypto_assets ca ON ca.asset_id=ct.asset_id
   147|        LEFT JOIN crypto_wallets fw ON fw.wallet_id=ct.from_wallet_id
   148|        LEFT JOIN crypto_wallets tw ON tw.wallet_id=ct.to_wallet_id
   149|        ORDER BY ct.transaction_datetime DESC, ct.created_at DESC LIMIT ?
   150|        """, (limit,)
   151|    ).fetchall())
   152|
   153|
   154|def get_platform_overview(conn: Connection) -> list[dict[str, str]]:
   155|    return rowdicts(conn.execute(
   156|        """
   157|        SELECT p.name AS platform, p.platform_type, a.account_name, a.account_type, a.currency,
   158|               a.performance_included, a.is_active
   159|        FROM accounts a JOIN platforms p ON p.platform_id=a.platform_id
   160|        ORDER BY p.name, a.account_name
   161|        """
   162|    ).fetchall())
   163|
   164|
   165|def get_cash_overview(conn: Connection) -> list[dict[str, str]]:
   166|    result = calculate_cash_balances(conn)
   167|    rows: list[dict[str, str]] = []
   168|    for (account_id, currency), bal in sorted(result.balances.items()):
   169|        row = conn.execute(
   170|            "SELECT a.account_name, p.name AS platform FROM accounts a JOIN platforms p ON p.platform_id=a.platform_id WHERE a.account_id=?",
   171|            (account_id,),
   172|        ).fetchone()
   173|        rows.append({
   174|            "platform": row["platform"] if row else "",
   175|            "account_name": row["account_name"] if row else account_id,
   176|            "currency": currency,
   177|            "amount_original": dec_text(bal.amount_original, "0.01"),
   178|            "amount_chf": dec_text(bal.amount_chf, "0.01") if bal.amount_chf is not None else "",
   179|            "quality_status": bal.quality_status,
   180|        })
   181|    return rows
   182|
   183|
   184|def get_positions(conn: Connection) -> list[dict[str, str]]:
   185|    result = calculate_positions(conn)
   186|    rows: list[dict[str, str]] = []
   187|    for (account_id, instrument_id), pos in sorted(result.positions.items()):
   188|        row = conn.execute(
   189|            """
   190|            SELECT a.account_name, p.name AS platform, i.name, i.ticker, i.isin, i.asset_class, i.currency
   191|            FROM accounts a JOIN platforms p ON p.platform_id=a.platform_id
   192|            JOIN instruments i ON i.instrument_id=? WHERE a.account_id=?
   193|            """,
   194|            (instrument_id, account_id),
   195|        ).fetchone()
   196|        rows.append({
   197|            "platform": row["platform"] if row else "",
   198|            "account_name": row["account_name"] if row else account_id,
   199|            "asset_class": row["asset_class"] if row else "",
   200|            "name": row["name"] if row else instrument_id,
   201|            "ticker": row["ticker"] if row else "",
   202|            "isin": row["isin"] if row else "",
   203|            "quantity": dec_text(pos.quantity),
   204|            "cost_basis_chf": dec_text(pos.cost_basis_chf, "0.01"),
   205|            "market_value_chf": dec_text(pos.market_value_chf, "0.01") if pos.market_value_chf is not None else "",
   206|            "total_return_chf": dec_text(pos.total_return_chf, "0.01") if pos.total_return_chf is not None else "",
   207|            "data_quality_status": pos.data_quality_status,
   208|        })
   209|    return rows
   210|
   211|
   212|def get_ledger_transactions(conn: Connection, limit: int = 200) -> list[dict[str, str]]:
   213|    return rowdicts(conn.execute(
   214|        """
   215|        SELECT t.trade_date, p.name AS platform, a.account_name, t.transaction_type,
   216|               i.name AS instrument, i.ticker, t.quantity, t.gross_amount_original,
   217|               t.net_amount_original, t.currency_original, t.fx_status, t.quality_status,
   218|               t.source_type, t.source_id
   219|        FROM transactions t
   220|        LEFT JOIN accounts a ON a.account_id=t.account_id
   221|        LEFT JOIN platforms p ON p.platform_id=a.platform_id
   222|        LEFT JOIN instruments i ON i.instrument_id=t.instrument_id
   223|        ORDER BY t.trade_date DESC, t.created_at DESC LIMIT ?
   224|        """, (limit,)
   225|    ).fetchall())
   226|
   227|
   228|def get_watchlist(conn: Connection) -> list[dict[str, str]]:
   229|    return rowdicts(conn.execute("SELECT name, asset_class, reason, status, target_entry_price, target_entry_currency, investment_case, bear_case, next_review_date FROM watchlist ORDER BY name").fetchall())
   230|
   231|
   232|def get_alerts(conn: Connection) -> list[dict[str, str]]:
   233|    return rowdicts(conn.execute(
   234|        """
   235|        SELECT priority, status, category, entity_type, entity_id, rule_id, message,
   236|               occurrence_count, last_seen_at, resolved_at, muted_until, fingerprint, dedup_key, created_at
   237|        FROM alerts
   238|        ORDER BY CASE priority WHEN 'kritisch' THEN 0 WHEN 'warnung' THEN 1 ELSE 2 END, status, created_at DESC
   239|        """
   240|    ).fetchall())
   241|
   242|
   243|def get_audit_events(conn: Connection, limit: int = 200) -> list[dict[str, str]]:
   244|    rows = conn.execute(
   245|        """
   246|        SELECT timestamp, source, action, entity_type, entity_id, old_values_json, new_values_json,
   247|               confirmed, created_by, quality_status
   248|        FROM audit_log ORDER BY timestamp DESC LIMIT ?
   249|        """, (limit,)
   250|    ).fetchall()
   251|    out: list[dict[str, str]] = []
   252|    for row in rows:
   253|        item = {k: ("" if row[k] is None else str(row[k])) for k in row.keys()}
   254|        for key in ("old_values_json", "new_values_json"):
   255|            if item[key]:
   256|                try:
   257|                    item[key] = json.dumps(json.loads(item[key]), ensure_ascii=False, indent=2, sort_keys=True)
   258|                except json.JSONDecodeError:
   259|                    pass
   260|        out.append(item)
   261|    return out
   262|
   263|
   264|def get_data_quality(conn: Connection) -> list[dict[str, str]]:
   265|    items: list[dict[str, str]] = []
   266|    checks = [
   267|        ("fehlende FX", "SELECT COUNT(*) AS c FROM transactions WHERE fx_status='missing'", "kritisch"),
   268|        ("fehlende CoinGecko-ID", "SELECT COUNT(*) AS c FROM crypto_assets WHERE coingecko_id IS NULL OR coingecko_id='' ", "warnung"),
   269|        ("fehlende Crypto-Preise", "SELECT COUNT(*) AS c FROM crypto_assets ca LEFT JOIN crypto_prices cp ON cp.asset_id=ca.asset_id WHERE cp.crypto_price_id IS NULL", "warnung"),
   270|        ("fehlende ISIN", "SELECT COUNT(*) AS c FROM instrument_mappings WHERE asset_class IN ('equity','etf') AND (isin IS NULL OR isin='') AND mapping_status!='ignored'", "warnung"),
   271|        ("fehlende FX in Import-Reviews", "SELECT COUNT(*) AS c FROM instrument_mappings WHERE quality_flags_json LIKE '%missing_fx%' AND mapping_status!='ignored'", "warnung"),
   272|        ("snapshot_only Positionen", "SELECT COUNT(*) AS c FROM instrument_mappings WHERE quality_flags_json LIKE '%snapshot_only%' AND mapping_status!='ignored'", "info"),
   273|        ("cost_basis_uncertain", "SELECT COUNT(*) AS c FROM instrument_mappings WHERE quality_flags_json LIKE '%cost_basis_uncertain%' AND mapping_status!='ignored'", "warnung"),
   274|        ("aktive Alerts", "SELECT COUNT(*) AS c FROM alerts WHERE status='active'", "warnung"),
   275|        ("resolved Alerts", "SELECT COUNT(*) AS c FROM alerts WHERE status='resolved'", "ok"),
   276|        ("alte aktive Alerts", "SELECT COUNT(*) AS c FROM alerts WHERE status='active' AND last_seen_at < datetime('now', '-7 days')", "info"),
   277|    ]
   278|    for label, sql, severity in checks:
   279|        row = conn.execute(sql).fetchone()
   280|        count = int(row["c"] or 0) if row else 0
   281|        sev = severity if count else "ok"
   282|        items.append({"check": label, "count": str(count), "severity": sev})
   283|    for flag_row in get_review_quality_flag_counts(conn):
   284|        count = int(flag_row["open_items"] or 0)
   285|        items.append({"check": f"Review Flag: {flag_row['quality_flag']}", "count": str(count), "severity": "warnung" if count else "ok"})
   286|    return items
   287|
   288|
   289|def get_import_wizard_summary(conn: Connection) -> dict[str, str]:
   290|    from jarvis_finance.imports.broker_mapping import build_import_wizard_summary
   291|
   292|    return build_import_wizard_summary(conn)
   293|
   294|
   295|def get_manual_review_items(conn: Connection) -> list[dict[str, str]]:
   296|    from jarvis_finance.imports.broker_mapping import get_manual_review_queue
   297|
   298|    return get_manual_review_queue(conn)
   299|
   300|
   301|def get_broker_dry_runs(conn: Connection, limit: int = 20) -> list[dict[str, str]]:
   302|    return rowdicts(conn.execute(
   303|        """
   304|        SELECT source_platform, source_file_type, detected_snapshot_date, snapshot_date_status,
   305|               rows_total, candidate_positions, candidate_cash_rows, mapped_positions,
   306|               blocked_positions, warnings_count, errors_count, quality_flags_json, created_at
   307|        FROM broker_import_dry_runs ORDER BY created_at DESC LIMIT ?
   308|        """, (limit,)
   309|    ).fetchall())
   310|
   311|
   312|def get_safe_mode_status(conn: Connection) -> dict[str, str]:
   313|    backup = conn.execute("SELECT created_at FROM audit_log WHERE action LIKE '%backup%' ORDER BY created_at DESC LIMIT 1").fetchone()
   314|    return {
   315|        "db_mode": "Runtime",
   316|        "write_mode": "Read-only",
   317|        "last_backup_at": backup["created_at"] if backup else "nicht verfügbar",
   318|        "git_safety_status": "zuletzt OK / CLI prüfen",
   319|        "data_location": "Echte Daten liegen außerhalb Git",
   320|    }
   321|
   322|
   323|def get_broker_review_items(conn: Connection, limit: int = 100) -> list[dict[str, str]]:
   324|    from jarvis_finance.imports.broker_mapping import get_broker_review_items as _items
   325|
   326|    return _items(conn, status=None, limit=limit)
   327|
   328|
   329|def get_review_quality_flag_counts(conn: Connection) -> list[dict[str, str]]:
   330|    from jarvis_finance.imports.broker_mapping import aggregate_review_quality_flags
   331|
   332|    return [{"quality_flag": k, "open_items": str(v)} for k, v in sorted(aggregate_review_quality_flags(conn).items())]
   333|
   334|
   335|def get_command_center_summary(conn: Connection) -> dict[str, str]:
   336|    cash_rows = get_cash_overview(conn)
   337|    crypto_rows = get_crypto_overview(conn)
   338|    cash_total = sum(d(r["amount_chf"]) for r in cash_rows if r["amount_chf"])
   339|    crypto_total = sum(d(r["market_value_chf"]) for r in crypto_rows if r["market_value_chf"])
   340|    alert_counts = conn.execute("SELECT priority, COUNT(*) AS c FROM alerts WHERE status='active' GROUP BY priority").fetchall()
   341|    quality = get_data_quality(conn)
   342|    return {
   343|        "base_currency": "CHF",
   344|        "cash_total_chf": dec_text(cash_total, "0.01"),
   345|        "crypto_total_chf": dec_text(crypto_total, "0.01"),
   346|        "total_value_chf": dec_text(cash_total + crypto_total, "0.01"),
   347|        "active_alerts": ", ".join(f"{r['priority']}: {r['c']}" for r in alert_counts) or "0",
   348|        "data_quality_status": "kritisch" if any(i["severity"] == "kritisch" and i["count"] != "0" for i in quality) else ("warnung" if any(i["severity"] == "warnung" and i["count"] != "0" for i in quality) else "ok"),
   349|        "crypto_price_data_as_of": max((r["fetched_at"] for r in crypto_rows if r.get("fetched_at")), default=""),
   350|    }
   351|


     1|from __future__ import annotations
     2|
     3|import zipfile
     4|from pathlib import Path
     5|
     6|from jarvis_finance.dashboard import data as dashboard_data
     7|from jarvis_finance.imports.broker_mapping import assert_write_enabled, confirm_review_item_mapping, get_broker_review_items, run_broker_import_dry_run
     8|from jarvis_finance.imports.broker_parsers import parse_postfinance_docx, parse_raiffeisen_xlsx, parse_true_wealth_docx
     9|from jarvis_finance.storage.database import connect_memory
    10|from jarvis_finance.storage.migrations import apply_migrations, utc_now
    11|
    12|
    13|def _docx(path: Path, rows: list[list[str]]) -> None:
    14|    def tc(text: str) -> str:
    15|        return f"<w:tc><w:p><w:r><w:t>{text}</w:t></w:r></w:p></w:tc>"
    16|    trs = "".join("<w:tr>" + "".join(tc(c) for c in row) + "</w:tr>" for row in rows)
    17|    xml = f'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    18|<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:tbl>{trs}</w:tbl></w:body></w:document>'''
    19|    with zipfile.ZipFile(path, "w") as zf:
    20|        zf.writestr("[Content_Types].xml", "")
    21|        zf.writestr("word/document.xml", xml)
    22|
    23|
    24|def _xlsx(path: Path, rows: list[list[str]]) -> None:
    25|    shared: list[str] = []
    26|    index: dict[str, int] = {}
    27|    sheet_rows = []
    28|    for r_idx, row in enumerate(rows, start=1):
    29|        cells = []
    30|        for c_idx, value in enumerate(row):
    31|            if value not in index:
    32|                index[value] = len(shared); shared.append(value)
    33|            ref = chr(ord('A') + c_idx) + str(r_idx)
    34|            cells.append(f'<c r="{ref}" t="s"><v>{index[value]}</v></c>')
    35|        sheet_rows.append(f'<row r="{r_idx}">' + ''.join(cells) + '</row>')
    36|    shared_xml = '<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">' + ''.join(f'<si><t>{s}</t></si>' for s in shared) + '</sst>'
    37|    sheet_xml = '<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>' + ''.join(sheet_rows) + '</sheetData></worksheet>'
    38|    with zipfile.ZipFile(path, "w") as zf:
    39|        zf.writestr("xl/sharedStrings.xml", shared_xml)
    40|        zf.writestr("xl/worksheets/sheet1.xml", sheet_xml)
    41|        zf.writestr("[Content_Types].xml", "")
    42|
    43|
    44|def conn():
    45|    c = connect_memory()
    46|    apply_migrations(c)
    47|    return c
    48|
    49|
    50|def seed_instrument(c):
    51|    c.execute("INSERT INTO instruments(instrument_id, asset_class, name, ticker, isin, exchange, currency, created_at) VALUES ('inst-1','etf','Mapped ETF','METF','CH0000000001','SIX','CHF',?)", (utc_now(),))
    52|
    53|
    54|def test_synthetic_postfinance_parser_recognizes_candidates_and_quality_flags(tmp_path: Path) -> None:
    55|    path = tmp_path / "synthetic_postfinance.docx"
    56|    _docx(path, [["Produkt", "Menge", "Währung", "Marktwert", "Stichtag 31.12.2025"], ["Synthetic ETF", "Menge vorhanden", "CHF", "Marktwert vorhanden", "Einstand legacy"], ["Cash Konto", "", "CHF", "", ""]])
    57|    parsed = parse_postfinance_docx(path)
    58|    assert parsed.source_platform == "PostFinance"
    59|    assert parsed.candidate_positions == 1
    60|    assert parsed.candidate_cash_rows == 1
    61|    flags = parsed.quality_flags_summary
    62|    assert flags["missing_isin"] == 1
    63|    assert flags["missing_ticker"] == 1
    64|    assert flags["market_value_legacy"] == 1
    65|
    66|
    67|def test_synthetic_true_wealth_parser_recognizes_candidates_and_missing_fx(tmp_path: Path) -> None:
    68|    path = tmp_path / "synthetic_true_wealth.docx"
    69|    _docx(path, [["Position", "Anteile", "Währung", "Wert", "Stichtag 31.12.2025"], ["Global ETF", "Anteile vorhanden", "USD", "Wert vorhanden", "Portfolio"], ["Cash USD", "", "USD", "Wert vorhanden", ""]])
    70|    parsed = parse_true_wealth_docx(path)
    71|    assert parsed.candidate_positions == 1
    72|    assert parsed.candidate_cash_rows == 1
    73|    assert parsed.quality_flags_summary["missing_fx"] == 2
    74|    assert parsed.quality_flags_summary["cost_basis_uncertain"] == 1
    75|
    76|
    77|def test_synthetic_raiffeisen_parser_recognizes_cash_and_aggregate_control_snapshot(tmp_path: Path) -> None:
    78|    path = tmp_path / "synthetic_raiffeisen.xlsx"
    79|    _xlsx(path, [["Label", "Typ", "Währung", "Saldo", "Datum 31.12.2025"], ["Privatkonto", "Konto", "CHF", "Saldo vorhanden", ""], ["Depot Anlagen gesamt", "Anlage", "CHF", "Wert vorhanden", ""]])
    80|    parsed = parse_raiffeisen_xlsx(path)
    81|    assert parsed.candidate_cash_rows == 1
    82|    assert parsed.candidate_positions == 1
    83|    assert parsed.quality_flags_summary["cash_snapshot_candidate"] == 1
    84|    assert parsed.quality_flags_summary["aggregate_only"] == 1
    85|    assert parsed.quality_flags_summary["missing_instrument_details"] == 1
    86|
    87|
    88|def test_dry_run_pipeline_writes_summary_and_review_items_without_productive_positions(tmp_path: Path) -> None:
    89|    path = tmp_path / "synthetic_postfinance.docx"
    90|    _docx(path, [["Produkt", "Menge", "Währung", "Marktwert"], ["Name Only ETF", "Menge vorhanden", "CHF", "Marktwert vorhanden"], ["Cash Konto", "", "CHF", ""]])
    91|    c = conn()
    92|    summary = run_broker_import_dry_run(c, parse_postfinance_docx(path), source_filename=path.name)
    93|    assert summary["rows_total"] == 3
    94|    assert summary["review_items_created"] == 2
    95|    assert summary["blocked_positions"] == 1
    96|    assert c.execute("SELECT COUNT(*) AS n FROM broker_import_dry_runs").fetchone()["n"] == 1
    97|    assert c.execute("SELECT COUNT(*) AS n FROM broker_import_review_items").fetchone()["n"] == 2
    98|    assert c.execute("SELECT COUNT(*) AS n FROM positions_snapshot").fetchone()["n"] == 0
    99|
   100|
   101|def test_name_without_isin_stays_needs_manual_review_via_review_item(tmp_path: Path) -> None:
   102|    path = tmp_path / "synthetic_postfinance.docx"
   103|    _docx(path, [["Produkt", "Menge", "Währung"], ["Name Only ETF", "Menge vorhanden", "CHF"]])
   104|    c = conn()
   105|    run_broker_import_dry_run(c, parse_postfinance_docx(path), source_filename=path.name)
   106|    item = c.execute("SELECT * FROM broker_import_review_items").fetchone()
   107|    assert item["review_status"] == "open"
   108|    assert "missing_isin" in item["quality_flags_json"]
   109|
   110|
   111|def test_raiffeisen_aggregate_only_is_blocked_not_productive_position(tmp_path: Path) -> None:
   112|    path = tmp_path / "synthetic_raiffeisen.xlsx"
   113|    _xlsx(path, [["Label", "Typ", "Währung"], ["Depot Anlagen gesamt", "Anlage", "CHF"]])
   114|    c = conn()
   115|    run_broker_import_dry_run(c, parse_raiffeisen_xlsx(path), source_filename=path.name)
   116|    row = c.execute("SELECT review_status, quality_flags_json FROM broker_import_review_items").fetchone()
   117|    assert row["review_status"] == "blocked"
   118|    assert "aggregate_only" in row["quality_flags_json"]
   119|    assert c.execute("SELECT COUNT(*) AS n FROM positions_snapshot").fetchone()["n"] == 0
   120|
   121|
   122|def test_manual_review_queue_and_data_quality_aggregate_review_flags(tmp_path: Path) -> None:
   123|    path = tmp_path / "synthetic_true_wealth.docx"
   124|    _docx(path, [["Position", "Währung"], ["Global ETF", "USD"]])
   125|    c = conn()
   126|    run_broker_import_dry_run(c, parse_true_wealth_docx(path), source_filename=path.name)
   127|    assert get_broker_review_items(c)
   128|    dq = {r["check"]: r["count"] for r in dashboard_data.get_data_quality(c)}
   129|    assert dq["Review Flag: missing_isin"] == "1"
   130|    assert dq["Review Flag: missing_fx"] == "1"
   131|
   132|
   133|def test_data_quality_includes_blocked_aggregate_only_review_flags(tmp_path: Path) -> None:
   134|    path = tmp_path / "synthetic_raiffeisen.xlsx"
   135|    _xlsx(path, [["Label", "Typ", "Währung"], ["Depot Anlagen gesamt", "Anlage", "CHF"]])
   136|    c = conn()
   137|    run_broker_import_dry_run(c, parse_raiffeisen_xlsx(path), source_filename=path.name)
   138|    dq = {r["check"]: r["count"] for r in dashboard_data.get_data_quality(c)}
   139|    assert dq["Review Flag: aggregate_only"] == "1"
   140|    assert dq["Review Flag: missing_instrument_details"] == "1"
   141|
   142|
   143|def test_safe_mode_default_read_only_and_commit_disabled() -> None:
   144|    c = conn()
   145|    status = dashboard_data.get_safe_mode_status(c)
   146|    assert status["write_mode"] == "Read-only"
   147|    try:
   148|        assert_write_enabled(write_enabled=False)
   149|    except PermissionError:
   150|        pass
   151|    else:
   152|        raise AssertionError("read-only commit guard did not block")
   153|
   154|
   155|def test_review_item_mapping_confirmation_writes_audit(tmp_path: Path) -> None:
   156|    path = tmp_path / "synthetic_postfinance.docx"
   157|    _docx(path, [["Produkt", "Menge", "Währung"], ["Mapped ETF", "Menge vorhanden", "CHF"]])
   158|    c = conn(); seed_instrument(c)
   159|    run_broker_import_dry_run(c, parse_postfinance_docx(path), source_filename=path.name)
   160|    review_item_id = c.execute("SELECT review_item_id FROM broker_import_review_items").fetchone()["review_item_id"]
   161|    audit_id = confirm_review_item_mapping(c, review_item_id=review_item_id, instrument_id="inst-1", note="Synthetic B2 confirmation")
   162|    assert audit_id
   163|    assert c.execute("SELECT COUNT(*) AS n FROM audit_log WHERE entity_id=?", (review_item_id,)).fetchone()["n"] == 1
   164|
   165|
   166|def test_no_real_broker_files_in_repo() -> None:
   167|    repo = Path('/home/agent/.hermes/repos/FinanceManager')
   168|    forbidden = [p for p in repo.rglob('*') if p.suffix.lower() in {'.xlsx', '.docx', '.pdf'} and 'tests' not in p.parts and 'examples' not in p.parts]
   169|    assert forbidden == []
   170|


     1|from __future__ import annotations
     2|
     3|import json
     4|import uuid
     5|from datetime import datetime, timezone
     6|from sqlite3 import Connection
     7|from typing import Any
     8|
     9|
    10|def utc_now() -> str:
    11|    return datetime.now(timezone.utc).isoformat()
    12|
    13|
    14|def record_audit_event(
    15|    conn: Connection,
    16|    *,
    17|    source: str,
    18|    action: str,
    19|    entity_type: str,
    20|    entity_id: str,
    21|    old_values: dict[str, Any] | None = None,
    22|    new_values: dict[str, Any] | None = None,
    23|    user_text_note: str | None = None,
    24|    confirmed: bool = True,
    25|    created_by: str = "system",
    26|) -> str:
    27|    audit_id = str(uuid.uuid4())
    28|    now = utc_now()
    29|    conn.execute(
    30|        """
    31|        INSERT INTO audit_log(
    32|            audit_id, timestamp, source, action, entity_type, entity_id,
    33|            old_values_json, new_values_json, user_text_note, confirmed,
    34|            confirmation_timestamp, created_by, created_at
    35|        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    36|        """,
    37|        (
    38|            audit_id,
    39|            now,
    40|            source,
    41|            action,
    42|            entity_type,
    43|            entity_id,
    44|            json.dumps(old_values or {}, sort_keys=True),
    45|            json.dumps(new_values or {}, sort_keys=True),
    46|            user_text_note,
    47|            1 if confirmed else 0,
    48|            now if confirmed else None,
    49|            created_by,
    50|            now,
    51|        ),
    52|    )
    53|    return audit_id
    54|


     1|from __future__ import annotations
     2|
     3|from jarvis_finance.dashboard import data
     4|from jarvis_finance.dashboard.components.tables import show_table
     5|
     6|PAGE_TITLE = "Manual Review Queue"
     7|
     8|
     9|def render(st, conn) -> None:
    10|    st.title("Manual Review Queue")
    11|    st.caption("Zeilenbezogene Runtime-Review-Items plus offene Instrument-/Account-Mappings. Keine produktive Positionserstellung in Phase B2.")
    12|
    13|    st.subheader("Offene Runtime Review Items")
    14|    show_table(st, data.get_broker_review_items(conn, limit=100))
    15|
    16|    st.subheader("Mapping-/Account-Probleme")
    17|    show_table(st, data.get_manual_review_items(conn))
    18|
    19|    st.subheader("Vorbereitete Aktionen")
    20|    st.write("ISIN ergänzen, Ticker ergänzen, Exchange ergänzen, Instrument Mapping bestätigen, Account Mapping bestätigen, Ignore with note, Mark resolved.")
    21|    st.warning("Alle Änderungen erfordern Audit-Log. Produktive Positionserstellung bleibt deaktiviert.")
    22|    st.button("ISIN/Ticker/Exchange speichern", disabled=True)
    23|    st.button("Instrument Mapping bestätigen", disabled=True)
    24|    st.button("Account Mapping bestätigen", disabled=True)
    25|    st.button("Ignore with note", disabled=True)
    26|    st.button("Mark resolved", disabled=True)
    27|


     1|from __future__ import annotations
     2|
     3|from jarvis_finance.dashboard import data
     4|from jarvis_finance.dashboard.components.tables import show_table
     5|
     6|PAGE_TITLE = "Data Quality Center"
     7|
     8|
     9|def render(st, conn) -> None:
    10|    st.title("Data Quality Center")
    11|    st.caption("MVP-Übersicht für Alerts, Preise, FX, ISIN, Parser-/Review-Flags und resolved/fixed Alerts.")
    12|    st.subheader("Data Quality Checks")
    13|    show_table(st, data.get_data_quality(conn))
    14|    st.subheader("Offene Review Items nach Quality Flag")
    15|    show_table(st, data.get_review_quality_flag_counts(conn))
    16|    st.subheader("Alerts")
    17|    show_table(st, data.get_alerts(conn))
    18|    st.subheader("Sichere Aktionen")
    19|    st.button("Data Quality Recheck", disabled=True)
    20|    st.button("Resolve Fixed Alerts", disabled=True)
    21|    st.info("Produktive Commit-Aktionen bleiben deaktiviert; Recheck/Resolve bleiben kontrolliert über CLI/Bestätigung.")
    22|


     1|from __future__ import annotations
     2|
     3|from jarvis_finance.dashboard import data
     4|from jarvis_finance.dashboard.components.tables import show_table
     5|from jarvis_finance.imports.broker_mapping import assert_write_enabled
     6|
     7|PAGE_TITLE = "Import Wizard"
     8|
     9|
    10|def render(st, conn) -> None:
    11|    st.title("Import Wizard")
    12|    st.caption("Phase B2: synthetische Parser-Dry-Runs, Review Items und Quality Flags — kein produktiver Aktien-/ETF-/Cash-Import.")
    13|
    14|    st.info("Modus: Read-only. Produktive Commit-Aktionen sind standardmäßig deaktiviert.")
    15|    st.warning("Echte Broker-/Bankdateien nur Runtime/Temp verarbeiten. Keine Dateien ins Repo kopieren.")
    16|
    17|    source = st.selectbox("Quelle", ["PostFinance", "True Wealth", "Raiffeisen", "Manuell / Sonstige"])
    18|    uploaded = st.file_uploader("Datei auswählen — wird nicht ins Repo geschrieben", type=["docx", "xlsx", "csv", "pdf"])
    19|    demo = st.button("Synthetischen Demo-Dry-Run ausführen", disabled=True)
    20|    st.write({"source": source, "file_type": getattr(uploaded, "type", "") if uploaded else "", "demo_dry_run_available_in_tests": True, "write_enabled": False})
    21|
    22|    st.subheader("Parser-Ergebnis / Dry-Run Summary")
    23|    show_table(st, data.get_broker_dry_runs(conn))
    24|
    25|    st.subheader("Quality Flags")
    26|    show_table(st, data.get_review_quality_flag_counts(conn))
    27|
    28|    st.subheader("Review Items")
    29|    show_table(st, data.get_broker_review_items(conn, limit=50))
    30|
    31|    st.subheader("Wizard Status")
    32|    st.write(data.get_import_wizard_summary(conn))
    33|    st.page_link("pages/page_15_manual_review_queue.py", label="Zur Manual Review Queue") if hasattr(st, "page_link") else st.caption("Manual Review Queue über Navigation öffnen.")
    34|
    35|    st.subheader("Commit")
    36|    try:
    37|        assert_write_enabled(write_enabled=False)
    38|    except PermissionError as exc:
    39|        st.error(str(exc))
    40|    st.button("Produktiven Import committen", disabled=True)
    41|


Explanation:
**Planning implementation steps**

I need to focus on implementing the necessary changes by writing targeted functions and tests. I’m considering adding schema columns for readiness status, reviewer confirmation, snapshot date confirmation, ticker exchange confirmation, and dry run status, along with designations like is_current and archived. Since compat _add columns seem sufficient, I might bump the migration version to 6. I need to ensure everything is tested and adjust the execute_code for more complex patches carefully to avoid any syntax errors. Let's get on with patching migrations!

from pathlib import Path
p=Path('/home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py')
s=p.read_text()
s=s.replace('MIGRATION_VERSION = 5\nMIGRATION_NAME = "005_broker_import_review_items"','MIGRATION_VERSION = 6\nMIGRATION_NAME = "006_broker_review_queue_hardening"')
s=s.replace('"summary_json": "TEXT DEFAULT \'{}\'",\n    })','"summary_json": "TEXT DEFAULT \'{}\'",\n        "session_status": "TEXT DEFAULT \'active\'",\n        "is_current": "INTEGER DEFAULT 0",\n        "archived_at": "TEXT",\n        "discarded_at": "TEXT",\n    })')
s=s.replace('    _add_missing_columns(conn, "broker_import_dry_runs", {\n', '    _add_missing_columns(conn, "broker_import_dry_runs", {\n')
s=s.replace('    conn.executescript(\n        """\n        CREATE TABLE IF NOT EXISTS broker_import_review_items (', '    conn.executescript(\n        """\n        CREATE TABLE IF NOT EXISTS broker_import_review_items (')
# add new columns via _add_missing_columns after create script block
old='''        """\n    )\n\ndef _apply_compat_migrations(conn: Connection) -> None:'''
new='''        """\n    )\n    _add_missing_columns(conn, "broker_import_review_items", {\n        "import_readiness_status": "TEXT DEFAULT 'not_ready'",\n        "reviewer_confirmed": "INTEGER DEFAULT 0",\n        "snapshot_date_confirmed": "INTEGER DEFAULT 0",\n        "ticker_exchange_confirmed": "INTEGER DEFAULT 0",\n        "account_mapping_status": "TEXT DEFAULT 'missing'",\n    })\n\ndef _apply_compat_migrations(conn: Connection) -> None:'''
if old not in s:
    raise SystemExit('anchor not found')
s=s.replace(old,new)
p.write_text(s)
print('migrations patched')


Output: migrations patched


   190|        "normalized_account_name": "TEXT",
   191|        "account_type": "TEXT DEFAULT 'other'",
   192|        "currency": "TEXT",
   193|        "internal_platform_id": "TEXT",
   194|        "internal_account_id": "TEXT",
   195|    })
   196|    _add_missing_columns(conn, "broker_import_dry_runs", {
   197|        "source_platform": "TEXT",
   198|        "source_filename_hash": "TEXT",
   199|        "detected_snapshot_date": "TEXT",
   200|        "snapshot_date_status": "TEXT DEFAULT 'missing'",
   201|        "candidate_positions": "INTEGER DEFAULT 0",
   202|        "candidate_cash_rows": "INTEGER DEFAULT 0",
   203|        "mapped_positions": "INTEGER DEFAULT 0",
   204|        "blocked_positions": "INTEGER DEFAULT 0",
   205|        "warnings_count": "INTEGER DEFAULT 0",
   206|        "errors_count": "INTEGER DEFAULT 0",
   207|        "summary_json": "TEXT DEFAULT '{}'",
   208|        "session_status": "TEXT DEFAULT 'active'",
   209|        "is_current": "INTEGER DEFAULT 0",
   210|        "archived_at": "TEXT",
   211|        "discarded_at": "TEXT",
   212|    })
   213|
   214|
   215|def _create_broker_import_review_items(conn: Connection) -> None:
   216|    conn.executescript(
   217|        """
   218|        CREATE TABLE IF NOT EXISTS broker_import_review_items (
   219|            review_item_id TEXT PRIMARY KEY,
   220|            dry_run_id TEXT NOT NULL REFERENCES broker_import_dry_runs(dry_run_id),
   221|            source_platform TEXT NOT NULL,
   222|            source_file_type TEXT NOT NULL,
   223|            source_row_ref TEXT NOT NULL,
   224|            row_hash TEXT NOT NULL,
   225|            source_label TEXT,
   226|            normalized_name TEXT,
   227|            detected_asset_class TEXT,
   228|            detected_currency TEXT,
   229|            detected_quantity_present INTEGER NOT NULL DEFAULT 0,
   230|            detected_market_value_present INTEGER NOT NULL DEFAULT 0,
   231|            isin TEXT,
   232|            ticker TEXT,
   233|            exchange TEXT,
   234|            mapped_instrument_id TEXT REFERENCES instruments(instrument_id),
   235|            mapped_account_id TEXT REFERENCES accounts(account_id),
   236|            quality_flags_json TEXT NOT NULL,
   237|            review_status TEXT NOT NULL DEFAULT 'open',
   238|            reviewer_note TEXT,
   239|            created_at TEXT NOT NULL,
   240|            updated_at TEXT
   241|        );
   242|        CREATE INDEX IF NOT EXISTS idx_broker_review_items_dry_run ON broker_import_review_items(dry_run_id);
   243|        CREATE INDEX IF NOT EXISTS idx_broker_review_items_status ON broker_import_review_items(review_status);
   244|        """
   245|    )
   246|    _add_missing_columns(conn, "broker_import_review_items", {
   247|        "import_readiness_status": "TEXT DEFAULT 'not_ready'",
   248|        "reviewer_confirmed": "INTEGER DEFAULT 0",
   249|        "snapshot_date_confirmed": "INTEGER DEFAULT 0",
   250|        "ticker_exchange_confirmed": "INTEGER DEFAULT 0",
   251|        "account_mapping_status": "TEXT DEFAULT 'missing'",
   252|    })
   253|
   254|def _apply_compat_migrations(conn: Connection) -> None:
   255|    _add_missing_instrument_columns(conn)
   256|    for table, text_columns in TEXT_AFFINITY_COLUMNS.items():
   257|        _rebuild_table_with_text_columns(conn, table, text_columns)
   258|    _create_broker_bank_mapping_tables(conn)
   259|    _create_broker_import_review_items(conn)
   260|
   261|
   262|def apply_migrations(conn: Connection) -> None:
   263|    conn.executescript(INITIAL_SCHEMA_SQL)
   264|    existing_initial = conn.execute("SELECT 1 FROM schema_migrations WHERE version = 1").fetchone()
   265|    if not existing_initial:
   266|        conn.execute(
   267|            "INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)",
   268|            (1, "001_initial_schema", utc_now(), checksum_sql(INITIAL_SCHEMA_SQL)),
   269|        )
   270|

Use offset=270 to continue reading (showing 190-269 of 277 lines)


Explanation:
**Planning function creation**

I need to create some functions, starting with an easy append in the broker mapping. Then, I have to develop an update function that includes audit capabilities, filters, and readiness checks. I might also need to incorporate accounts and instruments, so I'll inspect the schema to ensure I have all the necessary fields for accounts and instruments. That way, everything aligns well and works smoothly. Let's get this organized!

CREATE TABLE IF NOT EXISTS (instruments|accounts|platforms|audit_log)


     1|INITIAL_SCHEMA_SQL = "\nCREATE TABLE IF NOT EXISTS schema_migrations (\n    version INTEGER PRIMARY KEY,\n    name TEXT NOT NULL,\n    applied_at TEXT NOT NULL,\n    checksum TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS platforms (\n    platform_id TEXT PRIMARY KEY,\n    name TEXT NOT NULL UNIQUE,\n    platform_type TEXT NOT NULL,\n    country TEXT,\n    default_currency TEXT NOT NULL DEFAULT 'CHF',\n    is_active INTEGER NOT NULL DEFAULT 1,\n    notes TEXT,\n    created_at TEXT NOT NULL,\n    updated_at TEXT\n);\n\nCREATE TABLE IF NOT EXISTS accounts (\n    account_id TEXT PRIMARY KEY,\n    platform_id TEXT NOT NULL REFERENCES platforms(platform_id),\n    account_name TEXT NOT NULL,\n    account_type TEXT NOT NULL,\n    currency TEXT NOT NULL DEFAULT 'CHF',\n    performance_included INTEGER NOT NULL DEFAULT 1,\n    is_health_reserve INTEGER NOT NULL DEFAULT 0,\n    target_cash_min_chf NUMERIC,\n    target_cash_max_chf NUMERIC,\n    is_active INTEGER NOT NULL DEFAULT 1,\n    notes TEXT,\n    created_at TEXT NOT NULL,\n    updated_at TEXT,\n    UNIQUE(platform_id, account_name)\n);\n\nCREATE TABLE IF NOT EXISTS instruments (\n    instrument_id TEXT PRIMARY KEY,\n    asset_class TEXT NOT NULL,\n    name TEXT NOT NULL,\n    ticker TEXT,\n    isin TEXT,\n    exchange TEXT,\n    currency TEXT NOT NULL,\n    country TEXT,\n    sector TEXT,\n    industry TEXT,\n    provider_symbol TEXT,\n    data_provider_primary TEXT,\n    position_category TEXT,\n    ter TEXT,\n    distribution_policy TEXT,\n    index_name TEXT,\n    fund_domicile TEXT,\n    benchmark TEXT,\n    data_provider_fallback TEXT,\n    is_active INTEGER NOT NULL DEFAULT 1,\n    notes TEXT,\n    created_at TEXT NOT NULL,\n    updated_at TEXT\n);\n\nCREATE TABLE IF NOT EXISTS transactions (\n    transaction_id TEXT PRIMARY KEY,\n    transaction_type TEXT NOT NULL,\n    account_id TEXT NOT NULL REFERENCES accounts(account_id),\n    instrument_id TEXT REFERENCES instruments(instrument_id),\n    trade_date TEXT N... [truncated]
     2|
     3|REQUIRED_TABLES = [
     4|    "schema_migrations", "platforms", "accounts", "instruments", "transactions",
     5|    "positions_snapshot", "cash_balances", "fx_rates", "market_prices",
     6|    "crypto_wallets", "crypto_assets", "crypto_holdings", "crypto_transactions",
     7|    "crypto_prices", "watchlist", "reports", "alerts", "audit_log",
     8|    "import_sessions", "decision_journal", "instrument_mappings",
     9|    "platform_account_mappings", "broker_import_dry_runs",
    10|]
    11|