from __future__ import annotations

import hashlib
import json
import re
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from sqlite3 import Connection
from typing import Any

from jarvis_finance.audit.log import record_audit_event

SOURCE_PLATFORMS = {"PostFinance", "True Wealth", "Raiffeisen", "Manuell / Sonstige"}
ACCOUNT_TYPES = {"brokerage", "cash", "robo_portfolio", "reserve", "other"}
ASSET_CLASSES = {"equity", "etf", "cash", "other"}
REVIEW_STATUSES = {"needs_manual_review", "missing_isin", "missing_ticker", "ambiguous"}

def utc_now() -> str:
    return datetime.now(timezone.utc).isoformat()

def normalize_name(value: str | None) -> str:
    text = (value or "").strip().lower()
    text = re.sub(r"[^a-z0-9äöüéèàç]+", " ", text, flags=re.IGNORECASE)
    return re.sub(r"\s+", " ", text).strip()

def _json(data: dict[str, Any] | list[Any] | None) -> str:
    return json.dumps(data or {}, ensure_ascii=False, sort_keys=True)

def _hash_label(value: str | bytes | Path) -> str:
    if isinstance(value, Path):
        return hashlib.sha256(value.read_bytes()).hexdigest()
    if isinstance(value, bytes):
        return hashlib.sha256(value).hexdigest()
    return hashlib.sha256(str(value).encode("utf-8")).hexdigest()

def _first_matching_instrument(conn: Connection, *, isin: str | None, ticker: str | None, exchange: str | None, currency: str | None) -> str | None:
    if isin:
        row = conn.execute("SELECT instrument_id FROM instruments WHERE upper(isin)=upper(?) AND is_active=1 ORDER BY created_at LIMIT 1", (isin,)).fetchone()
        if row:
            return row["instrument_id"]
    if ticker and exchange:
        row = conn.execute(
            """
            SELECT instrument_id FROM instruments
            WHERE upper(ticker)=upper(?) AND upper(COALESCE(exchange,''))=upper(?)
              AND (? IS NULL OR upper(currency)=upper(?)) AND is_active=1
            ORDER BY created_at LIMIT 1
            """,
            (ticker, exchange, currency, currency),
        ).fetchone()
        if row:
            return row["instrument_id"]
    return None

def classify_instrument_mapping(
    conn: Connection,
    *,
    asset_class: str,
    source_label: str,
    isin: str | None = None,
    ticker: str | None = None,
    exchange: str | None = None,
    currency: str | None = None,
) -> tuple[str, str, str | None, list[str]]:
    flags: list[str] = []
    asset = asset_class if asset_class in ASSET_CLASSES else "other"
    instrument_id = _first_matching_instrument(conn, isin=isin, ticker=ticker, exchange=exchange, currency=currency)
    if asset == "cash":
        return "mapped", "0.90", instrument_id, flags
    if asset in {"equity", "etf"}:
        if isin and instrument_id:
            return "mapped", "0.98", instrument_id, flags
        if isin and not instrument_id:
            flags.append("needs_manual_review")
            return "needs_manual_review", "0.60", None, flags
        flags.append("missing_isin")
        if ticker and exchange and instrument_id:
            flags.append("ticker_exchange_fallback_requires_review")
            return "needs_manual_review", "0.75", instrument_id, flags
        if not ticker:
            flags.append("missing_ticker")
        if source_label and not ticker:
            flags.append("name_only_not_auto_mapped")
        return "needs_manual_review", "0.20", None, flags
    if not isin and not ticker:
        flags.append("name_only_not_auto_mapped")
        return "needs_manual_review", "0.10", None, flags
    return "needs_manual_review", "0.40", instrument_id, flags

def create_instrument_mapping(
    conn: Connection,
    *,
    source_platform: str,
    source_label: str,
    asset_class: str = "other",
    source_account: str | None = None,
    isin: str | None = None,
    ticker: str | None = None,
    exchange: str | None = None,
    currency: str | None = None,
    notes: str | None = None,
) -> dict[str, Any]:
    normalized = normalize_name(source_label)
    status, confidence, instrument_id, flags = classify_instrument_mapping(
        conn,
        asset_class=asset_class,
        source_label=source_label,
        isin=isin,
        ticker=ticker,
        exchange=exchange,
        currency=currency,
    )
    mapping_id = str(uuid.uuid4())
    now = utc_now()
    conn.execute(
        """
        INSERT INTO instrument_mappings(
            mapping_id, source_name, source_platform, source_account, source_label, normalized_name,
            isin, ticker, exchange, currency, asset_class, source_instrument_name,
            source_isin, source_ticker, source_exchange, source_currency, instrument_id,
            mapping_status, confidence, quality_flags_json, notes, created_at, updated_at
        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
        (
            mapping_id, source_platform, source_platform, source_account, source_label, normalized,
            isin, ticker, exchange, currency, asset_class, source_label,
            isin, ticker, exchange, currency, instrument_id,
            status, confidence, _json(flags), notes, now, now,
        ),
    )
    return {"mapping_id": mapping_id, "mapping_status": status, "confidence": confidence, "instrument_id": instrument_id, "quality_flags": flags}

def confirm_instrument_mapping(
    conn: Connection,
    *,
    mapping_id: str,
    instrument_id: str,
    note: str,
    created_by: str = "manual_review",
) -> str:
    if not note or not note.strip():
        raise ValueError("Manual mapping confirmation requires a note")
    row = conn.execute("SELECT * FROM instrument_mappings WHERE mapping_id=?", (mapping_id,)).fetchone()
    if not row:
        raise ValueError("instrument mapping not found")
    old = dict(row)
    now = utc_now()
    conn.execute(
        "UPDATE instrument_mappings SET instrument_id=?, mapping_status='mapped', confidence='1.00', notes=?, updated_at=? WHERE mapping_id=?",
        (instrument_id, note, now, mapping_id),
    )
    audit_id = record_audit_event(
        conn,
        source="manual_review_queue",
        action="confirm_instrument_mapping",
        entity_type="instrument_mapping",
        entity_id=mapping_id,
        old_values={"mapping_status": old.get("mapping_status"), "instrument_id": old.get("instrument_id")},
        new_values={"mapping_status": "mapped", "instrument_id": instrument_id},
        user_text_note=note,
        confirmed=True,
        created_by=created_by,
    )
    return audit_id

def create_platform_account_mapping(
    conn: Connection,
    *,
    source_platform: str,
    source_account_label: str,
    account_type: str = "other",
    currency: str | None = None,
    internal_platform_id: str | None = None,
    internal_account_id: str | None = None,
    notes: str | None = None,
) -> dict[str, Any]:
    if source_platform not in SOURCE_PLATFORMS:
        raise ValueError("unsupported source_platform")
    acct_type = account_type if account_type in ACCOUNT_TYPES else "other"
    flags: list[str] = []
    if source_platform == "Raiffeisen" and acct_type not in {"cash", "reserve", "other"}:
        flags.append("raiffeisen_instrument_detail_missing")
        acct_type = "cash"
    status = "mapped" if internal_platform_id and internal_account_id else "needs_manual_review"
    mapping_id = str(uuid.uuid4())
    now = utc_now()
    conn.execute(
        """
        INSERT INTO platform_account_mappings(
            mapping_id, source_name, source_platform, source_platform_label, source_account_label,
            normalized_platform, normalized_account_name, source_account_type, account_type,
            source_currency, currency, platform_id, account_id, internal_platform_id,
            internal_account_id, mapping_status, quality_flags_json, notes, created_at, updated_at
        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
        (
            mapping_id, source_platform, source_platform, source_platform, source_account_label,
            normalize_name(source_platform), normalize_name(source_account_label), acct_type, acct_type,
            currency, currency, internal_platform_id, internal_account_id, internal_platform_id,
            internal_account_id, status, _json(flags), notes, now, now,
        ),
    )
    return {"mapping_id": mapping_id, "mapping_status": status, "account_type": acct_type, "quality_flags": flags}

@dataclass(frozen=True)
class DryRunInput:
    source_platform: str
    source_file_type: str
    source_filename: str
    detected_snapshot_date: str | None = None
    snapshot_date_status: str = "missing"
    rows_total: int = 0
    candidate_positions: int = 0
    candidate_cash_rows: int = 0
    mapped_positions: int = 0
    blocked_positions: int = 0
    warnings_count: int = 0
    errors_count: int = 0
    quality_flags: dict[str, int] | None = None
    summary: dict[str, Any] | None = None
    detected_sections: list[str] | None = None

def create_broker_import_dry_run(conn: Connection, item: DryRunInput) -> str:
    dry_run_id = str(uuid.uuid4())
    now = utc_now()
    source_hash = _hash_label(item.source_filename)
    summary = dict(item.summary or {})
    summary.pop("rows", None)  # explicit guard: no raw rows in persisted summary
    summary.pop("values", None)
    conn.execute(
        """
        INSERT INTO broker_import_dry_runs(
            dry_run_id, source_name, source_platform, source_file_label, source_file_type,
            source_hash, source_filename_hash, detected_snapshot_date, snapshot_date_status,
            detected_sections_json, field_coverage_json, rows_total, rows_position_candidates,
            rows_cash_candidates, candidate_positions, candidate_cash_rows, mapped_positions,
            blocked_positions, warnings_count, errors_count, quality_flags_json, summary_json,
            created_at, notes
        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """,
        (
            dry_run_id, item.source_platform, item.source_platform, "[runtime-file-redacted]", item.source_file_type,
            source_hash, source_hash, item.detected_snapshot_date, item.snapshot_date_status,
            _json(item.detected_sections or []), _json({}), item.rows_total, item.candidate_positions,
            item.candidate_cash_rows, item.candidate_positions, item.candidate_cash_rows, item.mapped_positions,
            item.blocked_positions, item.warnings_count, item.errors_count, _json(item.quality_flags or {}),
            _json(summary), now, "dry-run summary only; no productive ledger writes",
        ),
    )
    return dry_run_id

def get_manual_review_queue(conn: Connection) -> list[dict[str, str]]:
    rows: list[dict[str, str]] = []
    for row in conn.execute(
        """
        SELECT 'instrument_mapping' AS item_type, mapping_id AS item_id, source_platform,
               source_account AS account_label, source_label AS label, mapping_status,
               quality_flags_json, notes, created_at
        FROM instrument_mappings
        WHERE mapping_status IN ('needs_manual_review','missing_isin','missing_ticker','ambiguous')
        UNION ALL
        SELECT 'platform_account_mapping' AS item_type, mapping_id AS item_id, source_platform,
               source_account_label AS account_label, source_account_label AS label, mapping_status,
               quality_flags_json, notes, created_at
        FROM platform_account_mappings
        WHERE mapping_status IN ('needs_manual_review','missing_isin','missing_ticker','ambiguous')
        ORDER BY created_at DESC
        """
    ).fetchall():
        rows.append({k: "" if row[k] is None else str(row[k]) for k in row.keys()})
    return rows

def build_import_wizard_summary(conn: Connection) -> dict[str, str]:
    last = conn.execute("SELECT * FROM broker_import_dry_runs ORDER BY created_at DESC LIMIT 1").fetchone()
    pending = len(get_manual_review_queue(conn))
    return {
        "mode": "read-only",
        "productive_import_enabled": "false",
        "pending_review_items": str(pending),
        "last_dry_run_source": last["source_platform"] if last else "",
        "last_dry_run_status": last["snapshot_date_status"] if last else "none",
    }

def assert_write_enabled(*, write_enabled: bool) -> None:
    if not write_enabled:
        raise PermissionError("Import commit is blocked in read-only mode")


from pathlib import Path
p=Path('/home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/dashboard/data.py')
s=p.read_text()
insert='''\n\ndef get_import_wizard_summary(conn: Connection) -> dict[str, str]:\n    from jarvis_finance.imports.broker_mapping import build_import_wizard_summary\n\n    return build_import_wizard_summary(conn)\n\n\ndef get_manual_review_items(conn: Connection) -> list[dict[str, str]]:\n    from jarvis_finance.imports.broker_mapping import get_manual_review_queue\n\n    return get_manual_review_queue(conn)\n\n\ndef get_broker_dry_runs(conn: Connection, limit: int = 20) -> list[dict[str, str]]:\n    return rowdicts(conn.execute(\n        """\n        SELECT source_platform, source_file_type, detected_snapshot_date, snapshot_date_status,\n               rows_total, candidate_positions, candidate_cash_rows, mapped_positions,\n               blocked_positions, warnings_count, errors_count, quality_flags_json, created_at\n        FROM broker_import_dry_runs ORDER BY created_at DESC LIMIT ?\n        """, (limit,)\n    ).fetchall())\n'''
marker='\n\ndef get_command_center_summary'
if marker not in s: raise SystemExit('marker missing')
s=s.replace(marker, insert+marker)
old='''def get_data_quality(conn: Connection) -> list[dict[str, str]]:\n    items: list[dict[str, str]] = []\n    for row in conn.execute("SELECT COUNT(*) AS c FROM transactions WHERE fx_status='missing'"):\n        items.append({"check": "fehlende FX", "count": str(row["c"]), "severity": "kritisch" if row["c"] else "ok"})\n    for row in conn.execute("SELECT COUNT(*) AS c FROM crypto_assets WHERE coingecko_id IS NULL OR coingecko_id='' "):\n        items.append({"check": "fehlende CoinGecko-ID", "count": str(row["c"]), "severity": "warnung" if row["c"] else "ok"})\n    for row in conn.execute("SELECT COUNT(*) AS c FROM crypto_assets ca LEFT JOIN crypto_prices cp ON cp.asset_id=ca.asset_id WHERE cp.crypto_price_id IS NULL"):\n        items.append({"check": "fehlende Crypto-Preise", "count": str(row["c"]), "severity": "warnung" if row["c"] else "ok"})\n    for row in conn.execute("SELECT COUNT(*) AS c FROM alerts WHERE status='active' AND priority='kritisch'"):\n        items.append({"check": "aktive kritische Alerts", "count": str(row["c"]), "severity": "kritisch" if row["c"] else "ok"})\n    return items\n'''
new='''def get_data_quality(conn: Connection) -> list[dict[str, str]]:\n    items: list[dict[str, str]] = []\n    checks = [\n        ("fehlende FX", "SELECT COUNT(*) AS c FROM transactions WHERE fx_status='missing'", "kritisch"),\n        ("fehlende CoinGecko-ID", "SELECT COUNT(*) AS c FROM crypto_assets WHERE coingecko_id IS NULL OR coingecko_id='' ", "warnung"),\n        ("fehlende Crypto-Preise", "SELECT COUNT(*) AS c FROM crypto_assets ca LEFT JOIN crypto_prices cp ON cp.asset_id=ca.asset_id WHERE cp.crypto_price_id IS NULL", "warnung"),\n        ("fehlende ISIN", "SELECT COUNT(*) AS c FROM instrument_mappings WHERE asset_class IN ('equity','etf') AND (isin IS NULL OR isin='') AND mapping_status!='ignored'", "warnung"),\n        ("fehlende FX in Import-Reviews", "SELECT COUNT(*) AS c FROM instrument_mappings WHERE quality_flags_json LIKE '%missing_fx%' AND mapping_status!='ignored'", "warnung"),\n        ("snapshot_only Positionen", "SELECT COUNT(*) AS c FROM instrument_mappings WHERE quality_flags_json LIKE '%snapshot_only%' AND mapping_status!='ignored'", "info"),\n        ("cost_basis_uncertain", "SELECT COUNT(*) AS c FROM instrument_mappings WHERE quality_flags_json LIKE '%cost_basis_uncertain%' AND mapping_status!='ignored'", "warnung"),\n        ("aktive Alerts", "SELECT COUNT(*) AS c FROM alerts WHERE status='active'", "warnung"),\n        ("resolved Alerts", "SELECT COUNT(*) AS c FROM alerts WHERE status='resolved'", "ok"),\n        ("alte aktive Alerts", "SELECT COUNT(*) AS c FROM alerts WHERE status='active' AND last_seen_at < datetime('now', '-7 days')", "info"),\n    ]\n    for label, sql, severity in checks:\n        row = conn.execute(sql).fetchone()\n        count = int(row["c"] or 0) if row else 0\n        sev = severity if count else "ok"\n        items.append({"check": label, "count": str(count), "severity": sev})\n    return items\n'''
if old not in s: raise SystemExit('old dq block missing')
p.write_text(s.replace(old,new))
print('dashboard data patched')


Output: dashboard data patched


from __future__ import annotations

from jarvis_finance.dashboard import data
from jarvis_finance.dashboard.components.tables import show_table
from jarvis_finance.imports.broker_mapping import SOURCE_PLATFORMS, assert_write_enabled

PAGE_TITLE = "Import Wizard"

def render(st, conn) -> None:
    st.title("Import Wizard")
    st.caption("Phase B1: Dry Run, Mapping und Review — kein produktiver Aktien-/ETF-/Cash-Import.")

    mode = "Read-only"
    st.info(f"Modus: {mode}. Produktive Commit-Aktionen sind standardmäßig deaktiviert.")
    st.warning("Echte Broker-/Bankdateien nur im Runtime-/Upload-Kontext verwenden. Keine Dateien ins Repo kopieren.")

    source = st.selectbox("Quelle", ["PostFinance", "True Wealth", "Raiffeisen", "Manuell / Sonstige"])
    uploaded = st.file_uploader("Datei auswählen — wird nicht ins Repo geschrieben", type=["docx", "xlsx", "csv", "pdf"])
    st.write({"source": source, "file_type": getattr(uploaded, "type", "") if uploaded else "", "write_enabled": False})

    st.subheader("Struktur erkennen")
    st.caption("MVP zeigt gespeicherte Dry-Run-Zusammenfassungen. Parser schreiben nur Runtime-Summary, keine produktiven Ledger-Positionen.")
    show_table(st, data.get_broker_dry_runs(conn))

    st.subheader("Mapping prüfen")
    show_table(st, data.get_manual_review_items(conn))

    st.subheader("Dry Run Summary")
    st.write(data.get_import_wizard_summary(conn))

    st.subheader("Review Queue")
    st.caption("missing_isin, missing_ticker, ambiguous_name, missing_fx, cost_basis_uncertain, snapshot_only, market_value_legacy")

    st.subheader("Commit")
    try:
        assert_write_enabled(write_enabled=False)
    except PermissionError as exc:
        st.error(str(exc))
    st.button("Produktiven Import committen", disabled=True)


from __future__ import annotations

from jarvis_finance.dashboard import data
from jarvis_finance.dashboard.components.tables import show_table

PAGE_TITLE = "Manual Review Queue"

def render(st, conn) -> None:
    st.title("Manual Review Queue")
    st.caption("Review von unsicheren Import- und Mapping-Problemen. Keine produktive Positionserstellung in Phase B1.")
    rows = data.get_manual_review_items(conn)
    show_table(st, rows)
    st.subheader("Aktionen")
    st.write("Mapping bestätigen, ISIN/Ticker/Exchange ergänzen, Account zuweisen, Position ignorieren, Notiz speichern.")
    st.warning("Alle Änderungen erfordern Audit-Log. Produktive Positionserstellung bleibt deaktiviert.")
    st.button("Mapping bestätigen", disabled=True)
    st.button("Position ignorieren", disabled=True)


from __future__ import annotations

from jarvis_finance.dashboard import data
from jarvis_finance.dashboard.components.tables import show_table

PAGE_TITLE = "Data Quality Center"

def render(st, conn) -> None:
    st.title("Data Quality Center")
    st.caption("MVP-Übersicht für Alerts, Preise, FX, ISIN, Snapshot- und Cost-Basis-Probleme.")
    show_table(st, data.get_data_quality(conn))
    st.subheader("Alerts")
    show_table(st, data.get_alerts(conn))
    st.subheader("Aktionen")
    st.button("Data Quality Recheck", disabled=True)
    st.button("Resolve Fixed Alerts", disabled=True)
    st.info("Buttons sind in Phase B1 sichtbar vorbereitet; produktive Aktion läuft weiterhin über CLI/gesonderte Bestätigung.")


from pathlib import Path
p=Path('/home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/dashboard/pages/__init__.py')
s=p.read_text()
s=s.replace('from . import page_13_equity_manage\n', 'from . import page_13_equity_manage\nfrom . import page_14_import_wizard\nfrom . import page_15_manual_review_queue\nfrom . import page_16_data_quality_center\n')
s=s.replace('    page_13_equity_manage,\n]', '    page_13_equity_manage,\n    page_14_import_wizard,\n    page_15_manual_review_queue,\n    page_16_data_quality_center,\n]')
p.write_text(s)
print('pages registered')


Output: pages registered


Explanation:
**Updating Streamlit Components**

I’m thinking about the need to update the Streamlit fake file uploader. It seems like `getattr` returns a method, but wait—`__getattr__` actually returns a recorder. So if the page calls `st.file_uploader`, it will return None by default. I should consider whether I need some kind of schema required for migration version updates. Testing seems essential here to ensure everything works correctly!


patch

--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
@@ -15,7 +15,7 @@
 def test_schema_version_recorded() -> None:
     conn = connect_memory()
     apply_migrations(conn)
-    assert get_schema_version(conn) == 3
+    assert get_schema_version(conn) == 4

 def test_decimal_sensitive_columns_use_text_affinity() -> None:

/home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py was modified by sibling subagent '20260515_094358_9baea0' but this agent never read it. Read the file before writing to avoid overwriting the sibling's changes.


    assert {"source_name", "source_platform", "source_label", "normalized_name", "isin", "ticker", "exchange", "currency", "asset_class", "instrument_id", "mapping_status", "confidence", "quality_flags_json"}.issubset(instrument_mapping_cols)
    assert {"source_platform", "source_account_label", "normalized_platform", "normalized_account_name", "internal_platform_id", "internal_account_id", "account_type", "currency", "mapping_status"}.issubset(account_mapping_cols)
    assert {"source_platform", "source_file_type", "source_filename_hash", "detected_snapshot_date", "snapshot_date_status", "candidate_positions", "candidate_cash_rows", "mapped_positions", "blocked_positions", "quality_flags_json", "summary_json"}.issubset(dry_run_cols)

    assert {"source_name", "source_isin", "source_ticker", "instrument_id", "mapping_status", "quality_flags_json"}.issubset(instrument_mapping_cols)
    assert {"source_name", "source_platform_label", "source_account_label", "platform_id", "account_id", "mapping_status"}.issubset(account_mapping_cols)
    assert {"source_name", "source_file_type", "detected_sections_json", "field_coverage_json", "quality_flags_json"}.issubset(dry_run_cols)


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
@@ -37,6 +37,6 @@
     account_mapping_cols = {row["name"] for row in conn.execute("PRAGMA table_info(platform_account_mappings)").fetchall()}
     dry_run_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_dry_runs)").fetchall()}

-    assert {"source_name", "source_isin", "source_ticker", "instrument_id", "mapping_status", "quality_flags_json"}.issubset(instrument_mapping_cols)
-    assert {"source_name", "source_platform_label", "source_account_label", "platform_id", "account_id", "mapping_status"}.issubset(account_mapping_cols)
-    assert {"source_name", "source_file_type", "detected_sections_json", "field_coverage_json", "quality_flags_json"}.issubset(dry_run_cols)
+    assert {"source_name", "source_platform", "source_label", "normalized_name", "isin", "ticker", "exchange", "currency", "asset_class", "instrument_id", "mapping_status", "confidence", "quality_flags_json"}.issubset(instrument_mapping_cols)
+    assert {"source_platform", "source_account_label", "normalized_platform", "normalized_account_name", "internal_platform_id", "internal_account_id", "account_type", "currency", "mapping_status"}.issubset(account_mapping_cols)
+    assert {"source_platform", "source_file_type", "source_filename_hash", "detected_snapshot_date", "snapshot_date_status", "candidate_positions", "candidate_cash_rows", "mapped_positions", "blocked_positions", "quality_flags_json", "summary_json"}.issubset(dry_run_cols)


Explanation:
**Evaluating testing needs**

I see that I need to create some tests. Maybe I could develop an instrument or helper based on existing tools? I should inspect the importer tests for instruments and accounts to get a clearer idea of what’s required. This way, I can ensure I’m thorough and systematic in handling any testing needs. Let’s keep this focused and productive!


     1|from __future__ import annotations
     2|
     3|from pathlib import Path
     4|
     5|from jarvis_finance.imports.instruments_importer import import_instruments_csv
     6|from jarvis_finance.storage.database import connect_memory
     7|from jarvis_finance.storage.migrations import apply_migrations
     8|
     9|
    10|def test_instruments_import_commit_and_idempotent(tmp_path: Path) -> None:
    11|    path = tmp_path / "instruments.csv"
    12|    path.write_text(
    13|        "asset_class,name,ticker,isin,exchange,currency,country,sector,data_provider_primary,notes\n"
    14|        "equity,Demo Global AG,DGA,CH0000000001,SIX,CHF,CH,Technology,dummy,Synthetic only\n",
    15|        encoding="utf-8",
    16|    )
    17|    conn = connect_memory()
    18|    apply_migrations(conn)
    19|
    20|    first = import_instruments_csv(conn, path, commit=True, source_filename="instruments.csv")
    21|    second = import_instruments_csv(conn, path, commit=True, source_filename="instruments.csv")
    22|
    23|    assert first.rows_new == 1
    24|    assert second.rows_existing == 1
    25|    assert conn.execute("SELECT COUNT(*) AS n FROM instruments").fetchone()["n"] == 1
    26|
    27|
    28|def test_instruments_dry_run_does_not_write(tmp_path: Path) -> None:
    29|    path = tmp_path / "instruments.csv"
    30|    path.write_text(
    31|        "asset_class,name,ticker,isin,exchange,currency,country,sector,data_provider_primary,notes\n"
    32|        "etf,Demo World ETF,DWLD,CH0000000002,SIX,CHF,CH,Diversified,dummy,Synthetic only\n",
    33|        encoding="utf-8",
    34|    )
    35|    conn = connect_memory()
    36|    apply_migrations(conn)
    37|
    38|    result = import_instruments_csv(conn, path, commit=False, source_filename="instruments.csv")
    39|
    40|    assert result.rows_new == 1
    41|    assert result.status == "dry_run_ok"
    42|    assert conn.execute("SELECT COUNT(*) AS n FROM instruments").fetchone()["n"] == 0
    43|
    44|
    45|def test_instruments_import_collects_validation_errors(tmp_path: Path) -> None:
    46|    path = tmp_path / "bad_instruments.csv"
    47|    path.write_text(
    48|        "asset_class,name,ticker,isin,exchange,currency\n"
    49|        "equity,,DGA,CH0000000001,SIX,CHF\n",
    50|        encoding="utf-8",
    51|    )
    52|    conn = connect_memory()
    53|    apply_migrations(conn)
    54|
    55|    result = import_instruments_csv(conn, path, commit=True, source_filename="bad_instruments.csv")
    56|
    57|    assert result.rows_failed == 1
    58|    assert result.status == "failed"
    59|    assert "name" in result.errors[0]
    60|


     1|from __future__ import annotations
     2|
     3|from pathlib import Path
     4|from sqlite3 import Connection
     5|
     6|from .common import ImportResult, clean, create_import_session, require_text, source_hash, stable_id, utc_now
     7|from .dry_run import read_csv_rows
     8|
     9|REQUIRED_COLUMNS = {"asset_class", "name", "currency"}
    10|
    11|
    12|def _instrument_key(row: dict[str, str]) -> tuple[str, str, str]:
    13|    isin = clean(row.get("isin")).upper()
    14|    ticker = clean(row.get("ticker")).upper()
    15|    name = clean(row.get("name"))
    16|    return isin, ticker, name
    17|
    18|
    19|def _existing_instrument(conn: Connection, row: dict[str, str]) -> bool:
    20|    isin, ticker, name = _instrument_key(row)
    21|    if isin:
    22|        existing = conn.execute("SELECT 1 FROM instruments WHERE isin = ? LIMIT 1", (isin,)).fetchone()
    23|        if existing:
    24|            return True
    25|    if ticker:
    26|        existing = conn.execute("SELECT 1 FROM instruments WHERE ticker = ? AND name = ? LIMIT 1", (ticker, name)).fetchone()
    27|        if existing:
    28|            return True
    29|    return False
    30|
    31|
    32|def import_instruments_csv(
    33|    conn: Connection,
    34|    path: str | Path,
    35|    *,
    36|    commit: bool,
    37|    source_filename: str | None = None,
    38|) -> ImportResult:
    39|    rows, errors = read_csv_rows(path, REQUIRED_COLUMNS)
    40|    source_filename = source_filename or Path(path).name
    41|    rows_new = rows_existing = 0
    42|    validated: list[dict[str, str]] = []
    43|    for line, row in enumerate(rows, start=2):
    44|        try:
    45|            asset_class = require_text(row, "asset_class", line)
    46|            name = require_text(row, "name", line)
    47|            currency = require_text(row, "currency", line).upper()
    48|            isin, ticker, _ = _instrument_key(row)
    49|            instrument_id = stable_id("instrument", isin or ticker or name, currency)
    50|            if _existing_instrument(conn, {**row, "name": name, "ticker": ticker, "isin": isin}):
    51|                rows_existing += 1
    52|            else:
    53|                rows_new += 1
    54|            validated.append({
    55|                "instrument_id": instrument_id,
    56|                "asset_class": asset_class,
    57|                "name": name,
    58|                "ticker": ticker,
    59|                "isin": isin,
    60|                "exchange": clean(row.get("exchange")),
    61|                "currency": currency,
    62|                "country": clean(row.get("country")),
    63|                "sector": clean(row.get("sector")),
    64|                "data_provider_primary": clean(row.get("data_provider_primary")),
    65|                "notes": clean(row.get("notes")),
    66|            })
    67|        except ValueError as exc:
    68|            errors.append(str(exc))
    69|    status = "failed" if errors else ("committed" if commit else "dry_run_ok")
    70|    if commit and not errors:
    71|        now = utc_now()
    72|        for item in validated:
    73|            conn.execute(
    74|                """
    75|                INSERT OR IGNORE INTO instruments(
    76|                    instrument_id, asset_class, name, ticker, isin, exchange, currency,
    77|                    country, sector, data_provider_primary, notes, created_at
    78|                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    79|                """,
    80|                (
    81|                    item["instrument_id"], item["asset_class"], item["name"], item["ticker"], item["isin"],
    82|                    item["exchange"], item["currency"], item["country"], item["sector"],
    83|                    item["data_provider_primary"], item["notes"], now,
    84|                ),
    85|            )
    86|    session_id = create_import_session(
    87|        conn,
    88|        import_type="instruments",
    89|        source_filename=source_filename,
    90|        file_hash=source_hash(path),
    91|        status=status,
    92|        rows_total=len(rows),
    93|        rows_imported=rows_new if commit and not errors else 0,
    94|        rows_failed=len(errors),
    95|        errors=errors,
    96|        notes="dry run" if not commit else None,
    97|    )
    98|    return ImportResult(session_id, "instruments", status, len(rows), rows_new if not errors else 0, rows_existing, len(errors), errors)
    99|


from __future__ import annotations

import json
from pathlib import Path

import pytest

from jarvis_finance.dashboard import data as dashboard_data
from jarvis_finance.dashboard.pages import page_14_import_wizard, page_15_manual_review_queue, page_16_data_quality_center
from jarvis_finance.imports.broker_mapping import (
    DryRunInput,
    assert_write_enabled,
    confirm_instrument_mapping,
    create_broker_import_dry_run,
    create_instrument_mapping,
    create_platform_account_mapping,
    get_manual_review_queue,
)
from jarvis_finance.storage.database import connect_memory
from jarvis_finance.storage.migrations import apply_migrations, utc_now

class FakeStreamlit:
    def __init__(self) -> None:
        self.calls: list[tuple[str, tuple[object, ...], dict[str, object]]] = []
        self.sidebar = self

    def __getattr__(self, name: str):
        def recorder(*args, **kwargs):
            self.calls.append((name, args, kwargs))
            if name in {"button", "checkbox"}:
                return False
            if name in {"selectbox", "radio"}:
                options = args[1] if len(args) > 1 else kwargs.get("options", [])
                return list(options)[0] if options else None
            if name == "tabs":
                return [self for _ in (args[0] if args else [])]
            if name == "columns":
                count = args[0] if args else 1
                return [self for _ in range(count)] if isinstance(count, int) else [self for _ in count]
            return None
        return recorder

    def __enter__(self):
        return self

    def __exit__(self, *args):
        return False

def conn():
    c = connect_memory()
    apply_migrations(c)
    return c

def seed_instrument(c, *, instrument_id="inst-demo", isin="CH0000000001", ticker="DGA", exchange="SIX", currency="CHF"):
    c.execute(
        "INSERT INTO instruments(instrument_id, asset_class, name, ticker, isin, exchange, currency, created_at) VALUES (?, 'etf', 'Demo ETF', ?, ?, ?, ?, ?)",
        (instrument_id, ticker, isin, exchange, currency, utc_now()),
    )
    return instrument_id

def seed_platform_account(c, *, platform_id="pf", account_id="pf-cash", platform="PostFinance", account="Cash"):
    now = utc_now()
    c.execute("INSERT INTO platforms(platform_id, name, platform_type, default_currency, created_at) VALUES (?, ?, 'bank', 'CHF', ?)", (platform_id, platform, now))
    c.execute("INSERT INTO accounts(account_id, platform_id, account_name, account_type, currency, created_at) VALUES (?, ?, ?, 'cash', 'CHF', ?)", (account_id, platform_id, account, now))
    return platform_id, account_id

def test_instrument_mapping_can_be_created_and_auto_maps_by_isin() -> None:
    c = conn()
    instrument_id = seed_instrument(c)
    result = create_instrument_mapping(c, source_platform="PostFinance", source_label="Demo ETF", asset_class="etf", isin="CH0000000001", currency="CHF")
    row = c.execute("SELECT * FROM instrument_mappings WHERE mapping_id=?", (result["mapping_id"],)).fetchone()
    assert row["mapping_status"] == "mapped"
    assert row["instrument_id"] == instrument_id

def test_missing_isin_creates_needs_manual_review() -> None:
    c = conn()
    result = create_instrument_mapping(c, source_platform="PostFinance", source_label="Demo ETF", asset_class="etf", ticker="DGA", currency="CHF")
    assert result["mapping_status"] == "needs_manual_review"
    assert "missing_isin" in result["quality_flags"]

def test_name_without_isin_or_ticker_is_not_automapped() -> None:
    c = conn()
    result = create_instrument_mapping(c, source_platform="True Wealth", source_label="Global Index Fund", asset_class="etf", currency="USD")
    assert result["mapping_status"] == "needs_manual_review"
    assert result["instrument_id"] is None
    assert "name_only_not_auto_mapped" in result["quality_flags"]

def test_platform_account_mapping_for_postfinance_and_true_wealth_and_raiffeisen_cash() -> None:
    c = conn()
    pf_platform, pf_account = seed_platform_account(c, platform_id="pf", account_id="pf-depot", platform="PostFinance", account="Depot")
    pf = create_platform_account_mapping(c, source_platform="PostFinance", source_account_label="Depot", account_type="brokerage", currency="CHF", internal_platform_id=pf_platform, internal_account_id=pf_account)
    tw = create_platform_account_mapping(c, source_platform="True Wealth", source_account_label="Robo Portfolio", account_type="robo_portfolio", currency="CHF")
    rf = create_platform_account_mapping(c, source_platform="Raiffeisen", source_account_label="Privatkonto", account_type="brokerage", currency="CHF")
    assert pf["mapping_status"] == "mapped"
    assert tw["mapping_status"] == "needs_manual_review"
    assert rf["account_type"] == "cash"
    assert "raiffeisen_instrument_detail_missing" in rf["quality_flags"]

def test_broker_import_dry_run_stores_summary_without_productive_positions() -> None:
    c = conn()
    dry_run_id = create_broker_import_dry_run(c, DryRunInput(source_platform="PostFinance", source_file_type="docx", source_filename="real-file-redacted.docx", snapshot_date_status="missing", rows_total=10, candidate_positions=3, candidate_cash_rows=2, mapped_positions=1, blocked_positions=2, warnings_count=4, quality_flags={"missing_isin": 2}, summary={"rows": [{"forbidden": "raw"}], "safe": "aggregate"}))
    row = c.execute("SELECT * FROM broker_import_dry_runs WHERE dry_run_id=?", (dry_run_id,)).fetchone()
    assert row["candidate_positions"] == 3
    assert json.loads(row["summary_json"]) == {"safe": "aggregate"}
    assert c.execute("SELECT COUNT(*) AS n FROM positions_snapshot").fetchone()["n"] == 0

def test_import_wizard_manual_review_and_data_quality_pages_import_and_render() -> None:
    c = conn()
    create_instrument_mapping(c, source_platform="PostFinance", source_label="Unknown ETF", asset_class="etf", currency="USD")
    st = FakeStreamlit()
    for module in [page_14_import_wizard, page_15_manual_review_queue, page_16_data_quality_center]:
        module.render(st, c)
    assert any(call[0] == "title" for call in st.calls)

def test_manual_review_queue_loads_open_mapping_problems() -> None:
    c = conn()
    create_instrument_mapping(c, source_platform="PostFinance", source_label="Unknown ETF", asset_class="etf")
    rows = get_manual_review_queue(c)
    assert rows
    assert rows[0]["item_type"] == "instrument_mapping"

def test_data_quality_center_reports_missing_isin_fx_and_prices() -> None:
    c = conn()
    create_instrument_mapping(c, source_platform="True Wealth", source_label="Unknown USD ETF", asset_class="etf", currency="USD")
    c.execute("UPDATE instrument_mappings SET quality_flags_json=?", (json.dumps(["missing_fx", "snapshot_only", "cost_basis_uncertain"]),))
    rows = dashboard_data.get_data_quality(c)
    checks = {row["check"]: row["count"] for row in rows}
    assert checks["fehlende ISIN"] == "1"
    assert checks["fehlende FX in Import-Reviews"] == "1"
    assert "fehlende Crypto-Preise" in checks

def test_read_only_mode_blocks_commit_actions() -> None:
    with pytest.raises(PermissionError):
        assert_write_enabled(write_enabled=False)

def test_audit_log_created_when_manual_mapping_confirmed() -> None:
    c = conn()
    instrument_id = seed_instrument(c)
    result = create_instrument_mapping(c, source_platform="PostFinance", source_label="Manual ETF", asset_class="etf", ticker="MAN", currency="CHF")
    audit_id = confirm_instrument_mapping(c, mapping_id=result["mapping_id"], instrument_id=instrument_id, note="Synthetic manual confirmation")
    assert audit_id
    assert c.execute("SELECT COUNT(*) AS n FROM audit_log WHERE entity_id=?", (result["mapping_id"],)).fetchone()["n"] == 1

def test_no_real_files_created_in_repo(tmp_path: Path) -> None:
    c = conn()
    create_broker_import_dry_run(c, DryRunInput(source_platform="Raiffeisen", source_file_type="xlsx", source_filename="redacted.xlsx", rows_total=1))
    assert not any(tmp_path.rglob("*.xlsx"))


terminal

set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src pytest tests/unit/test_broker_mapping_import_wizard.py tests/unit/test_schema.py tests/unit/test_dashboard_mvp.py -q


Output:
.....................                                                    [100%]
21 passed in 0.19s


Explanation:
**Considering schema upgrades**

I’m thinking about upgrading the initial schema. The existing schema doesn’t require creating new tables, but I’m wondering if I need to apply migrations at runtime. It might not be necessary for testing. Also, it seems like the old DB schema version will become version 4, which is good. I definitely need to make a note to update that and then compile the tests in full after making the changes.

todo

Schema/Modelle für Instrument-, Account-Mapping und Broker Dry-Run Phase B1 erweitern

Import Wizard, Manual Review Queue, Data Quality Center, Safe Mode UI als Dashboard-Seiten/MVP implementieren

Tests für Mapping, Dry-Run, Review, DQ, Read-only, Audit und Git-Safety ergänzen


Compile, pytest, Git-Safety, Status, Commit und Push durchführen


terminal

set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python -m compileall -q src tests
PYTHONPATH=src pytest -q


Output:
........................................................................ [ 53%]
..............................................................           [100%]
134 passed in 3.89s


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
rm -rf .pytest_cache src/jarvis_finance/**/__pycache__ tests/**/__pycache__
PYTHONPATH=src python -m jarvis_finance.cli.main git-safety-scan .
git status --short --branch


Output:
GIT_SAFETY_OK
## main...origin/main
 M src/jarvis_finance/dashboard/data.py
 M src/jarvis_finance/dashboard/pages/__init__.py
 M src/jarvis_finance/storage/migrations.py
 M tests/unit/test_schema.py
?? src/jarvis_finance/dashboard/pages/page_14_import_wizard.py
?? src/jarvis_finance/dashboard/pages/page_15_manual_review_queue.py
?? src/jarvis_finance/dashboard/pages/page_16_data_quality_center.py
?? src/jarvis_finance/imports/broker_mapping.py
?? tests/unit/test_broker_mapping_import_wizard.py


Explanation:
**Considering file structure**

I’m thinking about whether I need to do a dry-run on the actual real files. The user mentioned reading, but that’s not strictly required. Since the goal isn’t productivity-focused, maybe I can skip it? We’ve already used the previous version. I should commit and push, but before that, I should inspect the diff to make sure everything looks good. It’s a good practice to double-check before finalizing my changes.

set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
git diff --stat
git diff -- src/jarvis_finance/storage/migrations.py | sed -n '1,220p'


Output:
src/jarvis_finance/dashboard/data.py           | 48 ++++++++++---
 src/jarvis_finance/dashboard/pages/__init__.py |  6 ++
 src/jarvis_finance/storage/migrations.py       | 98 ++++++++++++++++++++++----
 tests/unit/test_schema.py                      |  8 +--
 4 files changed, 133 insertions(+), 27 deletions(-)
diff --git a/src/jarvis_finance/storage/migrations.py b/src/jarvis_finance/storage/migrations.py
index 8fb64f2..9c1f3b9 100644
--- a/src/jarvis_finance/storage/migrations.py
+++ b/src/jarvis_finance/storage/migrations.py
@@ -6,8 +6,8 @@ from sqlite3 import Connection

 from .schema import INITIAL_SCHEMA_SQL

-MIGRATION_VERSION = 3
-MIGRATION_NAME = "003_broker_bank_mapping_foundation"
+MIGRATION_VERSION = 4
+MIGRATION_NAME = "004_import_wizard_mapping_layer"

 INSTRUMENT_OPTIONAL_COLUMNS = {
     "position_category": "TEXT",
@@ -84,60 +84,128 @@ def _rebuild_table_with_text_columns(conn: Connection, table: str, text_columns:
         conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_crypto_holdings_asset_wallet ON crypto_holdings(asset_id, wallet_id)")

+def _add_missing_columns(conn: Connection, table: str, columns: dict[str, str]) -> None:
+    existing = _table_columns(conn, table)
+    for name, col_type in columns.items():
+        if name not in existing:
+            conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {col_type}")
+
+
 def _create_broker_bank_mapping_tables(conn: Connection) -> None:
     conn.executescript(
         """
         CREATE TABLE IF NOT EXISTS instrument_mappings (
             mapping_id TEXT PRIMARY KEY,
-            source_name TEXT NOT NULL,
+            source_name TEXT,
+            source_platform TEXT NOT NULL,
+            source_account TEXT,
+            source_label TEXT NOT NULL,
+            normalized_name TEXT NOT NULL,
+            isin TEXT,
+            ticker TEXT,
+            exchange TEXT,
+            currency TEXT,
+            asset_class TEXT NOT NULL DEFAULT 'other',
             source_instrument_name TEXT,
             source_isin TEXT,
             source_ticker TEXT,
             source_exchange TEXT,
             source_currency TEXT,
             instrument_id TEXT REFERENCES instruments(instrument_id),
-            mapping_status TEXT NOT NULL DEFAULT 'unmapped',
+            mapping_status TEXT NOT NULL DEFAULT 'needs_manual_review',
+            confidence TEXT NOT NULL DEFAULT '0',
             quality_flags_json TEXT,
             notes TEXT,
             created_at TEXT NOT NULL,
-            updated_at TEXT,
-            UNIQUE(source_name, source_isin, source_ticker, source_instrument_name, source_currency)
+            updated_at TEXT
         );

         CREATE TABLE IF NOT EXISTS platform_account_mappings (
             mapping_id TEXT PRIMARY KEY,
-            source_name TEXT NOT NULL,
+            source_name TEXT,
+            source_platform TEXT NOT NULL,
             source_platform_label TEXT,
-            source_account_label TEXT,
+            source_account_label TEXT NOT NULL,
+            normalized_platform TEXT NOT NULL,
+            normalized_account_name TEXT NOT NULL,
             source_account_type TEXT,
+            account_type TEXT NOT NULL DEFAULT 'other',
             source_currency TEXT,
+            currency TEXT,
             platform_id TEXT REFERENCES platforms(platform_id),
             account_id TEXT REFERENCES accounts(account_id),
-            mapping_status TEXT NOT NULL DEFAULT 'unmapped',
+            internal_platform_id TEXT REFERENCES platforms(platform_id),
+            internal_account_id TEXT REFERENCES accounts(account_id),
+            mapping_status TEXT NOT NULL DEFAULT 'needs_manual_review',
             quality_flags_json TEXT,
             notes TEXT,
             created_at TEXT NOT NULL,
-            updated_at TEXT,
-            UNIQUE(source_name, source_platform_label, source_account_label, source_currency)
+            updated_at TEXT
         );

         CREATE TABLE IF NOT EXISTS broker_import_dry_runs (
             dry_run_id TEXT PRIMARY KEY,
-            source_name TEXT NOT NULL,
-            source_file_label TEXT NOT NULL,
+            source_name TEXT,
+            source_platform TEXT NOT NULL,
+            source_file_label TEXT,
             source_file_type TEXT NOT NULL,
             source_hash TEXT,
+            source_filename_hash TEXT,
+            detected_snapshot_date TEXT,
+            snapshot_date_status TEXT NOT NULL DEFAULT 'missing',
             detected_sections_json TEXT NOT NULL,
-            field_coverage_json TEXT NOT NULL,
-            quality_flags_json TEXT NOT NULL,
+            field_coverage_json TEXT,
             rows_total INTEGER NOT NULL DEFAULT 0,
             rows_position_candidates INTEGER NOT NULL DEFAULT 0,
             rows_cash_candidates INTEGER NOT NULL DEFAULT 0,
+            candidate_positions INTEGER NOT NULL DEFAULT 0,
+            candidate_cash_rows INTEGER NOT NULL DEFAULT 0,
+            mapped_positions INTEGER NOT NULL DEFAULT 0,
+            blocked_positions INTEGER NOT NULL DEFAULT 0,
+            warnings_count INTEGER NOT NULL DEFAULT 0,
+            errors_count INTEGER NOT NULL DEFAULT 0,
+            quality_flags_json TEXT NOT NULL,
+            summary_json TEXT NOT NULL DEFAULT '{}',
             created_at TEXT NOT NULL,
             notes TEXT
         );
         """
     )
+    _add_missing_columns(conn, "instrument_mappings", {
+        "source_name": "TEXT",
+        "source_platform": "TEXT",
+        "source_account": "TEXT",
+        "source_label": "TEXT",
+        "normalized_name": "TEXT",
+        "isin": "TEXT",
+        "ticker": "TEXT",
+        "exchange": "TEXT",
+        "currency": "TEXT",
+        "asset_class": "TEXT DEFAULT 'other'",
+        "confidence": "TEXT DEFAULT '0'",
+    })
+    _add_missing_columns(conn, "platform_account_mappings", {
+        "source_platform": "TEXT",
+        "normalized_platform": "TEXT",
+        "normalized_account_name": "TEXT",
+        "account_type": "TEXT DEFAULT 'other'",
+        "currency": "TEXT",
+        "internal_platform_id": "TEXT",
+        "internal_account_id": "TEXT",
+    })
+    _add_missing_columns(conn, "broker_import_dry_runs", {
+        "source_platform": "TEXT",
+        "source_filename_hash": "TEXT",
+        "detected_snapshot_date": "TEXT",
+        "snapshot_date_status": "TEXT DEFAULT 'missing'",
+        "candidate_positions": "INTEGER DEFAULT 0",
+        "candidate_cash_rows": "INTEGER DEFAULT 0",
+        "mapped_positions": "INTEGER DEFAULT 0",
+        "blocked_positions": "INTEGER DEFAULT 0",
+        "warnings_count": "INTEGER DEFAULT 0",
+        "errors_count": "INTEGER DEFAULT 0",
+        "summary_json": "TEXT DEFAULT '{}'",
+    })

 def _apply_compat_migrations(conn: Connection) -> None:
     _add_missing_instrument_columns(conn)


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
git add src/jarvis_finance/storage/migrations.py src/jarvis_finance/dashboard/data.py src/jarvis_finance/dashboard/pages/__init__.py src/jarvis_finance/dashboard/pages/page_14_import_wizard.py src/jarvis_finance/dashboard/pages/page_15_manual_review_queue.py src/jarvis_finance/dashboard/pages/page_16_data_quality_center.py src/jarvis_finance/imports/broker_mapping.py tests/unit/test_schema.py tests/unit/test_broker_mapping_import_wizard.py
git commit -m "feat: add equity import wizard mapping layer"


Output:
[main cd2d954] feat: add equity import wizard mapping layer
 9 files changed, 673 insertions(+), 27 deletions(-)
 create mode 100644 src/jarvis_finance/dashboard/pages/page_14_import_wizard.py
 create mode 100644 src/jarvis_finance/dashboard/pages/page_15_manual_review_queue.py
 create mode 100644 src/jarvis_finance/dashboard/pages/page_16_data_quality_center.py
 create mode 100644 src/jarvis_finance/imports/broker_mapping.py
 create mode 100644 tests/unit/test_broker_mapping_import_wizard.py


Explanation:
**Verifying git push process**

I need to make sure everything is correct before pushing my changes. Maybe I should compile and check the status after committing. It seems like running a git status could help confirm that safety measures are in place. The requirement is to ensure everything is good before I complete the process. So, I’m thinking after I commit, it might be wise to verify the safety status again before proceeding with the push.

set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python -m compileall -q src tests
PYTHONPATH=src pytest -q
rm -rf .pytest_cache src/jarvis_finance/**/__pycache__ tests/**/__pycache__
PYTHONPATH=src python -m jarvis_finance.cli.main git-safety-scan .
git status --short --branch


Output:
........................................................................ [ 53%]
..............................................................           [100%]
134 passed in 3.77s
GIT_SAFETY_OK
## main...origin/main [voraus 1]


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
python - <<'PY'
import re, subprocess, sys
from pathlib import Path
repo=Path('/home/agent/.hermes/repos/FinanceManager')
text=Path.home().joinpath('jarvis_runtime/finance-system/secrets/Github_token_FinanceManager.csv').read_text()
m=re.search(r'(github_pat_[A-Za-z0-9_]+|ghp_[A-Za-z0-9_]+|gho_[A-Za-z0-9_]+)', text)
if not m:
    print('TOKEN_PARSE=failed'); sys.exit(12)
token=m.group(1); auth_url=f'https://{token}@github.com/Gamexgit/FinanceManager.git'
def run(args, check=True): return subprocess.run(args, cwd=repo, text=True, capture_output=True, check=check)
try:
    run(['git','fetch',auth_url,'main:refs/remotes/origin/main','--quiet'])
    local=run(['git','rev-parse','HEAD']).stdout.strip(); origin=run(['git','rev-parse','origin/main']).stdout.strip(); base=run(['git','merge-base','HEAD','origin/main']).stdout.strip()
    print('REMOTE_BEFORE='+origin[:7]); print('LOCAL_HEAD='+local[:7])
    if origin != base:
        print('REMOTE_HAS_NEW_COMMITS=yes'); sys.exit(3)
    push=run(['git','push',auth_url,'main:main'], check=False)
    if push.returncode:
        print('PUSH=failed'); print((push.stderr or push.stdout).replace(token,'[REDACTED]')[-1000:]); sys.exit(push.returncode)
    run(['git','fetch',auth_url,'main:refs/remotes/origin/main','--quiet'])
    print('PUSH=success'); print('REMOTE_AFTER='+run(['git','rev-parse','--short','origin/main']).stdout.strip())
finally:
    token=None; auth_url=None
PY
PYTHONPATH=src python -m jarvis_finance.cli.main git-safety-scan .
git status --short --branch


Output:
REMOTE_BEFORE=41f11d2
LOCAL_HEAD=cd2d954
PUSH=success
REMOTE_AFTER=cd2d954
GIT_SAFETY_OK
## main...origin/main


Erledigt, Sir. Phase B1 ist implementiert, getestet, committed und gepusht.

## Ergebnis

- Commit: `cd2d954`
- Remote HEAD nach Push: `cd2d954`
- Push Status: erfolgreich
- Git Status: sauber, `main...origin/main`
- Git-Safety: `GIT_SAFETY_OK`
- Tests: `134 passed`
- Compile: OK

## Implementierte Module

### 1. Instrument Master / Mapping Layer

Neu/erweitert:

- `instrument_mappings`
- Mapping-Logik in:
  - `src/jarvis_finance/imports/broker_mapping.py`

Implementiert:

- `source_platform`
- `source_account`
- `source_label`
- `normalized_name`
- `isin`
- `ticker`
- `exchange`
- `currency`
- `asset_class`
- optionales `instrument_id`
- `mapping_status`
- `confidence`
- `notes`
- `created_at`
- `updated_at`

Regeln umgesetzt:

- ISIN ist Primäridentifikation für Aktien/ETFs.
- Ticker + Exchange nur Fallback und review-pflichtig.
- Name allein wird nicht automatisch gemappt.
- Missing ISIN erzeugt `needs_manual_review` mit Quality Flag `missing_isin`.
- Manuelle Mapping-Bestätigung erzeugt Audit-Log.
- Unsichere Mappings bleiben blockiert/review-pflichtig.

### 2. Platform / Account Mapping

Erweitert:

- `platform_account_mappings`

Implementiert:

- `source_platform`
- `source_account_label`
- `normalized_platform`
- `normalized_account_name`
- `internal_platform_id`
- `internal_account_id`
- `account_type`
- `currency`
- `mapping_status`
- `notes`
- timestamps

Regeln umgesetzt:

- PostFinance, True Wealth, Raiffeisen werden klar getrennt.
- Account-Typen vorbereitet:
  - `brokerage`
  - `cash`
  - `robo_portfolio`
  - `reserve`
  - `other`
- Raiffeisen wird ohne detaillierte Instrumentpositionen defensiv als Cash-/Kontrollsnapshot behandelt.

### 3. Broker Import Dry-Run Model

Erweitert:

- `broker_import_dry_runs`

Implementiert:

- `source_platform`
- `source_file_type`
- `source_filename_hash`
- `detected_snapshot_date`
- `snapshot_date_status`
- `rows_total`
- `candidate_positions`
- `candidate_cash_rows`
- `mapped_positions`
- `blocked_positions`
- `warnings_count`
- `errors_count`
- `quality_flags_json`
- `summary_json`
- `created_at`

Wichtig:

- Dry-Run schreibt keine produktiven Positionen.
- Summary wird ohne raw rows / echte Werte persistiert.
- Produktiver Ledger-/Snapshot-Import bleibt deaktiviert.

## Dashboard/UI

### Neue Seite: Import Wizard

Datei:

- `src/jarvis_finance/dashboard/pages/page_14_import_wizard.py`

Status:

- eingebaut und registriert
- Read-only default
- Quelle auswählbar:
  - PostFinance
  - True Wealth
  - Raiffeisen
  - Manuell / Sonstige
- Upload-/Runtime-Hinweis sichtbar
- Dry-Run-Zusammenfassungen sichtbar
- Mapping-/Review-Daten sichtbar
- Commit-Button sichtbar, aber deaktiviert

### Neue Seite: Manual Review Queue

Datei:

- `src/jarvis_finance/dashboard/pages/page_15_manual_review_queue.py`

Status:

- eingebaut und registriert
- zeigt offene Mapping-Probleme
- vorbereitet für:
  - Mapping bestätigen
  - ISIN ergänzen
  - Ticker ergänzen
  - Exchange ergänzen
  - Account zuweisen
  - Position ignorieren
  - Notiz speichern
- produktive Positionserstellung bleibt deaktiviert

### Neue Seite: Data Quality Center

Datei:

- `src/jarvis_finance/dashboard/pages/page_16_data_quality_center.py`

Status:

- eingebaut und registriert
- zeigt:
  - aktive Alerts
  - resolved Alerts
  - fehlende Crypto-Preise
  - fehlende FX
  - fehlende ISIN
  - fehlende CoinGecko-ID
  - `snapshot_only`
  - `cost_basis_uncertain`
  - alte aktive Alerts
- Buttons vorbereitet:
  - Data Quality Recheck
  - Resolve Fixed Alerts
- Buttons sind in Phase B1 sichtbar, aber deaktiviert.

### Safe Mode / Read-only Mode

Implementiert:

- Import Wizard zeigt Read-only Mode.
- Produktive Commit-Aktionen sind blockiert.
- `assert_write_enabled(write_enabled=False)` blockiert Commit-Aktionen per `PermissionError`.
- Commit-Button in UI ist deaktiviert.
- Keine Rollen-/Auth-Komplexität eingebaut — wie gewünscht.

## Neue/geänderte Dateien

Neu:

- `src/jarvis_finance/imports/broker_mapping.py`
- `src/jarvis_finance/dashboard/pages/page_14_import_wizard.py`
- `src/jarvis_finance/dashboard/pages/page_15_manual_review_queue.py`
- `src/jarvis_finance/dashboard/pages/page_16_data_quality_center.py`
- `tests/unit/test_broker_mapping_import_wizard.py`

Geändert:

- `src/jarvis_finance/storage/migrations.py`
- `src/jarvis_finance/dashboard/data.py`
- `src/jarvis_finance/dashboard/pages/__init__.py`
- `tests/unit/test_schema.py`

## Tests ergänzt

Abgedeckt:

- `instrument_mappings` anlegen
- ISIN-Mapping auf vorhandenes Instrument
- fehlende ISIN → `needs_manual_review`
- Name ohne ISIN/Ticker wird nicht automatisch gemappt
- Platform-/Account-Mapping für PostFinance
- Platform-/Account-Mapping für True Wealth
- Raiffeisen defensiv als Cash/Kontrollsnapshot
- Broker Import Dry Run speichert nur Summary
- Dry Run erzeugt keine produktiven Positionen
- Import Wizard Module importieren/rendern sauber
- Manual Review Queue lädt offene Mapping-Probleme
- Data Quality Center zeigt fehlende ISIN / FX / Preise
- Read-only Mode blockiert Commit-Aktionen
- Audit-Log bei manueller Mapping-Bestätigung
- keine echten Dateien im Repo
- Schema-Version / Tabellenfelder aktualisiert

Testergebnis:

- `134 passed`

## Echte Daten / produktiver Import

Nicht durchgeführt:

- keine echten Aktien-/ETF-Daten produktiv importiert
- keine echten Cash-/Depotwerte produktiv importiert
- keine produktiven `positions_snapshot`-Writes aus Brokerdateien
- keine echten Dateien ins Repo kopiert
- keine echten Werte/Mengen im Chat ausgegeben

## Nächste Empfehlung

Als nächster kontrollierter Schritt wäre sinnvoll:

**Phase B2 — synthetische Parser/Dry-Run-Pipeline pro Quelle**

Reihenfolge:

1. PostFinance synthetischer DOCX-/Tabellenparser
2. True Wealth synthetischer DOCX-/Tabellenparser
3. Raiffeisen synthetischer XLSX-Kontrollsnapshot-Parser
4. Dry-Run UI mit aggregierter Summary füttern
5. Review Queue aus Parser-Ergebnissen erzeugen
6. weiterhin kein produktiver Import echter Daten ohne separate Freigabe

Monte-Carlo, Backtesting, Rebalancing, Scores und Heatmaps wurden nicht begonnen.