     1|from __future__ import annotations
     2|
     3|import hashlib
     4|from datetime import datetime, timezone
     5|from sqlite3 import Connection
     6|
     7|from .schema import INITIAL_SCHEMA_SQL
     8|
     9|MIGRATION_VERSION = 6
    10|MIGRATION_NAME = "006_broker_review_queue_hardening"
    11|
    12|INSTRUMENT_OPTIONAL_COLUMNS = {
    13|    "position_category": "TEXT",
    14|    "ter": "TEXT",
    15|    "distribution_policy": "TEXT",
    16|    "index_name": "TEXT",
    17|    "fund_domicile": "TEXT",
    18|    "benchmark": "TEXT",
    19|}
    20|
    21|TEXT_AFFINITY_COLUMNS = {
    22|    "transactions": {"quantity", "price_original", "gross_amount_original", "fee_original", "tax_original", "net_amount_original", "fx_rate_to_chf", "gross_amount_chf", "fee_chf", "tax_chf", "net_amount_chf"},
    23|    "crypto_holdings": {"quantity", "legacy_snapshot_value_original", "legacy_snapshot_value_chf"},
    24|    "crypto_transactions": {"quantity", "price_original", "gross_amount_original", "fee_quantity", "fee_original", "fx_rate_to_chf", "amount_chf"},
    25|    "crypto_prices": {"price", "market_cap", "volume_24h", "change_24h_pct"},
    26|}
    27|
    28|
    29|def utc_now() -> str:
    30|    return datetime.now(timezone.utc).isoformat()
    31|
    32|
    33|def checksum_sql(sql: str) -> str:
    34|    return hashlib.sha256(sql.encode("utf-8")).hexdigest()
    35|
    36|
    37|def get_schema_version(conn: Connection) -> int:
    38|    row = conn.execute("SELECT MAX(version) AS version FROM schema_migrations").fetchone()
    39|    return int(row["version"] or 0) if row else 0
    40|
    41|
    42|def _table_columns(conn: Connection, table: str) -> dict[str, str]:
    43|    return {row["name"]: (row["type"] or "") for row in conn.execute(f"PRAGMA table_info({table})").fetchall()}
    44|
    45|
    46|def _add_missing_instrument_columns(conn: Connection) -> None:
    47|    existing = _table_columns(conn, "instruments")
    48|    for name, col_type in INSTRUMENT_OPTIONAL_COLUMNS.items():
    49|        if name not in existing:
    50|            conn.execute(f"ALTER TABLE instruments ADD COLUMN {name} {col_type}")
    51|
    52|
    53|def _rebuild_table_with_text_columns(conn: Connection, table: str, text_columns: set[str]) -> None:
    54|    cols = conn.execute(f"PRAGMA table_info({table})").fetchall()
    55|    if not cols:
    56|        return
    57|    needs_rebuild = any(row["name"] in text_columns and (row["type"] or "").upper() != "TEXT" for row in cols)
    58|    if not needs_rebuild:
    59|        return
    60|    tmp = f"{table}__text_migration"
    61|    col_defs: list[str] = []
    62|    pk_cols = [row["name"] for row in cols if row["pk"]]
    63|    for row in cols:
    64|        name = row["name"]
    65|        col_type = "TEXT" if name in text_columns else (row["type"] or "TEXT")
    66|        parts = [name, col_type]
    67|        if row["pk"] and len(pk_cols) == 1:
    68|            parts.append("PRIMARY KEY")
    69|        if row["notnull"]:
    70|            parts.append("NOT NULL")
    71|        if row["dflt_value"] is not None:
    72|            parts.append(f"DEFAULT {row['dflt_value']}")
    73|        col_defs.append(" ".join(parts))
    74|    if len(pk_cols) > 1:
    75|        col_defs.append("PRIMARY KEY (" + ", ".join(pk_cols) + ")")
    76|    conn.execute(f"DROP TABLE IF EXISTS {tmp}")
    77|    conn.execute(f"CREATE TABLE {tmp} (" + ", ".join(col_defs) + ")")
    78|    names = [row["name"] for row in cols]
    79|    select_exprs = [f"CAST({name} AS TEXT)" if name in text_columns else name for name in names]
    80|    conn.execute(f"INSERT INTO {tmp} ({', '.join(names)}) SELECT {', '.join(select_exprs)} FROM {table}")
    81|    conn.execute(f"DROP TABLE {table}")
    82|    conn.execute(f"ALTER TABLE {tmp} RENAME TO {table}")
    83|    if table == "crypto_holdings":
    84|        conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_crypto_holdings_asset_wallet ON crypto_holdings(asset_id, wallet_id)")
    85|
    86|
    87|def _add_missing_columns(conn: Connection, table: str, columns: dict[str, str]) -> None:
    88|    existing = _table_columns(conn, table)
    89|    for name, col_type in columns.items():
    90|        if name not in existing:
    91|            conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {col_type}")
    92|
    93|
    94|def _create_broker_bank_mapping_tables(conn: Connection) -> None:
    95|    conn.executescript(
    96|        """
    97|        CREATE TABLE IF NOT EXISTS instrument_mappings (
    98|            mapping_id TEXT PRIMARY KEY,
    99|            source_name TEXT,
   100|            source_platform TEXT NOT NULL,
   101|            source_account TEXT,
   102|            source_label TEXT NOT NULL,
   103|            normalized_name TEXT NOT NULL,
   104|            isin TEXT,
   105|            ticker TEXT,
   106|            exchange TEXT,
   107|            currency TEXT,
   108|            asset_class TEXT NOT NULL DEFAULT 'other',
   109|            source_instrument_name TEXT,
   110|            source_isin TEXT,
   111|            source_ticker TEXT,
   112|            source_exchange TEXT,
   113|            source_currency TEXT,
   114|            instrument_id TEXT REFERENCES instruments(instrument_id),
   115|            mapping_status TEXT NOT NULL DEFAULT 'needs_manual_review',
   116|            confidence TEXT NOT NULL DEFAULT '0',
   117|            quality_flags_json TEXT,
   118|            notes TEXT,
   119|            created_at TEXT NOT NULL,
   120|            updated_at TEXT
   121|        );
   122|
   123|        CREATE TABLE IF NOT EXISTS platform_account_mappings (
   124|            mapping_id TEXT PRIMARY KEY,
   125|            source_name TEXT,
   126|            source_platform TEXT NOT NULL,
   127|            source_platform_label TEXT,
   128|            source_account_label TEXT NOT NULL,
   129|            normalized_platform TEXT NOT NULL,
   130|            normalized_account_name TEXT NOT NULL,
   131|            source_account_type TEXT,
   132|            account_type TEXT NOT NULL DEFAULT 'other',
   133|            source_currency TEXT,
   134|            currency TEXT,
   135|            platform_id TEXT REFERENCES platforms(platform_id),
   136|            account_id TEXT REFERENCES accounts(account_id),
   137|            internal_platform_id TEXT REFERENCES platforms(platform_id),
   138|            internal_account_id TEXT REFERENCES accounts(account_id),
   139|            mapping_status TEXT NOT NULL DEFAULT 'needs_manual_review',
   140|            quality_flags_json TEXT,
   141|            notes TEXT,
   142|            created_at TEXT NOT NULL,
   143|            updated_at TEXT
   144|        );
   145|
   146|        CREATE TABLE IF NOT EXISTS broker_import_dry_runs (
   147|            dry_run_id TEXT PRIMARY KEY,
   148|            source_name TEXT,
   149|            source_platform TEXT NOT NULL,
   150|            source_file_label TEXT,
   151|            source_file_type TEXT NOT NULL,
   152|            source_hash TEXT,
   153|            source_filename_hash TEXT,
   154|            detected_snapshot_date TEXT,
   155|            snapshot_date_status TEXT NOT NULL DEFAULT 'missing',
   156|            detected_sections_json TEXT NOT NULL,
   157|            field_coverage_json TEXT,
   158|            rows_total INTEGER NOT NULL DEFAULT 0,
   159|            rows_position_candidates INTEGER NOT NULL DEFAULT 0,
   160|            rows_cash_candidates INTEGER NOT NULL DEFAULT 0,
   161|            candidate_positions INTEGER NOT NULL DEFAULT 0,
   162|            candidate_cash_rows INTEGER NOT NULL DEFAULT 0,
   163|            mapped_positions INTEGER NOT NULL DEFAULT 0,
   164|            blocked_positions INTEGER NOT NULL DEFAULT 0,
   165|            warnings_count INTEGER NOT NULL DEFAULT 0,
   166|            errors_count INTEGER NOT NULL DEFAULT 0,
   167|            quality_flags_json TEXT NOT NULL,
   168|            summary_json TEXT NOT NULL DEFAULT '{}',
   169|            created_at TEXT NOT NULL,
   170|            notes TEXT
   171|        );
   172|        """
   173|    )
   174|    _add_missing_columns(conn, "instrument_mappings", {
   175|        "source_name": "TEXT",
   176|        "source_platform": "TEXT",
   177|        "source_account": "TEXT",
   178|        "source_label": "TEXT",
   179|        "normalized_name": "TEXT",
   180|        "isin": "TEXT",
   181|        "ticker": "TEXT",
   182|        "exchange": "TEXT",
   183|        "currency": "TEXT",
   184|        "asset_class": "TEXT DEFAULT 'other'",
   185|        "confidence": "TEXT DEFAULT '0'",
   186|    })
   187|    _add_missing_columns(conn, "platform_account_mappings", {
   188|        "source_platform": "TEXT",
   189|        "normalized_platform": "TEXT",
   190|        "normalized_account_name": "TEXT",
   191|        "account_type": "TEXT DEFAULT 'other'",
   192|        "currency": "TEXT",
   193|        "internal_platform_id": "TEXT",
   194|        "internal_account_id": "TEXT",
   195|    })
   196|    _add_missing_columns(conn, "broker_import_dry_runs", {
   197|        "source_platform": "TEXT",
   198|        "source_filename_hash": "TEXT",
   199|        "detected_snapshot_date": "TEXT",
   200|        "snapshot_date_status": "TEXT DEFAULT 'missing'",
   201|        "candidate_positions": "INTEGER DEFAULT 0",
   202|        "candidate_cash_rows": "INTEGER DEFAULT 0",
   203|        "mapped_positions": "INTEGER DEFAULT 0",
   204|        "blocked_positions": "INTEGER DEFAULT 0",
   205|        "warnings_count": "INTEGER DEFAULT 0",
   206|        "errors_count": "INTEGER DEFAULT 0",
   207|        "summary_json": "TEXT DEFAULT '{}'",
   208|        "session_status": "TEXT DEFAULT 'active'",
   209|        "is_current": "INTEGER DEFAULT 0",
   210|        "archived_at": "TEXT",
   211|        "discarded_at": "TEXT",
   212|    })
   213|
   214|
   215|def _create_broker_import_review_items(conn: Connection) -> None:
   216|    conn.executescript(
   217|        """
   218|        CREATE TABLE IF NOT EXISTS broker_import_review_items (
   219|            review_item_id TEXT PRIMARY KEY,
   220|            dry_run_id TEXT NOT NULL REFERENCES broker_import_dry_runs(dry_run_id),
   221|            source_platform TEXT NOT NULL,
   222|            source_file_type TEXT NOT NULL,
   223|            source_row_ref TEXT NOT NULL,
   224|            row_hash TEXT NOT NULL,
   225|            source_label TEXT,
   226|            normalized_name TEXT,
   227|            detected_asset_class TEXT,
   228|            detected_currency TEXT,
   229|            detected_quantity_present INTEGER NOT NULL DEFAULT 0,
   230|            detected_market_value_present INTEGER NOT NULL DEFAULT 0,
   231|            isin TEXT,
   232|            ticker TEXT,
   233|            exchange TEXT,
   234|            mapped_instrument_id TEXT REFERENCES instruments(instrument_id),
   235|            mapped_account_id TEXT REFERENCES accounts(account_id),
   236|            quality_flags_json TEXT NOT NULL,
   237|            review_status TEXT NOT NULL DEFAULT 'open',
   238|            reviewer_note TEXT,
   239|            created_at TEXT NOT NULL,
   240|            updated_at TEXT
   241|        );
   242|        CREATE INDEX IF NOT EXISTS idx_broker_review_items_dry_run ON broker_import_review_items(dry_run_id);
   243|        CREATE INDEX IF NOT EXISTS idx_broker_review_items_status ON broker_import_review_items(review_status);
   244|        """
   245|    )
   246|    _add_missing_columns(conn, "broker_import_review_items", {
   247|        "import_readiness_status": "TEXT DEFAULT 'not_ready'",
   248|        "reviewer_confirmed": "INTEGER DEFAULT 0",
   249|        "snapshot_date_confirmed": "INTEGER DEFAULT 0",
   250|        "ticker_exchange_confirmed": "INTEGER DEFAULT 0",
   251|        "account_mapping_status": "TEXT DEFAULT 'missing'",
   252|    })
   253|
   254|def _apply_compat_migrations(conn: Connection) -> None:
   255|    _add_missing_instrument_columns(conn)
   256|    for table, text_columns in TEXT_AFFINITY_COLUMNS.items():
   257|        _rebuild_table_with_text_columns(conn, table, text_columns)
   258|    _create_broker_bank_mapping_tables(conn)
   259|    _create_broker_import_review_items(conn)
   260|
   261|

Use offset=261 to continue reading (showing 1-260 of 277 lines)


   261|
   262|def apply_migrations(conn: Connection) -> None:
   263|    conn.executescript(INITIAL_SCHEMA_SQL)
   264|    existing_initial = conn.execute("SELECT 1 FROM schema_migrations WHERE version = 1").fetchone()
   265|    if not existing_initial:
   266|        conn.execute(
   267|            "INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)",
   268|            (1, "001_initial_schema", utc_now(), checksum_sql(INITIAL_SCHEMA_SQL)),
   269|        )
   270|    _apply_compat_migrations(conn)
   271|    existing = conn.execute("SELECT 1 FROM schema_migrations WHERE version = ?", (MIGRATION_VERSION,)).fetchone()
   272|    if not existing:
   273|        conn.execute(
   274|            "INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)",
   275|            (MIGRATION_VERSION, MIGRATION_NAME, utc_now(), checksum_sql(MIGRATION_NAME)),
   276|        )
   277|    conn.commit()
   278|


Explanation:
**Considering database migrations**

I need to add a table and maybe import statuses without needing a schema. I'm thinking about adding helper functions or doing a direct script, but I'll need tests if the code changes. It might be better to patch migrations and schema tests. Writing an import helper module could make it cleaner, although I could just focus on the migration itself. Creating the table will be necessary, and using write/patch seems like the way to go.

patch

MIGRATION_VERSION = 6
MIGRATION_NAME = "006_broker_review_queue_hardening"

MIGRATION_VERSION = 7
MIGRATION_NAME = "007_broker_import_execution_plans"


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py
@@ -6,8 +6,8 @@

 from .schema import INITIAL_SCHEMA_SQL

-MIGRATION_VERSION = 6
-MIGRATION_NAME = "006_broker_review_queue_hardening"
+MIGRATION_VERSION = 7
+MIGRATION_NAME = "007_broker_import_execution_plans"

 INSTRUMENT_OPTIONAL_COLUMNS = {
     "position_category": "TEXT",

/home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py was last read with offset/limit pagination (partial view). Re-read the whole file before overwriting it.


def _create_broker_import_execution_plans(conn: Connection) -> None:
    conn.executescript(
        """
        CREATE TABLE IF NOT EXISTS broker_import_execution_plans (
            execution_plan_id TEXT PRIMARY KEY,
            dry_run_id TEXT NOT NULL REFERENCES broker_import_dry_runs(dry_run_id),
            review_item_id TEXT NOT NULL REFERENCES broker_import_review_items(review_item_id),
            source_platform TEXT NOT NULL,
            target_account_id TEXT NOT NULL REFERENCES accounts(account_id),
            target_instrument_id TEXT NOT NULL REFERENCES instruments(instrument_id),
            transaction_type TEXT NOT NULL DEFAULT 'initial_position_snapshot',
            snapshot_date TEXT,
            payload_status TEXT NOT NULL,
            payload_quality_flags_json TEXT NOT NULL DEFAULT '[]',
            source_row_hash TEXT NOT NULL,
            planned_write_summary_json TEXT NOT NULL DEFAULT '{}',
            execution_status TEXT NOT NULL DEFAULT 'planned',
            transaction_id TEXT REFERENCES transactions(transaction_id),
            created_at TEXT NOT NULL,
            updated_at TEXT,
            executed_at TEXT,
            notes TEXT
        );
        CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_execution_plan_review_item ON broker_import_execution_plans(review_item_id);
        CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_execution_plan_source_hash ON broker_import_execution_plans(source_row_hash, target_account_id, target_instrument_id, transaction_type);
        """
    )

def _apply_compat_migrations(conn: Connection) -> None:
    _add_missing_instrument_columns(conn)
    for table, text_columns in TEXT_AFFINITY_COLUMNS.items():
        _rebuild_table_with_text_columns(conn, table, text_columns)
    _create_broker_bank_mapping_tables(conn)
    _create_broker_import_review_items(conn)
    _create_broker_import_execution_plans(conn)


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py
@@ -251,12 +251,41 @@
         "account_mapping_status": "TEXT DEFAULT 'missing'",
     })

+def _create_broker_import_execution_plans(conn: Connection) -> None:
+    conn.executescript(
+        """
+        CREATE TABLE IF NOT EXISTS broker_import_execution_plans (
+            execution_plan_id TEXT PRIMARY KEY,
+            dry_run_id TEXT NOT NULL REFERENCES broker_import_dry_runs(dry_run_id),
+            review_item_id TEXT NOT NULL REFERENCES broker_import_review_items(review_item_id),
+            source_platform TEXT NOT NULL,
+            target_account_id TEXT NOT NULL REFERENCES accounts(account_id),
+            target_instrument_id TEXT NOT NULL REFERENCES instruments(instrument_id),
+            transaction_type TEXT NOT NULL DEFAULT 'initial_position_snapshot',
+            snapshot_date TEXT,
+            payload_status TEXT NOT NULL,
+            payload_quality_flags_json TEXT NOT NULL DEFAULT '[]',
+            source_row_hash TEXT NOT NULL,
+            planned_write_summary_json TEXT NOT NULL DEFAULT '{}',
+            execution_status TEXT NOT NULL DEFAULT 'planned',
+            transaction_id TEXT REFERENCES transactions(transaction_id),
+            created_at TEXT NOT NULL,
+            updated_at TEXT,
+            executed_at TEXT,
+            notes TEXT
+        );
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_execution_plan_review_item ON broker_import_execution_plans(review_item_id);
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_execution_plan_source_hash ON broker_import_execution_plans(source_row_hash, target_account_id, target_instrument_id, transaction_type);
+        """
+    )
+
 def _apply_compat_migrations(conn: Connection) -> None:
     _add_missing_instrument_columns(conn)
     for table, text_columns in TEXT_AFFINITY_COLUMNS.items():
         _rebuild_table_with_text_columns(conn, table, text_columns)
     _create_broker_bank_mapping_tables(conn)
     _create_broker_import_review_items(conn)
+    _create_broker_import_execution_plans(conn)

 def apply_migrations(conn: Connection) -> None:


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
@@ -15,7 +15,7 @@
 def test_schema_version_recorded() -> None:
     conn = connect_memory()
     apply_migrations(conn)
-    assert get_schema_version(conn) == 6
+    assert get_schema_version(conn) == 7

 def test_decimal_sensitive_columns_use_text_affinity() -> None:

/home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py was modified by sibling subagent '20260515_113332_7cebca' but this agent never read it. Read the file before writing to avoid overwriting the sibling's changes.


    assert {"review_item_id", "dry_run_id", "source_platform", "source_row_ref", "row_hash", "source_label", "normalized_name", "detected_asset_class", "detected_currency", "quality_flags_json", "review_status", "import_readiness_status", "reviewer_confirmed", "snapshot_date_confirmed", "ticker_exchange_confirmed", "account_mapping_status"}.issubset(review_cols)
    execution_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_execution_plans)").fetchall()}
    assert {"execution_plan_id", "dry_run_id", "review_item_id", "source_platform", "target_account_id", "target_instrument_id", "transaction_type", "snapshot_date", "payload_status", "payload_quality_flags_json", "source_row_hash", "planned_write_summary_json", "execution_status", "transaction_id"}.issubset(execution_cols)


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
@@ -42,3 +42,5 @@
     assert {"source_platform", "source_file_type", "source_filename_hash", "detected_snapshot_date", "snapshot_date_status", "candidate_positions", "candidate_cash_rows", "mapped_positions", "blocked_positions", "quality_flags_json", "summary_json", "session_status", "is_current"}.issubset(dry_run_cols)
     review_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_review_items)").fetchall()}
     assert {"review_item_id", "dry_run_id", "source_platform", "source_row_ref", "row_hash", "source_label", "normalized_name", "detected_asset_class", "detected_currency", "quality_flags_json", "review_status", "import_readiness_status", "reviewer_confirmed", "snapshot_date_confirmed", "ticker_exchange_confirmed", "account_mapping_status"}.issubset(review_cols)
+    execution_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_execution_plans)").fetchall()}
+    assert {"execution_plan_id", "dry_run_id", "review_item_id", "source_platform", "target_account_id", "target_instrument_id", "transaction_type", "snapshot_date", "payload_status", "payload_quality_flags_json", "source_row_hash", "planned_write_summary_json", "execution_status", "transaction_id"}.issubset(execution_cols)


Explanation:
**Planning runtime migration**

I need to apply a runtime migration and build an execution plan carefully. I'll write the script without printing values. A snapshot date needs to be parsed, and I might compile it later. First, I need to use python-docx, and I'll utilize Decimal for parsing.

When identifying table rows by ISIN, quantities appear first as numeric labels. I plan to normalize text by removing currency symbols while ensuring to store strings accurately. I also need to compute a unique source row hash from the row text and prepare a summary JSON payload.
**Building transaction schema**

I need to clarify a few points for US ETFs in USD. I see that the detected currency is CHF, which might indicate the market currency, while the source price is in USD. It's essential to establish the planned transaction fields, using trade_date instead of snapshot_date.

For initial_position_snapshot, I might include gross_amount_original if the source provides it. However, I've realized that transaction details include gross_amount_chf and may need to address the contradictory "fx_status missing." I’ll keep price_original in USD and gross_amount_chf in CHF, while ensuring cost is noted as missing.
**Setting up import process**

The user mentioned "imported" or "completed," and I see existing filters do not include "imported," but I can set that up. It seems like the schema might not have any constraints, which is good.

I need to ensure there's an import execution log that includes execution plans, the execution status, and auditing. A backup command could be necessary — I’ll need to check if it can come after step B6a. I should look into the CLI backup and verify the command syntax since I previously used backup-runtime-db output.

terminal

set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python -m jarvis_finance.cli.main migrate
PYTHONPATH=src python - <<'PY'
import json, re, hashlib, uuid
from decimal import Decimal, InvalidOperation
from pathlib import Path
from docx import Document
from jarvis_finance.config.settings import load_settings
from jarvis_finance.storage.database import connect
from jarvis_finance.storage.migrations import apply_migrations, utc_now
from jarvis_finance.audit.log import record_audit_event

repo=Path('/home/agent/.hermes/repos/FinanceManager').resolve()
source_files=list(Path('/home/agent/jarvis_runtime/finance-system/imports').glob('**/*True*Wealth*.docx'))
if len(source_files) != 1:
    raise SystemExit(f'expected exactly one True Wealth file, found {len(source_files)}')
source=source_files[0]
if str(source.resolve()).startswith(str(repo)):
    raise SystemExit('source file is inside repo')

def parse_decimal(text):
    s=' '.join((text or '').replace('\xa0',' ').split())
    s=re.sub(r'^[A-Z]{3}\s+', '', s)
    s=s.replace("'", '').replace('’','').replace(' ', '')
    s=s.replace('%','')
    s=re.sub(r'[^0-9,.-]', '', s)
    if not s:
        raise InvalidOperation('empty')
    if ',' in s and '.' in s:
        s=s.replace(',', '')
    elif ',' in s:
        s=s.replace(',', '.')
    return str(Decimal(s))

def currency(text):
    m=re.search(r'\b(CHF|USD|EUR|GBP)\b', text or '')
    return m.group(1) if m else None

def snapshot_date(doc):
    txt=' '.join(p.text for p in doc.paragraphs)
    for table in doc.tables:
        for row in table.rows[:2]:
            txt += ' ' + ' '.join(c.text for c in row.cells)
    # German month in current True Wealth statement.
    m=re.search(r'per\s+(\d{1,2})\.\s*Mai\s+(\d{4})', txt, re.I)
    if m:
        return f"{m.group(2)}-05-{int(m.group(1)):02d}"
    return None

doc=Document(source)
snap=snapshot_date(doc)
rows_by_isin={}
for ti, table in enumerate(doc.tables):
    for ri, row in enumerate(table.rows):
        cells=[c.text.strip() for c in row.cells]
        joined=' | '.join(cells)
        m=re.search(r'\b[A-Z]{2}[A-Z0-9]{9}\d\b', joined)
        if m:
            rows_by_isin[m.group(0)]={'table':ti,'row':ri,'cells':cells,'joined':joined}

conn=connect(load_settings().db_path); apply_migrations(conn)
ready=conn.execute("""
SELECT * FROM broker_import_review_items
WHERE source_platform='True Wealth' AND import_readiness_status='ready_for_import'
ORDER BY updated_at, review_item_id
""").fetchall()
checked=ready_count=blocked=missing_fx=cost_uncertain=missing_qty=missing_acct=missing_inst=0
for item in ready:
    checked += 1
    flags=[]
    isin=item['isin']
    account_id=item['mapped_account_id']; instrument_id=item['mapped_instrument_id']
    if not account_id: flags.append('missing_account_mapping'); missing_acct += 1
    if not instrument_id: flags.append('missing_instrument_mapping'); missing_inst += 1
    src=rows_by_isin.get(isin)
    planned={}
    status='ready'
    if not src:
        flags.append('missing_source_row'); status='blocked'
    else:
        cells=src['cells']
        # True Wealth row layout after label: quantity duplicate, price duplicate, CHF market value duplicate.
        try:
            quantity=parse_decimal(cells[1] if len(cells)>1 else '')
        except Exception:
            quantity=None; flags.append('missing_quantity'); missing_qty += 1; status='blocked'
        price_original=None; price_currency=None
        try:
            price_original=parse_decimal(cells[3] if len(cells)>3 else '')
            price_currency=currency(cells[3] if len(cells)>3 else '')
        except Exception:
            flags.append('missing_price_original')
        market_value_chf=None
        try:
            market_value_chf=parse_decimal(cells[5] if len(cells)>5 else '')
        except Exception:
            flags.append('missing_market_value_chf')
        # No cost basis or explicit FX is present in this report extract.
        flags += ['cost_basis_uncertain','fx_missing']
        missing_fx += 1; cost_uncertain += 1
        source_row_hash=hashlib.sha256(src['joined'].encode()).hexdigest()
        planned={
            'transaction_type':'initial_position_snapshot',
            'trade_date': snap,
            'snapshot_date': snap,
            'account_id': account_id,
            'instrument_id': instrument_id,
            'quantity': quantity,
            'currency_original': price_currency or item['detected_currency'],
            'price_original': price_original,
            'gross_amount_chf': market_value_chf,
            'cost_basis_quality':'uncertain',
            'fx_rate_to_chf': None,
            'fx_status':'missing',
            'source_type':'broker_import_reviewed_snapshot',
            'source_id': item['review_item_id'],
            'source_row_ref': item['source_row_ref'],
        }
        if not snap:
            flags.append('missing_snapshot_date'); status='blocked'
        if not quantity:
            status='blocked'
    if status != 'blocked' and not (missing_acct or missing_inst):
        status='ready'
        ready_count += 1
    else:
        blocked += 1
    plan_id='plan-'+item['review_item_id']
    now=utc_now()
    conn.execute("""
    INSERT INTO broker_import_execution_plans(execution_plan_id,dry_run_id,review_item_id,source_platform,target_account_id,target_instrument_id,transaction_type,snapshot_date,payload_status,payload_quality_flags_json,source_row_hash,planned_write_summary_json,execution_status,created_at,updated_at,notes)
    VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
    ON CONFLICT(review_item_id) DO UPDATE SET
      target_account_id=excluded.target_account_id,
      target_instrument_id=excluded.target_instrument_id,
      snapshot_date=excluded.snapshot_date,
      payload_status=excluded.payload_status,
      payload_quality_flags_json=excluded.payload_quality_flags_json,
      source_row_hash=excluded.source_row_hash,
      planned_write_summary_json=excluded.planned_write_summary_json,
      updated_at=excluded.updated_at,
      notes=excluded.notes
    """, (plan_id,item['dry_run_id'],item['review_item_id'],item['source_platform'],account_id or '',instrument_id or '', 'initial_position_snapshot', snap, status, json.dumps(sorted(set(flags))), (source_row_hash if src else item['row_hash']), json.dumps(planned, sort_keys=True), 'planned', now, now, 'B6a final dry-run execution plan; runtime-only payload.'))
    record_audit_event(conn, source='broker_import_execution_plan', action='create_or_update_execution_plan', entity_type='broker_import_execution_plan', entity_id=plan_id, old_values={}, new_values={'review_item_id':item['review_item_id'],'payload_status':status,'quality_flags':sorted(set(flags))}, user_text_note='B6a final dry-run execution plan created/updated; no productive import.', confirmed=True, created_by='phase_b6a')
conn.commit()
print(json.dumps({'ready_review_items_checked':checked,'payloads_ready':ready_count,'payloads_blocked':blocked,'items_missing_fx':missing_fx,'items_cost_basis_uncertain':cost_uncertain,'items_missing_quantity':missing_qty,'items_missing_account_mapping':missing_acct,'items_missing_instrument_mapping':missing_inst,'source_files_read_temporarily':1,'source_files_in_repo':0}, sort_keys=True))
PY


Output:
SCHEMA_VERSION=7
{"items_cost_basis_uncertain": 3, "items_missing_account_mapping": 0, "items_missing_fx": 3, "items_missing_instrument_mapping": 0, "items_missing_quantity": 0, "payloads_blocked": 0, "payloads_ready": 3, "ready_review_items_checked": 3, "source_files_in_repo": 0, "source_files_read_temporarily": 1}


Explanation:
**Confirming backup process**

I realize I need to back up my files with a reliable command. It seems the code has changed with the migration, so I might need to commit those changes later. I should look for a backup command to ensure everything is safe. Also, I want to verify if the backup CLI has options or help commands that can guide me through this process effectively. Let’s make sure I’m covered!

todo

B6a Preflight und Runtime-Schema/Quellstruktur prüfen

Execution Plan Runtime-only für 3 True-Wealth ready Items erzeugen und validieren

B6b Backup erzeugen/verifizieren und genau 1 Payload produktiv als Ledger Initial Snapshot importieren


Nachkontrolle Dashboard/Data Quality/Audit/Git-Safety/Tests; ggf. Commit+Push bei Codeänderung


terminal

cd /home/agent/.hermes/repos/FinanceManager && PYTHONPATH=src python -m jarvis_finance.cli.main --help | sed -n '1,200p' && PYTHONPATH=src python -m jarvis_finance.cli.main verify-backup --help | sed -n '1,160p'


Output:
usage: finance [-h]
               {init-db,migrate,git-safety-scan,update-crypto-prices,check-data-quality,backup-runtime-db,restore-runtime-db,verify-backup}
               ...

positional arguments:
  {init-db,migrate,git-safety-scan,update-crypto-prices,check-data-quality,backup-runtime-db,restore-runtime-db,verify-backup}

options:
  -h, --help            show this help message and exit
usage: finance verify-backup [-h] --file FILE

options:
  -h, --help   show this help message and exit
  --file FILE


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python -m jarvis_finance.cli.main backup-runtime-db > /tmp/b6_backup_out.json
python - <<'PY'
import json, re, subprocess
from pathlib import Path
text=Path('/tmp/b6_backup_out.json').read_text().strip()
# Output is JSON from CLI.
data=json.loads(text)
path=data.get('backup_file') or data.get('path') or data.get('file')
if not path:
    # fallback regex
    m=re.search(r'(/[^\s"}]+\.db(?:\.gz)?)', text); path=m.group(1) if m else None
if not path:
    raise SystemExit('backup path not found')
res=subprocess.run(['python','-m','jarvis_finance.cli.main','verify-backup','--file',path], cwd='/home/agent/.hermes/repos/FinanceManager', env={**__import__('os').environ,'PYTHONPATH':'src'}, text=True, capture_output=True, check=True)
verify=json.loads(res.stdout)
print(json.dumps({'backup_created': True, 'backup_verified': True, 'checksum_ok': bool(verify.get('checksum_ok', verify.get('verified', True))), 'backup_dir': str(Path(path).parent)}, sort_keys=True))
PY


Output:
Traceback (most recent call last):
  File "<stdin>", line 5, in <module>
  File "/home/agent/.local/share/uv/python/cpython-3.11.15-linux-x86_64-gnu/lib/python3.11/json/__init__.py", line 346, in loads
    return _default_decoder.decode(s)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/agent/.local/share/uv/python/cpython-3.11.15-linux-x86_64-gnu/lib/python3.11/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/agent/.local/share/uv/python/cpython-3.11.15-linux-x86_64-gnu/lib/python3.11/json/decoder.py", line 355, in raw_decode
    raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)


     1|RUNTIME_DB_BACKUP backup_file=/home/agent/jarvis_runtime/finance-system/backups/finance-runtime-db-20260515T102041Z.sqlite3 checksum_file=/home/agent/jarvis_runtime/finance-system/backups/finance-runtime-db-20260515T102041Z.sqlite3.sha256 sha256=6116c198449544dc28b263882700e12671342028cd8bf943e2eed0f785a599b6 size_bytes=430080
     2|


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
python - <<'PY'
import re, subprocess, json, os
from pathlib import Path
text=Path('/tmp/b6_backup_out.json').read_text()
m=re.search(r'backup_file=([^\s]+)', text)
if not m: raise SystemExit('backup_file parse failed')
path=m.group(1)
res=subprocess.run(['python','-m','jarvis_finance.cli.main','verify-backup','--file',path], cwd='/home/agent/.hermes/repos/FinanceManager', env={**os.environ,'PYTHONPATH':'src'}, text=True, capture_output=True, check=True)
checksum_ok=('checksum_ok=True' in res.stdout) or ('CHECKSUM_OK' in res.stdout) or ('verified' in res.stdout.lower()) or (res.returncode==0)
print(json.dumps({'backup_created': True, 'backup_verified': True, 'checksum_ok': checksum_ok, 'backup_dir': str(Path(path).parent)}, sort_keys=True))
PY


Output: {"backup_created": true, "backup_dir": "/home/agent/jarvis_runtime/finance-system/backups", "backup_verified": true, "checksum_ok": true}


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python - <<'PY'
import json, uuid
from jarvis_finance.config.settings import load_settings
from jarvis_finance.storage.database import connect
from jarvis_finance.storage.migrations import apply_migrations, utc_now
from jarvis_finance.audit.log import record_audit_event
conn=connect(load_settings().db_path); apply_migrations(conn)
before_tx=conn.execute("SELECT COUNT(*) AS n FROM transactions").fetchone()['n']
before_pos=conn.execute("SELECT COUNT(*) AS n FROM positions_snapshot").fetchone()['n']
before_cash=conn.execute("SELECT COUNT(*) AS n FROM cash_balances").fetchone()['n']
plan=conn.execute("""
SELECT * FROM broker_import_execution_plans
WHERE source_platform='True Wealth' AND payload_status='ready' AND execution_status='planned'
ORDER BY created_at, execution_plan_id
LIMIT 1
""").fetchone()
if not plan:
    raise SystemExit('no planned ready payload')
# Idempotency: same execution plan/review/source hash must not import twice.
existing=conn.execute("""
SELECT transaction_id FROM transactions
WHERE source_type='broker_import_reviewed_snapshot'
  AND (source_id=? OR source_id=? OR row_hash=? OR external_transaction_id=? )
""", (plan['execution_plan_id'], plan['review_item_id'], plan['source_row_hash'], plan['execution_plan_id'])).fetchone()
if existing:
    imported=False; tx_id=existing['transaction_id']
else:
    payload=json.loads(plan['planned_write_summary_json'])
    tx_id='tx-'+str(uuid.uuid4())
    now=utc_now()
    conn.execute("""
    INSERT INTO transactions(
      transaction_id, transaction_type, account_id, instrument_id, trade_date, settlement_date,
      quantity, price_original, gross_amount_original, fee_original, tax_original, net_amount_original,
      currency_original, fx_rate_to_chf, fx_source, fx_status, gross_amount_chf, fee_chf, tax_chf, net_amount_chf,
      source_type, source_id, external_transaction_id, row_hash, is_confirmed, quality_status, notes, created_at, updated_at
    ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
    """, (
        tx_id, 'initial_position_snapshot', payload['account_id'], payload['instrument_id'], payload['trade_date'], payload['trade_date'],
        payload['quantity'], payload.get('price_original'), None, None, None, None,
        payload.get('currency_original'), payload.get('fx_rate_to_chf'), 'missing_in_source', payload.get('fx_status','missing'), payload.get('gross_amount_chf'), None, None, None,
        'broker_import_reviewed_snapshot', plan['execution_plan_id'], plan['execution_plan_id'], plan['source_row_hash'], 1, 'warning',
        'Initial Snapshot from reviewed True Wealth dry-run execution plan; cost basis/FX incomplete; no positions_snapshot direct write.', now, now
    ))
    conn.execute("UPDATE broker_import_execution_plans SET execution_status='imported', transaction_id=?, executed_at=?, updated_at=?, notes=? WHERE execution_plan_id=?", (tx_id, now, now, 'B6b mini-import executed for exactly one reviewed payload.', plan['execution_plan_id']))
    conn.execute("UPDATE broker_import_review_items SET review_status='imported', reviewer_note=?, updated_at=? WHERE review_item_id=?", ('B6b mini-import completed for exactly one reviewed payload.', now, plan['review_item_id']))
    record_audit_event(conn, source='broker_import_execution_plan', action='execute_mini_import', entity_type='transaction', entity_id=tx_id, old_values={}, new_values={'execution_plan_id':plan['execution_plan_id'], 'review_item_id':plan['review_item_id'], 'transaction_type':'initial_position_snapshot', 'quality_status':'warning'}, user_text_note='B6b productive mini-import of exactly one True Wealth reviewed initial snapshot. No values emitted to chat.', confirmed=True, created_by='phase_b6b')
    record_audit_event(conn, source='manual_review_queue', action='mark_review_item_imported', entity_type='broker_import_review_item', entity_id=plan['review_item_id'], old_values={'review_status':'resolved'}, new_values={'review_status':'imported','transaction_id':tx_id}, user_text_note='B6b linked imported transaction to review item; no mass import.', confirmed=True, created_by='phase_b6b')
    imported=True
conn.commit()
after_tx=conn.execute("SELECT COUNT(*) AS n FROM transactions").fetchone()['n']
after_pos=conn.execute("SELECT COUNT(*) AS n FROM positions_snapshot").fetchone()['n']
after_cash=conn.execute("SELECT COUNT(*) AS n FROM cash_balances").fetchone()['n']
# Ledger computability check without printing quantity.
ledger_rows=conn.execute("SELECT COUNT(*) AS n FROM transactions WHERE transaction_type='initial_position_snapshot' AND source_type='broker_import_reviewed_snapshot'").fetchone()['n']
other_ready=conn.execute("SELECT COUNT(*) AS n FROM broker_import_execution_plans WHERE source_platform='True Wealth' AND payload_status='ready' AND execution_status='planned'").fetchone()['n']
review_status=conn.execute("SELECT review_status, import_readiness_status FROM broker_import_review_items WHERE review_item_id=?", (plan['review_item_id'],)).fetchone()
audits=conn.execute("SELECT COUNT(*) AS n FROM audit_log WHERE entity_id IN (?, ?) OR json_extract(new_values_json, '$.execution_plan_id')=?", (tx_id, plan['review_item_id'], plan['execution_plan_id'])).fetchone()['n']
print(json.dumps({'mini_import_performed': imported, 'new_transactions': after_tx-before_tx, 'positions_snapshot_delta': after_pos-before_pos, 'cash_balances_delta': after_cash-before_cash, 'ledger_initial_snapshot_count': ledger_rows, 'remaining_ready_planned_payloads': other_ready, 'imported_review_status': dict(review_status), 'audit_logs_related_min': audits}, sort_keys=True))
PY


Output: {"audit_logs_related_min": 9, "cash_balances_delta": 0, "imported_review_status": {"import_readiness_status": "ready_for_import", "review_status": "imported"}, "ledger_initial_snapshot_count": 1, "mini_import_performed": true, "new_transactions": 1, "positions_snapshot_delta": 0, "remaining_ready_planned_payloads": 2}


Explanation:
**Planning database checks**

I’m considering a dashboard check, maybe needing to inspect the import wizard and run a query. Data quality seems important, and I should compile tests, though they might fail due to a schema version update. I'll need to check the git status on code changes and commit afterward, but first, I should run validation. The migration from version 6 to 7 seems necessary, noting that the existing DB has version 6 while the new memory only creates 1 and 7.


     1|from __future__ import annotations
     2|
     3|from jarvis_finance.dashboard import data
     4|from jarvis_finance.dashboard.components.tables import show_table
     5|from jarvis_finance.equity.manage import add_equity_initial_snapshot, add_equity_transaction, get_equity_management_options
     6|
     7|PAGE_TITLE = "Equity/ETF Manage"
     8|
     9|
    10|def _account_label(row: dict[str, str]) -> str:
    11|    return f"{row['platform_name']} / {row['account_name']} [{row['currency']} ]"
    12|
    13|
    14|def _instrument_label(row: dict[str, str]) -> str:
    15|    suffix = row.get("isin") or row.get("ticker") or row["instrument_id"]
    16|    return f"{row['name']} ({suffix})"
    17|
    18|
    19|def _selected(rows: list[dict[str, str]], label: str | None, label_fn) -> dict[str, str] | None:
    20|    if not label:
    21|        return None
    22|    for row in rows:
    23|        if label_fn(row) == label:
    24|            return row
    25|    return None
    26|
    27|
    28|def render(st, conn) -> None:
    29|    st.title("Equity/ETF Manage")
    30|    st.caption("Auditierbare manuelle Aktien-/ETF-Erfassung. Dashboard liest lokal; keine Broker-Dateien oder Live-APIs beim Rendern.")
    31|    st.warning("Keine stillen Überschreibungen: Initial Snapshots, Transaktionen und Korrekturen werden als bestätigte Ledger-Ereignisse mit Audit-Log gespeichert.")
    32|
    33|    options = get_equity_management_options(conn)
    34|    account_labels = [_account_label(row) for row in options["accounts"]]
    35|    instrument_labels = [_instrument_label(row) for row in options["instruments"]]
    36|    tabs = st.tabs(["Hinzufügen", "Kaufen", "Partial Sell", "Full Sell", "Dividende", "ETF-Ausschüttung", "Fee", "Korrigieren", "Review"])
    37|
    38|    with tabs[0]:
    39|        st.subheader("Aktie/ETF als Initial Snapshot hinzufügen")
    40|        account_label = st.selectbox("Depot/Konto", account_labels, key="equity_add_account") if account_labels else None
    41|        asset_class = st.selectbox("Assetklasse", ["stock", "ETF"], key="equity_add_class")
    42|        name = st.text_input("Name", key="equity_add_name")
    43|        isin = st.text_input("ISIN (primäre Identifikation)", key="equity_add_isin")
    44|        ticker = st.text_input("Ticker optional", key="equity_add_ticker")
    45|        exchange = st.text_input("Exchange optional", key="equity_add_exchange")
    46|        currency = st.selectbox("Währung", ["CHF", "EUR", "USD"], key="equity_add_currency")
    47|        quantity = st.text_input("Menge als Decimal-Text", key="equity_add_qty")
    48|        cost_basis = st.text_input("Einstandspreis / Cost Basis Originalwährung", key="equity_add_cost")
    49|        fx_rate = st.text_input("FX-Kurs zu CHF (bei EUR/USD)", key="equity_add_fx")
    50|        fx_source = st.text_input("FX-Quelle", key="equity_add_fx_source")
    51|        fx_status = st.selectbox("FX-Status", ["ok", "missing"], key="equity_add_fx_status")
    52|        snapshot_date = st.date_input("Snapshot-Datum", key="equity_add_date")
    53|        category = st.selectbox("Kategorie", ["Core", "Opportunity", "Unknown"], key="equity_add_category")
    54|        note = st.text_area("Notiz (Pflicht bei unvollständiger Historie)", key="equity_add_note")
    55|        with st.expander("ETF-spezifische Felder"):
    56|            ter = st.text_input("TER optional", key="equity_add_ter")
    57|            distribution_policy = st.text_input("ausschüttend/thesaurierend optional", key="equity_add_distribution")
    58|            index_name = st.text_input("Index optional", key="equity_add_index")
    59|            fund_domicile = st.text_input("Fondsdomizil optional", key="equity_add_domicile")
    60|            benchmark = st.text_input("Benchmark optional", key="equity_add_benchmark")
    61|        confirm = st.checkbox("Ich bestätige diese einzelne Änderung", key="equity_add_confirm")
    62|        selected_account = _selected(options["accounts"], account_label, _account_label)
    63|        st.write({"Aktion": "initial_position_snapshot", "Name": name, "ISIN": isin or "<fehlt>", "Ticker": ticker or "<fehlt>", "Währung": currency, "Datum": str(snapshot_date), "Hinweis": "Keine Massenänderung; Speicherung erst nach Bestätigung."})
    64|        if st.button("Bestätigt speichern", key="equity_add_save", type="primary"):
    65|            try:
    66|                if not selected_account:
    67|                    raise ValueError("account is required")
    68|                result = add_equity_initial_snapshot(
    69|                    conn,
    70|                    account_id=selected_account["account_id"],
    71|                    asset_class=asset_class,
    72|                    name=name,
    73|                    isin=isin,
    74|                    ticker=ticker,
    75|                    exchange=exchange,
    76|                    currency=currency,
    77|                    quantity_text=quantity,
    78|                    cost_basis_original_text=cost_basis,
    79|                    snapshot_date=str(snapshot_date),
    80|                    category=category,
    81|                    note=note,
    82|                    fx_rate_to_chf_text=fx_rate or None,
    83|                    fx_source=fx_source or None,
    84|                    fx_status=fx_status,
    85|                    ter_text=ter or None,
    86|                    distribution_policy=distribution_policy or None,
    87|                    index_name=index_name or None,
    88|                    fund_domicile=fund_domicile or None,
    89|                    benchmark=benchmark or None,
    90|                    confirm=bool(confirm),
    91|                )
    92|                st.success("Initial Snapshot gespeichert und auditierbar erfasst.")
    93|                st.write({"instrument_id": result.instrument_id, "transaction_id": result.transaction_id, "warnings": result.warnings})
    94|            except Exception as exc:
    95|                st.error(f"Speichern abgelehnt: {exc}")
    96|
    97|    def transaction_form(tab, title: str, tx_type: str) -> None:
    98|        with tab:
    99|            st.subheader(title)
   100|            account_label = st.selectbox("Depot/Konto", account_labels, key=f"equity_{tx_type}_account") if account_labels else None
   101|            instrument_label = st.selectbox("Instrument", instrument_labels, key=f"equity_{tx_type}_instrument") if instrument_labels else None
   102|            date_value = st.date_input("Datum", key=f"equity_{tx_type}_date")
   103|            qty_default = "0" if tx_type in {"dividend", "etf_distribution", "fee"} else ""
   104|            quantity = st.text_input("Menge als Decimal-Text", value=qty_default, key=f"equity_{tx_type}_qty")
   105|            gross = st.text_input("Brutto/Cost/Proceeds Originalwährung", key=f"equity_{tx_type}_gross")
   106|            currency = st.selectbox("Währung", ["CHF", "EUR", "USD"], key=f"equity_{tx_type}_currency")
   107|            fx_rate = st.text_input("FX-Kurs zu CHF (bei EUR/USD)", key=f"equity_{tx_type}_fx")
   108|            fx_source = st.text_input("FX-Quelle", key=f"equity_{tx_type}_fx_source")
   109|            fx_status = st.selectbox("FX-Status", ["ok", "missing"], key=f"equity_{tx_type}_fx_status")
   110|            note = st.text_area("Notiz / Grund", key=f"equity_{tx_type}_note")
   111|            confirm = st.checkbox("Ich bestätige diese Änderung", key=f"equity_{tx_type}_confirm")
   112|            selected_account = _selected(options["accounts"], account_label, _account_label)
   113|            selected_instrument = _selected(options["instruments"], instrument_label, _instrument_label)
   114|            st.write({"Aktion": tx_type, "Instrument": instrument_label, "Datum": str(date_value), "FX-Status": fx_status})
   115|            if st.button("Bestätigt speichern", key=f"equity_{tx_type}_save"):
   116|                try:
   117|                    if not selected_account or not selected_instrument:
   118|                        raise ValueError("account and instrument are required")
   119|                    result = add_equity_transaction(
   120|                        conn,
   121|                        account_id=selected_account["account_id"],
   122|                        instrument_id=selected_instrument["instrument_id"],
   123|                        transaction_type=tx_type,
   124|                        trade_date=str(date_value),
   125|                        quantity_text=quantity,
   126|                        gross_amount_original_text=gross,
   127|                        currency=currency,
   128|                        fx_rate_to_chf_text=fx_rate or None,
   129|                        fx_source=fx_source or None,
   130|                        fx_status=fx_status,
   131|                        note=note,
   132|                        confirm=bool(confirm),
   133|                    )
   134|                    st.success("Transaktion gespeichert und auditierbar erfasst.")
   135|                    st.write({"transaction_id": result.transaction_id, "warnings": result.warnings})
   136|                except Exception as exc:
   137|                    st.error(f"Speichern abgelehnt: {exc}")
   138|
   139|    transaction_form(tabs[1], "Kauf erfassen", "buy")
   140|    transaction_form(tabs[2], "Teilverkauf erfassen", "partial_sell")
   141|    transaction_form(tabs[3], "Vollverkauf erfassen", "full_sell")
   142|    transaction_form(tabs[4], "Dividende erfassen", "dividend")
   143|    transaction_form(tabs[5], "ETF-Ausschüttung erfassen", "etf_distribution")
   144|    transaction_form(tabs[6], "Fee erfassen", "fee")
   145|    transaction_form(tabs[7], "Korrektur als manual_adjustment", "manual_adjustment")
   146|
   147|    with tabs[8]:
   148|        st.subheader("Aktuelle Equity/ETF Übersicht")
   149|        show_table(st, data.get_positions(conn))
   150|


Explanation:
**Investigating position discrepancies**

It looks like the page is getting positions from a snapshot that it likely won't recognize since there’s no direct snapshot available. The user mentions that positions from the ledger are calculable, and the dashboard page seems to recognize these imported positions. I think the current page is using get_positions, but it might not be connected to the ledger. I need to inspect the data from get_positions to clarify this situation.


     1|from __future__ import annotations
     2|
     3|import json
     4|from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
     5|from pathlib import Path
     6|from sqlite3 import Connection
     7|
     8|from jarvis_finance.config.settings import Settings
     9|from jarvis_finance.crypto.holdings import calculate_crypto_holdings
    10|from jarvis_finance.ledger.cash import calculate_cash_balances
    11|from jarvis_finance.ledger.positions import calculate_positions
    12|from jarvis_finance.market.providers import is_price_stale
    13|
    14|MONEY = Decimal("0.01")
    15|
    16|
    17|def d(value: object, default: str = "0") -> Decimal:
    18|    if value is None or str(value).strip() == "":
    19|        return Decimal(default)
    20|    try:
    21|        return Decimal(str(value))
    22|    except InvalidOperation:
    23|        return Decimal(default)
    24|
    25|
    26|def dec_text(value: Decimal | int | str | None, places: str | None = None) -> str:
    27|    if value is None:
    28|        return ""
    29|    value = d(value)
    30|    if places:
    31|        value = value.quantize(Decimal(places), rounding=ROUND_HALF_UP)
    32|    return format(value, "f")
    33|
    34|
    35|def money_text(value: Decimal | None, currency: str = "CHF") -> str:
    36|    if value is None:
    37|        return f"n/a {currency}"
    38|    return f"{dec_text(value, '0.01')} {currency}"
    39|
    40|
    41|def rowdicts(rows) -> list[dict[str, str]]:
    42|    return [{k: ("" if row[k] is None else str(row[k])) for k in row.keys()} for row in rows]
    43|
    44|
    45|def db_exists(path: str | Path) -> bool:
    46|    return Path(path).expanduser().exists()
    47|
    48|
    49|def runtime_outside_repo(settings: Settings, repo_root: Path) -> bool:
    50|    try:
    51|        settings.runtime_paths.base_dir.resolve().relative_to(repo_root.resolve())
    52|        return False
    53|    except ValueError:
    54|        return True
    55|
    56|
    57|def latest_crypto_price(conn: Connection, asset_id: str, currency: str = "CHF", *, max_age_seconds: int = 86_400) -> dict[str, str] | None:
    58|    row = conn.execute(
    59|        """
    60|        SELECT price, price_currency, fetched_at, provider_timestamp, quality_status, provider
    61|        FROM crypto_prices
    62|        WHERE asset_id=? AND price_currency=?
    63|        ORDER BY COALESCE(provider_timestamp, fetched_at, '') DESC, fetched_at DESC LIMIT 1
    64|        """,
    65|        (asset_id, currency.upper()),
    66|    ).fetchone()
    67|    if not row:
    68|        return None
    69|    item = dict(row)
    70|    ts = item.get("provider_timestamp") or item.get("fetched_at")
    71|    if item.get("quality_status") == "fresh" and is_price_stale(ts, max_age_seconds=max_age_seconds):
    72|        item["quality_status"] = "stale"
    73|    return item
    74|
    75|
    76|def get_crypto_overview(conn: Connection, *, price_max_age_seconds: int = 86_400) -> list[dict[str, str]]:
    77|    holdings = calculate_crypto_holdings(conn)
    78|    rows: list[dict[str, str]] = []
    79|    for asset_id, holding in sorted(holdings.total_by_asset.items()):
    80|        qty = holding.quantity
    81|        asset = conn.execute("SELECT * FROM crypto_assets WHERE asset_id=?", (asset_id,)).fetchone()
    82|        if not asset:
    83|            continue
    84|        price = latest_crypto_price(conn, asset_id, "CHF", max_age_seconds=price_max_age_seconds)
    85|        price_dec = d(price["price"]) if price and price["price"] else None
    86|        value = qty * price_dec if price_dec is not None else None
    87|        rows.append({
    88|            "asset_id": asset_id,
    89|            "coin_name": asset["coin_name"],
    90|            "symbol": asset["symbol"],
    91|            "coingecko_id": asset["coingecko_id"] or "",
    92|            "quantity": dec_text(qty),
    93|            "latest_price_chf": dec_text(price_dec, "0.01") if price_dec is not None else "",
    94|            "market_value_chf": dec_text(value, "0.01") if value is not None else "",
    95|            "price_quality": price["quality_status"] if price else "missing",
    96|            "fetched_at": price["fetched_at"] if price else "",
    97|        })
    98|    return rows
    99|
   100|
   101|def get_crypto_by_wallet(conn: Connection, *, price_max_age_seconds: int = 86_400) -> list[dict[str, str]]:
   102|    holdings = calculate_crypto_holdings(conn)
   103|    rows: list[dict[str, str]] = []
   104|    for (wallet_id, asset_id), holding in sorted(holdings.wallet_holdings.items()):
   105|        wallet = conn.execute("SELECT wallet_name, wallet_type, platform_provider FROM crypto_wallets WHERE wallet_id=?", (wallet_id,)).fetchone()
   106|        asset = conn.execute("SELECT coin_name, symbol, coingecko_id FROM crypto_assets WHERE asset_id=?", (asset_id,)).fetchone()
   107|        price = latest_crypto_price(conn, asset_id, "CHF", max_age_seconds=price_max_age_seconds)
   108|        price_dec = d(price["price"]) if price and price["price"] else None
   109|        value = holding.quantity * price_dec if price_dec is not None else None
   110|        rows.append({
   111|            "wallet_id": wallet_id,
   112|            "wallet_name": wallet["wallet_name"] if wallet else wallet_id,
   113|            "wallet_type": wallet["wallet_type"] if wallet else "",
   114|            "provider": wallet["platform_provider"] if wallet else "",
   115|            "coin_name": asset["coin_name"] if asset else asset_id,
   116|            "symbol": asset["symbol"] if asset else "",
   117|            "coingecko_id": asset["coingecko_id"] if asset else "",
   118|            "quantity": dec_text(holding.quantity),
   119|            "market_value_chf": dec_text(value, "0.01") if value is not None else "",
   120|            "wallet_value_chf": dec_text(value, "0.01") if value is not None else "",
   121|            "price_quality": price["quality_status"] if price else "missing",
   122|            "price_fetched_at": price["fetched_at"] if price else "",
   123|            "verification_status": holding.verification_status,
   124|            "last_verified_at": holding.last_verified_at or "",
   125|        })
   126|    return rows
   127|
   128|
   129|def get_wallets(conn: Connection) -> list[dict[str, str]]:
   130|    return rowdicts(conn.execute(
   131|        """
   132|        SELECT wallet_id, wallet_name, wallet_type, platform_provider AS provider,
   133|               network_chain, last_verified_at, is_active, notes
   134|        FROM crypto_wallets ORDER BY wallet_name
   135|        """
   136|    ).fetchall())
   137|
   138|
   139|def get_latest_crypto_transactions(conn: Connection, limit: int = 20) -> list[dict[str, str]]:
   140|    return rowdicts(conn.execute(
   141|        """
   142|        SELECT ct.transaction_datetime, ct.transaction_type, ca.symbol, ct.quantity, ct.fee_quantity,
   143|               fw.wallet_name AS from_wallet, tw.wallet_name AS to_wallet, ct.currency_original,
   144|               ct.amount_chf, ct.confirmation_status, ct.source
   145|        FROM crypto_transactions ct
   146|        LEFT JOIN crypto_assets ca ON ca.asset_id=ct.asset_id
   147|        LEFT JOIN crypto_wallets fw ON fw.wallet_id=ct.from_wallet_id
   148|        LEFT JOIN crypto_wallets tw ON tw.wallet_id=ct.to_wallet_id
   149|        ORDER BY ct.transaction_datetime DESC, ct.created_at DESC LIMIT ?
   150|        """, (limit,)
   151|    ).fetchall())
   152|
   153|
   154|def get_platform_overview(conn: Connection) -> list[dict[str, str]]:
   155|    return rowdicts(conn.execute(
   156|        """
   157|        SELECT p.name AS platform, p.platform_type, a.account_name, a.account_type, a.currency,
   158|               a.performance_included, a.is_active
   159|        FROM accounts a JOIN platforms p ON p.platform_id=a.platform_id
   160|        ORDER BY p.name, a.account_name
   161|        """
   162|    ).fetchall())
   163|
   164|
   165|def get_cash_overview(conn: Connection) -> list[dict[str, str]]:
   166|    result = calculate_cash_balances(conn)
   167|    rows: list[dict[str, str]] = []
   168|    for (account_id, currency), bal in sorted(result.balances.items()):
   169|        row = conn.execute(
   170|            "SELECT a.account_name, p.name AS platform FROM accounts a JOIN platforms p ON p.platform_id=a.platform_id WHERE a.account_id=?",
   171|            (account_id,),
   172|        ).fetchone()
   173|        rows.append({
   174|            "platform": row["platform"] if row else "",
   175|            "account_name": row["account_name"] if row else account_id,
   176|            "currency": currency,
   177|            "amount_original": dec_text(bal.amount_original, "0.01"),
   178|            "amount_chf": dec_text(bal.amount_chf, "0.01") if bal.amount_chf is not None else "",
   179|            "quality_status": bal.quality_status,
   180|        })
   181|    return rows
   182|
   183|
   184|def get_positions(conn: Connection) -> list[dict[str, str]]:
   185|    result = calculate_positions(conn)
   186|    rows: list[dict[str, str]] = []
   187|    for (account_id, instrument_id), pos in sorted(result.positions.items()):
   188|        row = conn.execute(
   189|            """
   190|            SELECT a.account_name, p.name AS platform, i.name, i.ticker, i.isin, i.asset_class, i.currency
   191|            FROM accounts a JOIN platforms p ON p.platform_id=a.platform_id
   192|            JOIN instruments i ON i.instrument_id=? WHERE a.account_id=?
   193|            """,
   194|            (instrument_id, account_id),
   195|        ).fetchone()
   196|        rows.append({
   197|            "platform": row["platform"] if row else "",
   198|            "account_name": row["account_name"] if row else account_id,
   199|            "asset_class": row["asset_class"] if row else "",
   200|            "name": row["name"] if row else instrument_id,
   201|            "ticker": row["ticker"] if row else "",
   202|            "isin": row["isin"] if row else "",
   203|            "quantity": dec_text(pos.quantity),
   204|            "cost_basis_chf": dec_text(pos.cost_basis_chf, "0.01"),
   205|            "market_value_chf": dec_text(pos.market_value_chf, "0.01") if pos.market_value_chf is not None else "",
   206|            "total_return_chf": dec_text(pos.total_return_chf, "0.01") if pos.total_return_chf is not None else "",
   207|            "data_quality_status": pos.data_quality_status,
   208|        })
   209|    return rows
   210|
   211|
   212|def get_ledger_transactions(conn: Connection, limit: int = 200) -> list[dict[str, str]]:
   213|    return rowdicts(conn.execute(
   214|        """
   215|        SELECT t.trade_date, p.name AS platform, a.account_name, t.transaction_type,
   216|               i.name AS instrument, i.ticker, t.quantity, t.gross_amount_original,
   217|               t.net_amount_original, t.currency_original, t.fx_status, t.quality_status,
   218|               t.source_type, t.source_id
   219|        FROM transactions t
   220|        LEFT JOIN accounts a ON a.account_id=t.account_id
   221|        LEFT JOIN platforms p ON p.platform_id=a.platform_id
   222|        LEFT JOIN instruments i ON i.instrument_id=t.instrument_id
   223|        ORDER BY t.trade_date DESC, t.created_at DESC LIMIT ?
   224|        """, (limit,)
   225|    ).fetchall())
   226|
   227|
   228|def get_watchlist(conn: Connection) -> list[dict[str, str]]:
   229|    return rowdicts(conn.execute("SELECT name, asset_class, reason, status, target_entry_price, target_entry_currency, investment_case, bear_case, next_review_date FROM watchlist ORDER BY name").fetchall())
   230|
   231|
   232|def get_alerts(conn: Connection) -> list[dict[str, str]]:
   233|    return rowdicts(conn.execute(
   234|        """
   235|        SELECT priority, status, category, entity_type, entity_id, rule_id, message,
   236|               occurrence_count, last_seen_at, resolved_at, muted_until, fingerprint, dedup_key, created_at
   237|        FROM alerts
   238|        ORDER BY CASE priority WHEN 'kritisch' THEN 0 WHEN 'warnung' THEN 1 ELSE 2 END, status, created_at DESC
   239|        """
   240|    ).fetchall())
   241|
   242|
   243|def get_audit_events(conn: Connection, limit: int = 200) -> list[dict[str, str]]:
   244|    rows = conn.execute(
   245|        """
   246|        SELECT timestamp, source, action, entity_type, entity_id, old_values_json, new_values_json,
   247|               confirmed, created_by, quality_status
   248|        FROM audit_log ORDER BY timestamp DESC LIMIT ?
   249|        """, (limit,)
   250|    ).fetchall()
   251|    out: list[dict[str, str]] = []
   252|    for row in rows:
   253|        item = {k: ("" if row[k] is None else str(row[k])) for k in row.keys()}
   254|        for key in ("old_values_json", "new_values_json"):
   255|            if item[key]:
   256|                try:
   257|                    item[key] = json.dumps(json.loads(item[key]), ensure_ascii=False, indent=2, sort_keys=True)
   258|                except json.JSONDecodeError:
   259|                    pass
   260|        out.append(item)
   261|

Use offset=261 to continue reading (showing 1-260 of 367 lines)


Explanation:
**Evaluating transaction types**

I need to think about whether `calculate_positions` should support different transaction types. It's possible that incorporating this could enhance its functionality. I’m wondering if every transaction type would require a unique handling process or if they could be managed under a single approach. I should consider how different types might interact and whether the added complexity would be worth it in terms of overall utility. Let’s dig deeper into this!


     1|from __future__ import annotations
     2|
     3|from dataclasses import dataclass, field
     4|from decimal import Decimal, ROUND_HALF_UP
     5|from sqlite3 import Connection
     6|
     7|from jarvis_finance.audit.log import record_audit_event
     8|from jarvis_finance.imports.common import stable_id, utc_now
     9|from jarvis_finance.quality.alerts import create_alert
    10|
    11|from .cost_basis import WeightedAverageLot
    12|
    13|MONEY = Decimal("0.01")
    14|
    15|
    16|def _d(value: object, default: str = "0") -> Decimal:
    17|    if value is None or str(value).strip() == "":
    18|        return Decimal(default)
    19|    return Decimal(str(value))
    20|
    21|
    22|def _money(value: Decimal | None) -> Decimal | None:
    23|    if value is None:
    24|        return None
    25|    return value.quantize(MONEY, rounding=ROUND_HALF_UP)
    26|
    27|
    28|@dataclass
    29|class Position:
    30|    account_id: str
    31|    instrument_id: str
    32|    quantity: Decimal = Decimal("0")
    33|    cost_basis_original: Decimal = Decimal("0")
    34|    cost_basis_chf: Decimal = Decimal("0")
    35|    average_cost_original: Decimal | None = None
    36|    average_cost_chf: Decimal | None = None
    37|    realized_pnl_chf: Decimal = Decimal("0")
    38|    income_chf: Decimal = Decimal("0")
    39|    fees_chf: Decimal = Decimal("0")
    40|    taxes_chf: Decimal = Decimal("0")
    41|    market_price_original: Decimal | None = None
    42|    market_value_original: Decimal | None = None
    43|    market_value_chf: Decimal | None = None
    44|    unrealized_pnl_chf: Decimal | None = None
    45|    total_return_chf: Decimal | None = None
    46|    data_quality_status: str = "ok"
    47|    quality_warnings: list[str] = field(default_factory=list)
    48|
    49|
    50|@dataclass
    51|class PositionCalculationResult:
    52|    positions: dict[tuple[str, str], Position] = field(default_factory=dict)
    53|    as_of_date: str | None = None
    54|
    55|
    56|def _latest_market_price(conn: Connection, instrument_id: str, as_of_date: str | None):
    57|    if as_of_date:
    58|        return conn.execute(
    59|            """
    60|            SELECT close, currency FROM market_prices
    61|            WHERE instrument_id=? AND price_date <= ? AND quality_status='ok'
    62|            ORDER BY price_date DESC, created_at DESC LIMIT 1
    63|            """,
    64|            (instrument_id, as_of_date),
    65|        ).fetchone()
    66|    return conn.execute(
    67|        """
    68|        SELECT close, currency FROM market_prices
    69|        WHERE instrument_id=? AND quality_status='ok'
    70|        ORDER BY price_date DESC, created_at DESC LIMIT 1
    71|        """,
    72|        (instrument_id,),
    73|    ).fetchone()
    74|
    75|
    76|def _add_warning(conn: Connection, pos: Position, rule_id: str, message: str) -> None:
    77|    if rule_id not in pos.quality_warnings:
    78|        pos.quality_warnings.append(rule_id)
    79|    pos.data_quality_status = "incomplete"
    80|    create_alert(
    81|        conn,
    82|        priority="kritisch" if rule_id == "missing_fx" else "warnung",
    83|        category="ledger",
    84|        entity_type="instrument",
    85|        entity_id=pos.instrument_id,
    86|        rule_id=rule_id,
    87|        message=message,
    88|        evidence={"account_id": pos.account_id, "instrument_id": pos.instrument_id},
    89|    )
    90|
    91|
    92|def calculate_positions(conn: Connection, *, as_of_date: str | None = None) -> PositionCalculationResult:
    93|    result = PositionCalculationResult(as_of_date=as_of_date)
    94|    lots: dict[tuple[str, str], WeightedAverageLot] = {}
    95|    params: tuple[object, ...] = ()
    96|    where = "WHERE instrument_id IS NOT NULL"
    97|    if as_of_date:
    98|        where += " AND trade_date <= ?"
    99|        params = (as_of_date,)
   100|    rows = conn.execute(
   101|        f"""
   102|        SELECT * FROM transactions
   103|        {where}
   104|        ORDER BY account_id, instrument_id, trade_date, created_at, transaction_id
   105|        """,
   106|        params,
   107|    ).fetchall()
   108|    for row in rows:
   109|        key = (row["account_id"], row["instrument_id"])
   110|        pos = result.positions.setdefault(key, Position(row["account_id"], row["instrument_id"]))
   111|        lot = lots.setdefault(key, WeightedAverageLot())
   112|        t = row["transaction_type"]
   113|        qty = _d(row["quantity"])
   114|        gross = _d(row["gross_amount_original"])
   115|        fee = _d(row["fee_original"])
   116|        tax = _d(row["tax_original"])
   117|        fx = _d(row["fx_rate_to_chf"], "1") if row["fx_rate_to_chf"] is not None else None
   118|        if fx is None or row["fx_status"] == "missing":
   119|            _add_warning(conn, pos, "missing_fx", "Historic FX is missing; precise CHF total return cannot be calculated.")
   120|        if t in {"initial_position_snapshot", "buy"}:
   121|            if qty > 0:
   122|                cost_original = gross + fee
   123|                cost_chf = (gross + fee) * fx if fx is not None else Decimal("0")
   124|                lot.buy(qty, cost_original, cost_chf)
   125|                pos.fees_chf += fee * fx if fx is not None else Decimal("0")
   126|                pos.taxes_chf += tax * fx if fx is not None else Decimal("0")
   127|        elif t in {"partial_sell", "full_sell"}:
   128|            if qty > 0:
   129|                proceeds_chf = (gross - fee - tax) * fx if fx is not None else Decimal("0")
   130|                lot.sell(qty, proceeds_chf)
   131|                pos.fees_chf += fee * fx if fx is not None else Decimal("0")
   132|                pos.taxes_chf += tax * fx if fx is not None else Decimal("0")
   133|        elif t in {"dividend", "etf_distribution"}:
   134|            income_original = _d(row["net_amount_original"]) if row["net_amount_original"] is not None else gross - fee - tax
   135|            if fx is not None:
   136|                pos.income_chf += income_original * fx
   137|                pos.fees_chf += fee * fx
   138|                pos.taxes_chf += tax * fx
   139|    for key, lot in lots.items():
   140|        pos = result.positions[key]
   141|        pos.quantity = lot.quantity.normalize()
   142|        pos.cost_basis_original = _money(lot.cost_basis_original) or Decimal("0")
   143|        pos.cost_basis_chf = _money(lot.cost_basis_chf) or Decimal("0")
   144|        pos.average_cost_original = lot.average_cost_original
   145|        pos.average_cost_chf = lot.average_cost_chf
   146|        pos.realized_pnl_chf = _money(lot.realized_pnl_chf) or Decimal("0")
   147|        pos.income_chf = _money(pos.income_chf) or Decimal("0")
   148|        pos.fees_chf = _money(pos.fees_chf) or Decimal("0")
   149|        pos.taxes_chf = _money(pos.taxes_chf) or Decimal("0")
   150|        market = _latest_market_price(conn, pos.instrument_id, as_of_date)
   151|        if market is None:
   152|            _add_warning(conn, pos, "missing_market_price", "Market price is missing; unrealized P&L cannot be calculated.")
   153|        else:
   154|            pos.market_price_original = _d(market["close"])
   155|            pos.market_value_original = _money(pos.quantity * pos.market_price_original)
   156|            # MVP: market FX is 1 for CHF prices; non-CHF market FX will be added when price/FX provider pipeline lands.
   157|            if market["currency"] == "CHF":
   158|                pos.market_value_chf = pos.market_value_original
   159|                pos.unrealized_pnl_chf = _money(pos.market_value_chf - pos.cost_basis_chf)
   160|            else:
   161|                _add_warning(conn, pos, "missing_fx", "Market FX is missing; CHF valuation is incomplete.")
   162|        if pos.market_value_chf is not None and "missing_fx" not in pos.quality_warnings:
   163|            pos.total_return_chf = _money((pos.unrealized_pnl_chf or Decimal("0")) + pos.realized_pnl_chf + pos.income_chf - pos.fees_chf - pos.taxes_chf)
   164|        else:
   165|            pos.total_return_chf = None
   166|    conn.commit()
   167|    return result
   168|
   169|
   170|def save_position_snapshots(conn: Connection, result: PositionCalculationResult, *, note: str) -> list[str]:
   171|    if not note.strip():
   172|        raise ValueError("position snapshot requires a note")
   173|    now = utc_now()
   174|    snapshot_date = result.as_of_date or now[:10]
   175|    ids: list[str] = []
   176|    for (account_id, instrument_id), pos in result.positions.items():
   177|        row = conn.execute("SELECT platform_id FROM accounts WHERE account_id=?", (account_id,)).fetchone()
   178|        platform_id = row["platform_id"] if row else None
   179|        snapshot_id = stable_id("possnap", snapshot_date, account_id, instrument_id)
   180|        conn.execute(
   181|            """
   182|            INSERT OR REPLACE INTO positions_snapshot(
   183|                position_snapshot_id, snapshot_date, account_id, platform_id, instrument_id,
   184|                quantity, average_cost_original, cost_basis_original, cost_basis_chf,
   185|                market_price_original, market_value_original, market_value_chf,
   186|                unrealized_price_pnl_chf, realized_pnl_chf, income_chf, fees_chf,
   187|                taxes_chf, total_return_chf, data_quality_status, created_at
   188|            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
   189|            """,
   190|            (
   191|                snapshot_id, snapshot_date, account_id, platform_id, instrument_id,
   192|                str(pos.quantity), str(pos.average_cost_original) if pos.average_cost_original is not None else None,
   193|                str(pos.cost_basis_original), str(pos.cost_basis_chf),
   194|                str(pos.market_price_original) if pos.market_price_original is not None else None,
   195|                str(pos.market_value_original) if pos.market_value_original is not None else None,
   196|                str(pos.market_value_chf) if pos.market_value_chf is not None else None,
   197|                str(pos.unrealized_pnl_chf) if pos.unrealized_pnl_chf is not None else None,
   198|                str(pos.realized_pnl_chf), str(pos.income_chf), str(pos.fees_chf),
   199|                str(pos.taxes_chf), str(pos.total_return_chf) if pos.total_return_chf is not None else None,
   200|                pos.data_quality_status, now,
   201|            ),
   202|        )
   203|        record_audit_event(
   204|            conn,
   205|            source="ledger_positions",
   206|            action="calculate_position_snapshot",
   207|            entity_type="position_snapshot",
   208|            entity_id=snapshot_id,
   209|            new_values={"account_id": account_id, "instrument_id": instrument_id, "snapshot_date": snapshot_date},
   210|            user_text_note=note,
   211|            confirmed=True,
   212|            created_by="system",
   213|        )
   214|        ids.append(snapshot_id)
   215|    conn.commit()
   216|    return ids
   217|


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python - <<'PY'
import json
from jarvis_finance.config.settings import load_settings
from jarvis_finance.storage.database import connect
from jarvis_finance.storage.migrations import apply_migrations
from jarvis_finance.dashboard import data as dashboard_data
conn=connect(load_settings().db_path); apply_migrations(conn)
positions=dashboard_data.get_positions(conn)
ledger=dashboard_data.get_ledger_transactions(conn, limit=20)
review_imported=conn.execute("SELECT COUNT(*) AS n FROM broker_import_review_items WHERE source_platform='True Wealth' AND review_status='imported'").fetchone()['n']
review_ready_other=conn.execute("SELECT COUNT(*) AS n FROM broker_import_execution_plans WHERE source_platform='True Wealth' AND payload_status='ready' AND execution_status='planned'").fetchone()['n']
exec_imported=conn.execute("SELECT COUNT(*) AS n FROM broker_import_execution_plans WHERE execution_status='imported'").fetchone()['n']
aud_import=conn.execute("SELECT COUNT(*) AS n FROM audit_log WHERE action IN ('execute_mini_import','mark_review_item_imported')").fetchone()['n']
dq=[r for r in dashboard_data.get_data_quality(conn) if r.get('check','').startswith('Review Readiness') or r.get('check','') in {'Alert: missing_fx','Alert: missing_market_price'}]
print(json.dumps({'dashboard_positions_count':len(positions),'ledger_rows_count':len(ledger),'manual_review_imported_count':review_imported,'other_ready_planned_count':review_ready_other,'execution_imported_count':exec_imported,'import_audit_count':aud_import,'dq_relevant_count':len(dq)}, sort_keys=True))
PY


Output: {"dashboard_positions_count": 1, "dq_relevant_count": 4, "execution_imported_count": 1, "import_audit_count": 2, "ledger_rows_count": 1, "manual_review_imported_count": 1, "other_ready_planned_count": 2}


Explanation:
**Checking data quality and testing**

I’m considering whether I need to update the Data Quality for execution/import, but it seems it's not necessary. I should run some compile/tests, even though there might be a failure due to the pycache produced—I'll remove that later. There’s a possibility of test failures because of the schema version, but the migrations table is maxed out at 7. I'll run everything, check the git safety/status, and commit those code changes. Oh, I should remember to include the test for the execution plan table before committing!


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python -m compileall -q src tests
PYTHONPATH=src pytest -q
rm -rf .pytest_cache src/jarvis_finance/**/__pycache__ tests/**/__pycache__
PYTHONPATH=src python -m jarvis_finance.cli.main git-safety-scan .
git status --short --branch


Output:
........................................................................ [ 46%]
........................................................................ [ 92%]
...........                                                              [100%]
155 passed in 5.34s
GIT_SAFETY_OK
## main...origin/main
 M src/jarvis_finance/storage/migrations.py
 M tests/unit/test_schema.py


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
git add src/jarvis_finance/storage/migrations.py tests/unit/test_schema.py
git commit -m "feat: add broker import execution plans"
PYTHONPATH=src python -m compileall -q src tests
PYTHONPATH=src pytest -q
rm -rf .pytest_cache src/jarvis_finance/**/__pycache__ tests/**/__pycache__
PYTHONPATH=src python -m jarvis_finance.cli.main git-safety-scan .
git status --short --branch


Output:
[main 48562fe] feat: add broker import execution plans
 2 files changed, 34 insertions(+), 3 deletions(-)
........................................................................ [ 46%]
........................................................................ [ 92%]
...........                                                              [100%]
155 passed in 5.52s
GIT_SAFETY_OK
## main...origin/main [voraus 1]


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
python - <<'PY'
import re, subprocess, sys
from pathlib import Path
repo=Path('/home/agent/.hermes/repos/FinanceManager')
text=Path.home().joinpath('jarvis_runtime/finance-system/secrets/Github_token_FinanceManager.csv').read_text()
m=re.search(r'(github_pat_[A-Za-z0-9_]+|ghp_[A-Za-z0-9_]+|gho_[A-Za-z0-9_]+)', text)
if not m:
    print('TOKEN_PARSE=failed'); sys.exit(12)
token=m.group(1); auth_url=f'https://{token}@github.com/Gamexgit/FinanceManager.git'
def run(args, check=True): return subprocess.run(args, cwd=repo, text=True, capture_output=True, check=check)
try:
    run(['git','fetch',auth_url,'main:refs/remotes/origin/main','--quiet'])
    local=run(['git','rev-parse','HEAD']).stdout.strip(); origin=run(['git','rev-parse','origin/main']).stdout.strip(); base=run(['git','merge-base','HEAD','origin/main']).stdout.strip()
    print('REMOTE_BEFORE='+origin[:7]); print('LOCAL_HEAD='+local[:7])
    if origin != base:
        print('REMOTE_HAS_NEW_COMMITS=yes'); sys.exit(3)
    push=run(['git','push',auth_url,'main:main'], check=False)
    if push.returncode:
        print('PUSH=failed'); print((push.stderr or push.stdout).replace(token,'[REDACTED]')[-1000:]); sys.exit(push.returncode)
    run(['git','fetch',auth_url,'main:refs/remotes/origin/main','--quiet'])
    print('PUSH=success'); print('REMOTE_AFTER='+run(['git','rev-parse','--short','origin/main']).stdout.strip())
finally:
    token=None; auth_url=None
PY
PYTHONPATH=src python -m jarvis_finance.cli.main git-safety-scan .
git status --short --branch


Output:
REMOTE_BEFORE=1c5c3a0
LOCAL_HEAD=48562fe
PUSH=success
REMOTE_AFTER=48562fe
GIT_SAFETY_OK
## main...origin/main


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python - <<'PY'
import json
from jarvis_finance.config.settings import load_settings
from jarvis_finance.storage.database import connect
from jarvis_finance.storage.migrations import apply_migrations
conn=connect(load_settings().db_path); apply_migrations(conn)
summary={
 'execution_plans': [dict(r) for r in conn.execute("SELECT payload_status, execution_status, COUNT(*) AS n FROM broker_import_execution_plans GROUP BY payload_status, execution_status ORDER BY payload_status, execution_status").fetchall()],
 'review_status_true_wealth': [dict(r) for r in conn.execute("SELECT review_status, import_readiness_status, COUNT(*) AS n FROM broker_import_review_items WHERE source_platform='True Wealth' GROUP BY review_status, import_readiness_status ORDER BY review_status, import_readiness_status").fetchall()],
 'productive_counts': {t: conn.execute(f'SELECT COUNT(*) AS n FROM {t}').fetchone()['n'] for t in ['transactions','positions_snapshot','cash_balances']},
 'mini_import_transactions': conn.execute("SELECT COUNT(*) AS n FROM transactions WHERE source_type='broker_import_reviewed_snapshot' AND transaction_type='initial_position_snapshot'").fetchone()['n'],
 'audit_import_events': conn.execute("SELECT COUNT(*) AS n FROM audit_log WHERE action IN ('execute_mini_import','mark_review_item_imported')").fetchone()['n'],
 'dashboard_position_count': 1 if conn.execute("SELECT COUNT(*) AS n FROM transactions WHERE transaction_type='initial_position_snapshot'").fetchone()['n'] >= 1 else 0,
}
print(json.dumps(summary, sort_keys=True))
PY
git status --short --branch


Output:
{"audit_import_events": 2, "dashboard_position_count": 1, "execution_plans": [{"execution_status": "imported", "n": 1, "payload_status": "ready"}, {"execution_status": "planned", "n": 2, "payload_status": "ready"}], "mini_import_transactions": 1, "productive_counts": {"cash_balances": 0, "positions_snapshot": 0, "transactions": 1}, "review_status_true_wealth": [{"import_readiness_status": "ready_for_import", "n": 1, "review_status": "imported"}, {"import_readiness_status": "not_ready", "n": 14, "review_status": "open"}, {"import_readiness_status": "review_needed", "n": 1, "review_status": "open"}, {"import_readiness_status": "ready_for_import", "n": 2, "review_status": "resolved"}]}
## main...origin/main


Phase B6 ist abgeschlossen, Sir — in zwei kontrollierten Teilphasen, wie befohlen.

## Gesamtstatus

- B6a finaler Trockenlauf: **abgeschlossen**
- B6b produktiver Mini-Import: **abgeschlossen**
- Produktiv importierte Positionen: **genau 1**
- Neue produktive Transaktionen: **1**
- Direkte `positions_snapshot` Writes: **0**
- Cash-Snapshots: **0**
- Ledger-Massenimport: **nein**
- Echte Dateien ins Git: **nein**
- Echte Werte/Mengen/Kontostände im Chat: **nein**
- Tests: **155 passed**
- Git-Safety: **GIT_SAFETY_OK**
- Commit: `48562fe`
- Push Status: **success**
- Git Status: **clean**, `main...origin/main`

---

## B6a — Import Execution Plan / finaler Trockenlauf

### Preflight

Geprüft:

- Git Status clean vor Start: **ja**
- Git-Safety OK: **ja**
- Runtime-DB außerhalb Repo: **ja**
- letztes Backup vorhanden: **ja**
- echte Dateien im Repo: **0**
- True-Wealth-Datei nur aus Runtime-Kontext gelesen: **ja**
- Datei nicht ins Repo kopiert: **ja**
- 3 True-Wealth Items mit `ready_for_import`: **ja**
- Account Mapping vorhanden: **ja**
- Instrument Mapping vorhanden: **ja**
- Snapshot Date bestätigt: **ja**

### Execution-Plan-Struktur

Neu vorbereitet:

- Tabelle: `broker_import_execution_plans`
- Migration: `007_broker_import_execution_plans`
- Schema-Version: `7`

Felder enthalten u.a.:

- execution plan ID
- dry run ID
- review item ID
- source platform
- target account
- target instrument
- transaction type
- snapshot date
- payload status
- payload quality flags
- source row hash
- runtime-only planned write summary
- execution status
- linked transaction ID nach Import

### Payload-Ergebnis

Für die 3 True-Wealth `ready_for_import` Items:

- ready Review Items geprüft: **3**
- Import Payloads ready: **3**
- Import Payloads blocked: **0**
- Items mit fehlendem FX: **3**
- Items mit `cost_basis_uncertain`: **3**
- Items mit fehlender Quantity: **0**
- Items mit fehlendem Account Mapping: **0**
- Items mit fehlendem Instrument Mapping: **0**

Bewertung:

- Technischer Mini-Import war vertretbar.
- Qualität ist aber bewusst als unvollständig/warning markiert:
  - historische FX nicht explizit vorhanden
  - Cost Basis nicht sauber vorhanden
  - keine Scheingenauigkeit erzeugt
  - kein verstecktes Ableiten von Cost Basis aus Marktwert

Sehr unromantisch. Genau richtig.

---

## B6b — Produktiver Mini-Import von genau 1 Position

### Backup

Vor dem produktiven Mini-Import ausgeführt:

- Runtime-DB Backup erzeugt: **ja**
- Backup verifiziert: **ja**
- Checksum OK: **ja**
- Speicherort: **Runtime-Backup-Ordner**

### Import

Durchgeführt:

- Mini-Import durchgeführt: **ja**
- Anzahl produktiv importierte Positionen: **1**
- Anzahl neue Transaktionen: **1**
- Transaktionstyp: `initial_position_snapshot`
- Quelle: reviewed True-Wealth execution plan
- `source_type`: `broker_import_reviewed_snapshot`
- Re-Import-Schutz über source/execution plan/row hash: **ja**
- Audit-Log Pflicht erfüllt: **ja**
- Execution-Log Pflicht erfüllt: **ja**
- Keine weiteren Payloads importiert: **ja**

Nicht durchgeführt:

- kein direkter `positions_snapshot` Write
- kein Cash-Snapshot
- keine Ledger-Massenänderung
- keine anderen True-Wealth-Items importiert
- keine PostFinance-/Raiffeisen-Imports

### Runtime Status nach Import

Execution Plans:

- imported: **1**
- planned / ready: **2**

True Wealth Review Items:

- imported + ready_for_import: **1**
- resolved + ready_for_import: **2**
- open + not_ready: **14**
- open + review_needed: **1**

Produktive Tabellen:

- `transactions`: **1**
- `positions_snapshot`: **0**
- `cash_balances`: **0**

Audit:

- Import-spezifische Audit Events: **2**
- Zusätzlich bestehen die Review-/Mapping-Audits aus B5 und Plan-Audits aus B6a.

---

## Nachkontrolle

### Ledger / Position

Geprüft:

- genau 1 neue produktive Transaktion: **ja**
- `transaction_type = initial_position_snapshot`: **ja**
- account mapping korrekt vorhanden: **ja**
- instrument mapping korrekt vorhanden: **ja**
- Audit-Log vorhanden: **ja**
- Execution Plan auf imported gesetzt: **ja**
- Position aus Ledger berechenbar: **ja**
- Dashboard-Positionserkennung aus Ledger: **ja**
- direkte Snapshot-Wahrheit nicht geschrieben: **ja**

### Andere ready Items

- andere 2 ready Items unverändert importierbar: **ja**
- nicht importiert: **ja**
- kein Massenimport: **ja**

### Data Quality

Warnings erwartungsgemäß:

- Missing FX / unvollständige FX-Historie
- Cost Basis unsicher
- Market-/Performance-Präzision eingeschränkt

Das ist fachlich korrekt: Die Position existiert nun im Ledger als Initial Snapshot, aber Performance-/Cost-Basis-Auswertungen müssen vorsichtig bleiben.

### Dashboard-Prüfung

Geprüft ohne Werteausgabe:

- Portfolio/Equity-Seite erkennt importierte Ledger-Position: **ja**
- Import Wizard / Execution Status korrekt: **ja**
- Manual Review Queue zeigt importiertes Item korrekt: **ja**
- Data Quality Center zeigt relevante Hinweise: **ja**
- Audit-Seite enthält Importhistorie: **ja**

---

## Codeänderungen

Geändert:

- `src/jarvis_finance/storage/migrations.py`
- `tests/unit/test_schema.py`

Neu:

- Migration `007_broker_import_execution_plans`
- Tabelle `broker_import_execution_plans`
- Schema-Test für Execution-Plan-Tabelle

Commit:

- `48562fe` — `feat: add broker import execution plans`

Push:

- **success**
- Remote HEAD: `48562fe`

Tests / Safety:

- Compile: **OK**
- pytest: **155 passed**
- Git-Safety: **GIT_SAFETY_OK**
- Git Status: **clean**

---

## B6a Bericht kurz

- Payloads geprüft: **3**
- Payloads ready: **3**
- Payloads blocked: **0**
- relevante Quality Flags:
  - missing FX: **3**
  - cost_basis_uncertain: **3**
- technische Importfähigkeit: **ja, für Mini-Import geeignet**

## B6b Bericht kurz

- Mini-Import durchgeführt: **ja**
- produktiv importierte Positionen: **1**
- neue Transaktionen: **1**
- Audit-Logs: **vorhanden**
- importiertes Review Item Status: **imported**
- andere ready Items unverändert: **ja**
- Data Quality Warnings: **ja, erwartungsgemäß**
- Backup erzeugt/verifiziert: **ja / ja**
- Git-Safety: **OK**
- Git Status: **clean**

## Empfehlung

Ein nächster Mini-Import wäre **vertretbar**, aber weiterhin nur einzeln:

1. jeweils genau 1 weiteres ready True-Wealth Item
2. vorher Backup
3. nachher Audit-/Execution-/Ledger-Check
4. keine Massenübernahme, bis FX-/Cost-Basis-Qualität besser modelliert ist

Kurz: Der erste echte Aktien-/ETF-Ledger-Import ist gelandet. Nicht glamourös, aber auditierbar, reversibel nachvollziehbar und ohne dass das System heimlich ein Depot-Märchen schreibt.