terminal

set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python - <<'PY'
import json
from decimal import Decimal, InvalidOperation
from jarvis_finance.config.settings import load_settings
from jarvis_finance.storage.database import connect
from jarvis_finance.storage.migrations import apply_migrations
from jarvis_finance.ledger.positions import calculate_positions
conn=connect(load_settings().db_path); apply_migrations(conn)
rows=conn.execute("""
SELECT t.*, ep.execution_plan_id, ep.review_item_id, ep.execution_status, ri.review_status
FROM transactions t
LEFT JOIN broker_import_execution_plans ep ON ep.execution_plan_id=t.source_id OR ep.transaction_id=t.transaction_id
LEFT JOIN broker_import_review_items ri ON ri.review_item_id=ep.review_item_id
WHERE t.source_type='broker_import_reviewed_snapshot' AND t.transaction_type='initial_position_snapshot'
""").fetchall()
checks=[]
for r in rows:
    try:
        Decimal(str(r['quantity'])); qty_parse=True
    except Exception:
        qty_parse=False
    checks.append({
        'has_tx_id': bool(r['transaction_id']), 'tx_type_ok': r['transaction_type']=='initial_position_snapshot', 'source_type_ok': r['source_type']=='broker_import_reviewed_snapshot',
        'has_execution_plan': bool(r['execution_plan_id']), 'has_review_item': bool(r['review_item_id']), 'has_instrument': bool(r['instrument_id']), 'has_account': bool(r['account_id']),
        'qty_present': bool(r['quantity']), 'qty_decimal_parseable': qty_parse, 'quality_status': r['quality_status'], 'fx_status': r['fx_status'], 'execution_status': r['execution_status'], 'review_status': r['review_status']
    })
positions=calculate_positions(conn)
alerts=conn.execute("SELECT rule_id, COUNT(*) n FROM alerts WHERE status='open' GROUP BY rule_id").fetchall()
aud=conn.execute("SELECT COUNT(*) n FROM audit_log WHERE action IN ('execute_mini_import','mark_review_item_imported')").fetchone()['n']
# reimport blockers checked via duplicate keys in existing query shape
block_plan=conn.execute("SELECT COUNT(*) n FROM transactions t JOIN broker_import_execution_plans ep ON t.source_id=ep.execution_plan_id WHERE t.source_type='broker_import_reviewed_snapshot'").fetchone()['n']
block_hash=conn.execute("SELECT COUNT(*) n FROM transactions t JOIN broker_import_execution_plans ep ON t.row_hash=ep.source_row_hash WHERE t.source_type='broker_import_reviewed_snapshot'").fetchone()['n']
print(json.dumps({'b6b_tx_count':len(rows),'checks':checks,'positions_calculated':len(positions.positions),'open_alert_rule_counts':[dict(a) for a in alerts], 'audit_import_events':aud, 'reimport_plan_existing_matches':block_plan, 'reimport_hash_existing_matches':block_hash}, sort_keys=True))
PY


Output: {"audit_import_events": 2, "b6b_tx_count": 1, "checks": [{"execution_status": "imported", "fx_status": "missing", "has_account": true, "has_execution_plan": true, "has_instrument": true, "has_review_item": true, "has_tx_id": true, "qty_decimal_parseable": true, "qty_present": true, "quality_status": "warning", "review_status": "imported", "source_type_ok": true, "tx_type_ok": true}], "open_alert_rule_counts": [], "positions_calculated": 1, "reimport_hash_existing_matches": 1, "reimport_plan_existing_matches": 1}


Explanation:
**Inspecting schema transactions**

I need to implement something regarding inspecting the schema for transaction columns. It seems like there's some work to do to ensure everything is set up properly. I'll take the time to look into the structure, making sure the columns are functioning as expected. I wonder if there are any specific attributes I should pay extra attention to? It's always good to double-check these details to avoid issues down the line!

CREATE TABLE IF NOT EXISTS transactions|transactions \(


CREATE TABLE.*transactions|transaction_id TEXT PRIMARY KEY


File not found: /home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/schema.sql


     1|INITIAL_SCHEMA_SQL = "\nCREATE TABLE IF NOT EXISTS schema_migrations (\n    version INTEGER PRIMARY KEY,\n    name TEXT NOT NULL,\n    applied_at TEXT NOT NULL,\n    checksum TEXT NOT NULL\n);\n\nCREATE TABLE IF NOT EXISTS platforms (\n    platform_id TEXT PRIMARY KEY,\n    name TEXT NOT NULL UNIQUE,\n    platform_type TEXT NOT NULL,\n    country TEXT,\n    default_currency TEXT NOT NULL DEFAULT 'CHF',\n    is_active INTEGER NOT NULL DEFAULT 1,\n    notes TEXT,\n    created_at TEXT NOT NULL,\n    updated_at TEXT\n);\n\nCREATE TABLE IF NOT EXISTS accounts (\n    account_id TEXT PRIMARY KEY,\n    platform_id TEXT NOT NULL REFERENCES platforms(platform_id),\n    account_name TEXT NOT NULL,\n    account_type TEXT NOT NULL,\n    currency TEXT NOT NULL DEFAULT 'CHF',\n    performance_included INTEGER NOT NULL DEFAULT 1,\n    is_health_reserve INTEGER NOT NULL DEFAULT 0,\n    target_cash_min_chf NUMERIC,\n    target_cash_max_chf NUMERIC,\n    is_active INTEGER NOT NULL DEFAULT 1,\n    notes TEXT,\n    created_at TEXT NOT NULL,\n    updated_at TEXT,\n    UNIQUE(platform_id, account_name)\n);\n\nCREATE TABLE IF NOT EXISTS instruments (\n    instrument_id TEXT PRIMARY KEY,\n    asset_class TEXT NOT NULL,\n    name TEXT NOT NULL,\n    ticker TEXT,\n    isin TEXT,\n    exchange TEXT,\n    currency TEXT NOT NULL,\n    country TEXT,\n    sector TEXT,\n    industry TEXT,\n    provider_symbol TEXT,\n    data_provider_primary TEXT,\n    position_category TEXT,\n    ter TEXT,\n    distribution_policy TEXT,\n    index_name TEXT,\n    fund_domicile TEXT,\n    benchmark TEXT,\n    data_provider_fallback TEXT,\n    is_active INTEGER NOT NULL DEFAULT 1,\n    notes TEXT,\n    created_at TEXT NOT NULL,\n    updated_at TEXT\n);\n\nCREATE TABLE IF NOT EXISTS transactions (\n    transaction_id TEXT PRIMARY KEY,\n    transaction_type TEXT NOT NULL,\n    account_id TEXT NOT NULL REFERENCES accounts(account_id),\n    instrument_id TEXT REFERENCES instruments(instrument_id),\n    trade_date TEXT N... [truncated]
     2|
     3|REQUIRED_TABLES = [
     4|    "schema_migrations", "platforms", "accounts", "instruments", "transactions",
     5|    "positions_snapshot", "cash_balances", "fx_rates", "market_prices",
     6|    "crypto_wallets", "crypto_assets", "crypto_holdings", "crypto_transactions",
     7|    "crypto_prices", "watchlist", "reports", "alerts", "audit_log",
     8|    "import_sessions", "decision_journal", "instrument_mappings",
     9|    "platform_account_mappings", "broker_import_dry_runs",
    10|]
    11|


Explanation:
**Inspecting migration details**

I’m looking into using a Python introspect PRAGMA to help with database migrations. It seems like I need to add columns through migration version 8, but I’m wondering if I should also update REQUIRED_TABLES. Maybe it’s not necessary for the plans? I’m not sure about that. I’ll need to inspect the test_schema to clarify what tests I might need or any additional requirements. It feels like there’s a lot to consider to get this right!


     1|from __future__ import annotations
     2|
     3|from jarvis_finance.storage.database import connect_memory
     4|from jarvis_finance.storage.migrations import apply_migrations, get_schema_version
     5|from jarvis_finance.storage.schema import REQUIRED_TABLES
     6|
     7|
     8|def test_db_schema_can_be_created() -> None:
     9|    conn = connect_memory()
    10|    apply_migrations(conn)
    11|    tables = {row["name"] for row in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")}
    12|    assert set(REQUIRED_TABLES).issubset(tables)
    13|
    14|
    15|def test_schema_version_recorded() -> None:
    16|    conn = connect_memory()
    17|    apply_migrations(conn)
    18|    assert get_schema_version(conn) == 7
    19|
    20|
    21|def test_decimal_sensitive_columns_use_text_affinity() -> None:
    22|    conn = connect_memory()
    23|    apply_migrations(conn)
    24|    tx_cols = {row["name"]: row["type"] for row in conn.execute("PRAGMA table_info(transactions)").fetchall()}
    25|    crypto_cols = {row["name"]: row["type"] for row in conn.execute("PRAGMA table_info(crypto_holdings)").fetchall()}
    26|    instrument_cols = {row["name"] for row in conn.execute("PRAGMA table_info(instruments)").fetchall()}
    27|    assert tx_cols["quantity"].upper() == "TEXT"
    28|    assert tx_cols["fx_rate_to_chf"].upper() == "TEXT"
    29|    assert crypto_cols["quantity"].upper() == "TEXT"
    30|    assert {"position_category", "ter", "distribution_policy", "index_name", "fund_domicile", "benchmark"}.issubset(instrument_cols)
    31|
    32|
    33|def test_broker_bank_mapping_tables_exist_with_required_columns() -> None:
    34|    conn = connect_memory()
    35|    apply_migrations(conn)
    36|    instrument_mapping_cols = {row["name"] for row in conn.execute("PRAGMA table_info(instrument_mappings)").fetchall()}
    37|    account_mapping_cols = {row["name"] for row in conn.execute("PRAGMA table_info(platform_account_mappings)").fetchall()}
    38|    dry_run_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_dry_runs)").fetchall()}
    39|
    40|    assert {"source_name", "source_platform", "source_label", "normalized_name", "isin", "ticker", "exchange", "currency", "asset_class", "instrument_id", "mapping_status", "confidence", "quality_flags_json"}.issubset(instrument_mapping_cols)
    41|    assert {"source_platform", "source_account_label", "normalized_platform", "normalized_account_name", "internal_platform_id", "internal_account_id", "account_type", "currency", "mapping_status"}.issubset(account_mapping_cols)
    42|    assert {"source_platform", "source_file_type", "source_filename_hash", "detected_snapshot_date", "snapshot_date_status", "candidate_positions", "candidate_cash_rows", "mapped_positions", "blocked_positions", "quality_flags_json", "summary_json", "session_status", "is_current"}.issubset(dry_run_cols)
    43|    review_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_review_items)").fetchall()}
    44|    assert {"review_item_id", "dry_run_id", "source_platform", "source_row_ref", "row_hash", "source_label", "normalized_name", "detected_asset_class", "detected_currency", "quality_flags_json", "review_status", "import_readiness_status", "reviewer_confirmed", "snapshot_date_confirmed", "ticker_exchange_confirmed", "account_mapping_status"}.issubset(review_cols)
    45|    execution_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_execution_plans)").fetchall()}
    46|    assert {"execution_plan_id", "dry_run_id", "review_item_id", "source_platform", "target_account_id", "target_instrument_id", "transaction_type", "snapshot_date", "payload_status", "payload_quality_flags_json", "source_row_hash", "planned_write_summary_json", "execution_status", "transaction_id"}.issubset(execution_cols)
    47|


Explanation:
**Exploring ledger corrections**

I think I need to implement a module for ledger corrections. I’ll inspect the transaction tests to ensure everything is functioning correctly. It’s important to make sure that my corrections are precise and effective, as this will help maintain the accuracy of the overall system. Let’s take a closer look at the tests to see if they highlight any potential issues that I should address. I want to get this right!


     1|def test_ledger_equities_placeholder_until_task_13() -> None:
     2|    assert True
     3|


     1|from __future__ import annotations
     2|
     3|from decimal import Decimal
     4|
     5|from jarvis_finance.ledger.cash import apply_manual_cash_correction, calculate_cash_balances
     6|from jarvis_finance.ledger.positions import calculate_positions, save_position_snapshots
     7|from jarvis_finance.storage.database import connect_memory
     8|from jarvis_finance.storage.migrations import apply_migrations
     9|
    10|
    11|def setup_conn():
    12|    conn = connect_memory()
    13|    apply_migrations(conn)
    14|    conn.execute("INSERT INTO platforms(platform_id,name,platform_type,default_currency,created_at) VALUES('p1','Demo Broker','broker','CHF','now')")
    15|    conn.execute("INSERT INTO accounts(account_id,platform_id,account_name,account_type,currency,created_at) VALUES('a1','p1','Main','brokerage','CHF','now')")
    16|    conn.execute("INSERT INTO instruments(instrument_id,asset_class,name,ticker,isin,currency,created_at) VALUES('i1','equity','Demo AG','DAG','CH0001','CHF','now')")
    17|    conn.execute("INSERT INTO instruments(instrument_id,asset_class,name,ticker,isin,currency,created_at) VALUES('u1','equity','USD Demo Inc','UDI','US0001','USD','now')")
    18|    return conn
    19|
    20|
    21|def tx(conn, tid, ttype, *, instrument='i1', date='2026-01-01', qty=None, gross=None, fee=0, tax=0, net=None, currency='CHF', fx=1, note='synthetic'):
    22|    conn.execute(
    23|        """INSERT INTO transactions(
    24|            transaction_id, transaction_type, account_id, instrument_id, trade_date,
    25|            quantity, price_original, gross_amount_original, fee_original, tax_original,
    26|            net_amount_original, currency_original, fx_rate_to_chf, fx_status,
    27|            gross_amount_chf, fee_chf, tax_chf, net_amount_chf,
    28|            source_type, is_confirmed, quality_status, notes, created_at
    29|        ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
    30|        (
    31|            tid, ttype, 'a1', instrument, date, str(qty) if qty is not None else None, None,
    32|            str(gross) if gross is not None else None, str(fee), str(tax), str(net) if net is not None else None,
    33|            currency, str(fx) if fx is not None else None, 'ok' if fx is not None else 'missing',
    34|            str(Decimal(str(gross))*Decimal(str(fx))) if gross is not None and fx is not None else None,
    35|            str(Decimal(str(fee))*Decimal(str(fx))) if fx is not None else None,
    36|            str(Decimal(str(tax))*Decimal(str(fx))) if fx is not None else None,
    37|            str(Decimal(str(net))*Decimal(str(fx))) if net is not None and fx is not None else None,
    38|            'synthetic', 1, 'ok' if fx is not None else 'incomplete', note, 'now'
    39|        ),
    40|    )
    41|
    42|
    43|def test_initial_cash_snapshot_is_start_truth_and_buy_sell_dividend_fee_tax_move_cash() -> None:
    44|    conn = setup_conn()
    45|    tx(conn, 'c0', 'initial_cash_snapshot', instrument=None, gross=1000, net=1000)
    46|    tx(conn, 'b1', 'buy', qty=5, gross=500, fee=5, net=505)
    47|    tx(conn, 's1', 'partial_sell', qty=2, gross=260, fee=2, tax=1, net=257)
    48|    tx(conn, 'd1', 'dividend', qty=None, gross=20, tax=7, net=13)
    49|    tx(conn, 'f1', 'fee', instrument=None, gross=3, net=3)
    50|    tx(conn, 't1', 'tax', instrument=None, gross=4, net=4)
    51|
    52|    result = calculate_cash_balances(conn)
    53|
    54|    key = ('a1', 'CHF')
    55|    assert result.balances[key].amount_original == Decimal('758')
    56|    assert result.balances[key].amount_chf == Decimal('758')
    57|    assert result.negative_cash_warnings == []
    58|
    59|
    60|def test_cash_negative_creates_warning_not_correction() -> None:
    61|    conn = setup_conn()
    62|    tx(conn, 'c0', 'initial_cash_snapshot', instrument=None, gross=100, net=100)
    63|    tx(conn, 'b1', 'buy', qty=2, gross=200, fee=0, net=200)
    64|
    65|    result = calculate_cash_balances(conn)
    66|
    67|    assert result.balances[('a1', 'CHF')].amount_original == Decimal('-100')
    68|    assert result.negative_cash_warnings
    69|    alert = conn.execute("SELECT priority, rule_id FROM alerts WHERE rule_id='negative_cash'").fetchone()
    70|    assert alert['priority'] == 'warnung'
    71|
    72|
    73|def test_manual_cash_correction_requires_note_and_audit() -> None:
    74|    conn = setup_conn()
    75|    try:
    76|        apply_manual_cash_correction(conn, account_id='a1', currency='CHF', amount=Decimal('10'), note='')
    77|    except ValueError as exc:
    78|        assert 'note' in str(exc)
    79|    else:
    80|        raise AssertionError('expected note validation')
    81|
    82|    audit_id = apply_manual_cash_correction(conn, account_id='a1', currency='CHF', amount=Decimal('10'), note='Synthetic reconciliation')
    83|    assert audit_id
    84|    assert conn.execute("SELECT COUNT(*) AS n FROM audit_log WHERE action='manual_cash_correction'").fetchone()['n'] == 1
    85|
    86|
    87|def test_weighted_average_cost_buy_fee_usd_fx_and_partial_sell_realized_pnl() -> None:
    88|    conn = setup_conn()
    89|    tx(conn, 'b1', 'buy', instrument='u1', date='2026-01-01', qty=10, gross=1000, fee=10, net=1010, currency='USD', fx='0.90')
    90|    tx(conn, 'b2', 'buy', instrument='u1', date='2026-01-02', qty=10, gross=1200, fee=12, net=1212, currency='USD', fx='0.80')
    91|    tx(conn, 's1', 'partial_sell', instrument='u1', date='2026-01-03', qty=5, gross=700, fee=7, net=693, currency='USD', fx='0.85')
    92|
    93|    positions = calculate_positions(conn)
    94|    pos = positions.positions[('a1', 'u1')]
    95|
    96|    assert pos.quantity == Decimal('15')
    97|    assert pos.cost_basis_original == Decimal('1666.5')
    98|    assert pos.cost_basis_chf == Decimal('1408.95')
    99|    assert pos.average_cost_original == Decimal('111.1')
   100|    assert pos.realized_pnl_chf == Decimal('119.40')
   101|    assert pos.fees_chf == Decimal('24.55')
   102|
   103|
   104|def test_full_sell_sets_quantity_zero_and_keeps_historical_position_visible() -> None:
   105|    conn = setup_conn()
   106|    tx(conn, 'b1', 'buy', qty=3, gross=300, fee=0, net=300)
   107|    tx(conn, 's1', 'full_sell', qty=3, gross=330, fee=0, net=330)
   108|
   109|    pos = calculate_positions(conn).positions[('a1', 'i1')]
   110|
   111|    assert pos.quantity == Decimal('0')
   112|    assert pos.cost_basis_chf == Decimal('0')
   113|    assert pos.realized_pnl_chf == Decimal('30')
   114|
   115|
   116|def test_dividend_changes_income_not_quantity_or_cost_basis() -> None:
   117|    conn = setup_conn()
   118|    tx(conn, 'b1', 'buy', qty=4, gross=400, fee=4, net=404)
   119|    tx(conn, 'd1', 'dividend', qty=None, gross=40, tax=14, net=26)
   120|
   121|    pos = calculate_positions(conn).positions[('a1', 'i1')]
   122|
   123|    assert pos.quantity == Decimal('4')
   124|    assert pos.cost_basis_chf == Decimal('404')
   125|    assert pos.income_chf == Decimal('26')
   126|    assert pos.taxes_chf == Decimal('14')
   127|
   128|
   129|def test_missing_market_price_warns_and_missing_fx_blocks_precise_total_return() -> None:
   130|    conn = setup_conn()
   131|    tx(conn, 'b1', 'buy', qty=2, gross=200, fee=0, net=200, fx=None)
   132|
   133|    result = calculate_positions(conn)
   134|    pos = result.positions[('a1', 'i1')]
   135|
   136|    assert pos.data_quality_status == 'incomplete'
   137|    assert 'missing_market_price' in pos.quality_warnings
   138|    assert 'missing_fx' in pos.quality_warnings
   139|    assert pos.total_return_chf is None
   140|    assert conn.execute("SELECT COUNT(*) AS n FROM alerts WHERE rule_id IN ('missing_market_price','missing_fx')").fetchone()['n'] >= 2
   141|
   142|
   143|def test_market_price_enables_unrealized_pnl_total_return_and_snapshot_audit() -> None:
   144|    conn = setup_conn()
   145|    tx(conn, 'b1', 'buy', qty=2, gross=200, fee=0, net=200)
   146|    conn.execute("INSERT INTO market_prices(market_price_id,instrument_id,price_date,close,currency,provider,quality_status,created_at) VALUES('mp1','i1','2026-01-02',120,'CHF','synthetic','ok','now')")
   147|
   148|    result = calculate_positions(conn, as_of_date='2026-01-02')
   149|    pos = result.positions[('a1', 'i1')]
   150|
   151|    assert pos.market_value_chf == Decimal('240')
   152|    assert pos.unrealized_pnl_chf == Decimal('40')
   153|    assert pos.total_return_chf == Decimal('40')
   154|
   155|    save_position_snapshots(conn, result, note='Synthetic calculated snapshot')
   156|    assert conn.execute("SELECT COUNT(*) AS n FROM positions_snapshot").fetchone()['n'] == 1
   157|    assert conn.execute("SELECT COUNT(*) AS n FROM audit_log WHERE action='calculate_position_snapshot'").fetchone()['n'] == 1
   158|


Explanation:
**Considering execution service**

I need to think about potentially importing an execution service to handle duplicates. Right now, the process is managed by a manual script, which feels a little clunky. It might be worth it to implement a module for broker execution. Also, I need to remember that testing is essential to ensure everything works smoothly. And I should keep in mind the backup functions, as they’ll be crucial for maintaining robustness.


     1|from __future__ import annotations
     2|
     3|import hashlib
     4|import shutil
     5|from dataclasses import dataclass
     6|from datetime import datetime, timezone
     7|from pathlib import Path
     8|
     9|from jarvis_finance.config.paths import is_relative_to
    10|
    11|
    12|@dataclass(frozen=True)
    13|class BackupResult:
    14|    backup_file: Path
    15|    checksum_file: Path
    16|    sha256: str
    17|    size_bytes: int
    18|
    19|
    20|@dataclass(frozen=True)
    21|class VerifyBackupResult:
    22|    backup_file: Path
    23|    checksum_file: Path
    24|    sha256: str
    25|    checksum_ok: bool
    26|    exists: bool
    27|
    28|
    29|@dataclass(frozen=True)
    30|class RestoreResult:
    31|    restored_to: Path
    32|    source_file: Path
    33|    verified: bool
    34|
    35|
    36|def sha256_file(path: Path) -> str:
    37|    digest = hashlib.sha256()
    38|    with path.open("rb") as handle:
    39|        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
    40|            digest.update(chunk)
    41|    return digest.hexdigest()
    42|
    43|
    44|def write_checksum_file(backup_file: Path, sha256: str) -> Path:
    45|    checksum_file = backup_file.with_suffix(backup_file.suffix + ".sha256")
    46|    checksum_file.write_text(f"{sha256}  {backup_file.name}\n", encoding="utf-8")
    47|    return checksum_file
    48|
    49|
    50|def read_expected_checksum(checksum_file: Path) -> str | None:
    51|    if not checksum_file.exists():
    52|        return None
    53|    text = checksum_file.read_text(encoding="utf-8", errors="ignore").strip()
    54|    if not text:
    55|        return None
    56|    return text.split()[0]
    57|
    58|
    59|def ensure_outside_repo(path: Path, repo_root: Path) -> None:
    60|    if is_relative_to(path.resolve(), repo_root.resolve()):
    61|        raise ValueError(f"Backup/restore path must be outside the Git repository: {path}")
    62|
    63|
    64|def backup_runtime_db(*, db_path: Path, backups_dir: Path, repo_root: Path, timestamp: str | None = None) -> BackupResult:
    65|    db_path = db_path.expanduser().resolve()
    66|    backups_dir = backups_dir.expanduser().resolve()
    67|    ensure_outside_repo(backups_dir, repo_root)
    68|    ensure_outside_repo(db_path, repo_root)
    69|    if not db_path.exists():
    70|        raise FileNotFoundError(f"Runtime DB not found: {db_path}")
    71|    backups_dir.mkdir(parents=True, exist_ok=True)
    72|    stamp = timestamp or datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
    73|    backup_file = backups_dir / f"finance-runtime-db-{stamp}.sqlite3"
    74|    shutil.copy2(db_path, backup_file)
    75|    checksum = sha256_file(backup_file)
    76|    checksum_file = write_checksum_file(backup_file, checksum)
    77|    return BackupResult(
    78|        backup_file=backup_file,
    79|        checksum_file=checksum_file,
    80|        sha256=checksum,
    81|        size_bytes=backup_file.stat().st_size,
    82|    )
    83|
    84|
    85|def verify_backup(*, backup_file: Path, repo_root: Path | None = None) -> VerifyBackupResult:
    86|    backup_file = backup_file.expanduser().resolve()
    87|    if repo_root is not None:
    88|        ensure_outside_repo(backup_file, repo_root)
    89|    exists = backup_file.exists() and backup_file.is_file()
    90|    if not exists:
    91|        return VerifyBackupResult(
    92|            backup_file=backup_file,
    93|            checksum_file=backup_file.with_suffix(backup_file.suffix + ".sha256"),
    94|            sha256="",
    95|            checksum_ok=False,
    96|            exists=False,
    97|        )
    98|    actual = sha256_file(backup_file)
    99|    checksum_file = backup_file.with_suffix(backup_file.suffix + ".sha256")
   100|    expected = read_expected_checksum(checksum_file)
   101|    return VerifyBackupResult(
   102|        backup_file=backup_file,
   103|        checksum_file=checksum_file,
   104|        sha256=actual,
   105|        checksum_ok=bool(expected and expected == actual),
   106|        exists=True,
   107|    )
   108|
   109|
   110|def restore_runtime_db(*, backup_file: Path, db_path: Path, repo_root: Path, yes: bool = False) -> RestoreResult:
   111|    if not yes:
   112|        raise PermissionError("Restore requires explicit confirmation via --yes")
   113|    backup_file = backup_file.expanduser().resolve()
   114|    db_path = db_path.expanduser().resolve()
   115|    ensure_outside_repo(backup_file, repo_root)
   116|    ensure_outside_repo(db_path, repo_root)
   117|    verification = verify_backup(backup_file=backup_file, repo_root=repo_root)
   118|    if not verification.exists:
   119|        raise FileNotFoundError(f"Backup file not found: {backup_file}")
   120|    if not verification.checksum_ok:
   121|        raise ValueError(f"Backup checksum verification failed: {backup_file}")
   122|    db_path.parent.mkdir(parents=True, exist_ok=True)
   123|    shutil.copy2(backup_file, db_path)
   124|    return RestoreResult(restored_to=db_path, source_file=backup_file, verified=True)
   125|


Explanation:
**Considering transaction migration**

I'm implementing a migration to add columns to transactions, including fields like voided_at, void_reason, voided_by, correction_of_transaction_id, and correction_reason. There's also an is_voided integer that defaults to 0. I need to ensure that positions ignore any transactions marked as voided. The user mentioned that these transactions should be ignored in calculations. I’m also wondering if cash balances should take voided transactions into account. It seems I need to inspect the cash calculations further.


     1|from __future__ import annotations
     2|
     3|from dataclasses import dataclass, field
     4|from decimal import Decimal
     5|from sqlite3 import Connection
     6|
     7|from jarvis_finance.audit.log import record_audit_event
     8|from jarvis_finance.imports.common import stable_id, utc_now
     9|from jarvis_finance.quality.alerts import create_alert
    10|
    11|
    12|@dataclass
    13|class CashBalance:
    14|    account_id: str
    15|    currency: str
    16|    amount_original: Decimal = Decimal("0")
    17|    amount_chf: Decimal | None = Decimal("0")
    18|    quality_status: str = "ok"
    19|
    20|
    21|@dataclass
    22|class CashCalculationResult:
    23|    balances: dict[tuple[str, str], CashBalance] = field(default_factory=dict)
    24|    negative_cash_warnings: list[str] = field(default_factory=list)
    25|
    26|
    27|def _d(value: object, default: str = "0") -> Decimal:
    28|    if value is None or str(value).strip() == "":
    29|        return Decimal(default)
    30|    return Decimal(str(value))
    31|
    32|
    33|def _signed_cash_delta(row) -> tuple[Decimal, Decimal | None]:
    34|    t = row["transaction_type"]
    35|    gross = _d(row["gross_amount_original"])
    36|    net = _d(row["net_amount_original"]) if row["net_amount_original"] is not None else None
    37|    fee = _d(row["fee_original"])
    38|    tax = _d(row["tax_original"])
    39|    fx = _d(row["fx_rate_to_chf"], "1") if row["fx_rate_to_chf"] is not None else None
    40|
    41|    if t == "initial_cash_snapshot":
    42|        original = net if net is not None else gross
    43|    elif t == "cash_deposit":
    44|        original = net if net is not None else gross
    45|    elif t == "cash_withdrawal":
    46|        original = -(net if net is not None else gross)
    47|    elif t == "buy":
    48|        original = -(net if net is not None else gross + fee + tax)
    49|    elif t in {"partial_sell", "full_sell"}:
    50|        original = net if net is not None else gross - fee - tax
    51|    elif t in {"dividend", "etf_distribution"}:
    52|        original = net if net is not None else gross - tax - fee
    53|    elif t in {"fee", "tax"}:
    54|        original = -(net if net is not None else gross or fee or tax)
    55|    elif t == "fx_conversion":
    56|        original = net if net is not None else gross
    57|    elif t == "manual_cash_correction":
    58|        original = net if net is not None else gross
    59|    else:
    60|        original = Decimal("0")
    61|    chf = original * fx if fx is not None else None
    62|    return original, chf
    63|
    64|
    65|def calculate_cash_balances(conn: Connection, *, as_of_date: str | None = None) -> CashCalculationResult:
    66|    result = CashCalculationResult()
    67|    params: tuple[object, ...] = ()
    68|    where = ""
    69|    if as_of_date:
    70|        where = "WHERE trade_date <= ?"
    71|        params = (as_of_date,)
    72|    rows = conn.execute(
    73|        f"""
    74|        SELECT * FROM transactions
    75|        {where}
    76|        ORDER BY account_id, currency_original, trade_date, created_at, transaction_id
    77|        """,
    78|        params,
    79|    ).fetchall()
    80|    for row in rows:
    81|        account_id = row["account_id"]
    82|        currency = row["currency_original"]
    83|        key = (account_id, currency)
    84|        balance = result.balances.setdefault(key, CashBalance(account_id, currency))
    85|        original, chf = _signed_cash_delta(row)
    86|        balance.amount_original += original
    87|        if chf is None:
    88|            balance.amount_chf = None
    89|            balance.quality_status = "incomplete"
    90|        elif balance.amount_chf is not None:
    91|            balance.amount_chf += chf
    92|    for key, balance in result.balances.items():
    93|        if balance.amount_original < 0:
    94|            msg = f"Negative cash balance for account={balance.account_id} currency={balance.currency}: {balance.amount_original}"
    95|            result.negative_cash_warnings.append(msg)
    96|            create_alert(
    97|                conn,
    98|                priority="warnung",
    99|                category="cash",
   100|                entity_type="account",
   101|                entity_id=balance.account_id,
   102|                rule_id="negative_cash",
   103|                message=msg,
   104|                evidence={"currency": balance.currency, "amount_original": str(balance.amount_original)},
   105|            )
   106|    conn.commit()
   107|    return result
   108|
   109|
   110|def apply_manual_cash_correction(
   111|    conn: Connection,
   112|    *,
   113|    account_id: str,
   114|    currency: str,
   115|    amount: Decimal,
   116|    note: str,
   117|    trade_date: str | None = None,
   118|) -> str:
   119|    if not note.strip():
   120|        raise ValueError("manual cash correction requires a note")
   121|    now = utc_now()
   122|    trade_date = trade_date or now[:10]
   123|    transaction_id = stable_id("cashcorr", account_id, currency, trade_date, amount, note)
   124|    fx_rate = Decimal("1") if currency.upper() == "CHF" else None
   125|    conn.execute(
   126|        """
   127|        INSERT INTO transactions(
   128|            transaction_id, transaction_type, account_id, trade_date, gross_amount_original,
   129|            net_amount_original, currency_original, fx_rate_to_chf, fx_status,
   130|            gross_amount_chf, net_amount_chf, source_type, is_confirmed,
   131|            quality_status, notes, created_at
   132|        ) VALUES (?, 'manual_cash_correction', ?, ?, ?, ?, ?, ?, ?, ?, ?, 'manual', 1, ?, ?, ?)
   133|        """,
   134|        (
   135|            transaction_id, account_id, trade_date, str(amount), str(amount), currency.upper(),
   136|            str(fx_rate) if fx_rate is not None else None,
   137|            "ok" if fx_rate is not None else "missing",
   138|            str(amount * fx_rate) if fx_rate is not None else None,
   139|            str(amount * fx_rate) if fx_rate is not None else None,
   140|            "ok" if fx_rate is not None else "incomplete",
   141|            note,
   142|            now,
   143|        ),
   144|    )
   145|    audit_id = record_audit_event(
   146|        conn,
   147|        source="manual_cash_correction",
   148|        action="manual_cash_correction",
   149|        entity_type="transaction",
   150|        entity_id=transaction_id,
   151|        new_values={"account_id": account_id, "currency": currency.upper(), "amount": str(amount)},
   152|        user_text_note=note,
   153|        confirmed=True,
   154|        created_by="system",
   155|    )
   156|    conn.commit()
   157|    return audit_id
   158|


Explanation:
**Updating cash services**

I need to update where cash is being tracked and implement new services. I'll look for the right source and use the "write_file" function to create a new file. There might be a need for migrations to patch the system and update schema tests to version 8 while addressing columns. I think adding tests in one file could be a good idea. Also, I should consider backup and restore tests if they already exist.


     1|from __future__ import annotations
     2|
     3|from pathlib import Path
     4|
     5|import pytest
     6|
     7|from jarvis_finance.cli.main import main
     8|from jarvis_finance.crypto.assets import create_crypto_asset
     9|from jarvis_finance.runtime.backup import backup_runtime_db, restore_runtime_db, verify_backup
    10|from jarvis_finance.storage.database import connect
    11|from jarvis_finance.storage.migrations import apply_migrations
    12|
    13|
    14|def _seed_db(db_path: Path, marker: str = "demo") -> None:
    15|    conn = connect(db_path)
    16|    apply_migrations(conn)
    17|    create_crypto_asset(conn, coin_name=f"Synthetic {marker}", symbol=marker.upper()[:6], coingecko_id=f"synthetic-{marker}")
    18|    conn.close()
    19|
    20|
    21|def test_backup_runtime_db_creates_backup_outside_repo(tmp_path: Path) -> None:
    22|    repo = tmp_path / "repo"
    23|    repo.mkdir()
    24|    runtime = tmp_path / "runtime"
    25|    db = runtime / "data" / "finance.sqlite3"
    26|    backups = runtime / "backups"
    27|    _seed_db(db)
    28|
    29|    result = backup_runtime_db(db_path=db, backups_dir=backups, repo_root=repo, timestamp="20260101T000000Z")
    30|
    31|    assert result.backup_file.exists()
    32|    assert result.checksum_file.exists()
    33|    assert result.backup_file.parent == backups.resolve()
    34|    assert not result.backup_file.resolve().is_relative_to(repo.resolve())
    35|    assert len(result.sha256) == 64
    36|
    37|
    38|def test_verify_backup_checks_checksum(tmp_path: Path) -> None:
    39|    repo = tmp_path / "repo"
    40|    repo.mkdir()
    41|    runtime = tmp_path / "runtime"
    42|    db = runtime / "data" / "finance.sqlite3"
    43|    _seed_db(db)
    44|    backup = backup_runtime_db(db_path=db, backups_dir=runtime / "backups", repo_root=repo)
    45|
    46|    verified = verify_backup(backup_file=backup.backup_file, repo_root=repo)
    47|    assert verified.exists is True
    48|    assert verified.checksum_ok is True
    49|
    50|    backup.backup_file.write_bytes(backup.backup_file.read_bytes() + b"tamper")
    51|    tampered = verify_backup(backup_file=backup.backup_file, repo_root=repo)
    52|    assert tampered.checksum_ok is False
    53|
    54|
    55|def test_restore_runtime_db_requires_yes_and_restores_test_db(tmp_path: Path) -> None:
    56|    repo = tmp_path / "repo"
    57|    repo.mkdir()
    58|    runtime = tmp_path / "runtime"
    59|    db = runtime / "data" / "finance.sqlite3"
    60|    _seed_db(db, marker="before")
    61|    backup = backup_runtime_db(db_path=db, backups_dir=runtime / "backups", repo_root=repo)
    62|
    63|    conn = connect(db)
    64|    conn.execute("DELETE FROM crypto_assets")
    65|    conn.commit()
    66|    conn.close()
    67|
    68|    with pytest.raises(PermissionError):
    69|        restore_runtime_db(backup_file=backup.backup_file, db_path=db, repo_root=repo, yes=False)
    70|
    71|    result = restore_runtime_db(backup_file=backup.backup_file, db_path=db, repo_root=repo, yes=True)
    72|    assert result.verified is True
    73|    conn = connect(db)
    74|    count = conn.execute("SELECT COUNT(*) AS n FROM crypto_assets").fetchone()["n"]
    75|    conn.close()
    76|    assert count == 1
    77|
    78|
    79|def test_backup_cli_verify_and_restore(monkeypatch, tmp_path: Path, capsys) -> None:
    80|    runtime = tmp_path / "runtime"
    81|    monkeypatch.setenv("JARVIS_FINANCE_RUNTIME_DIR", str(runtime))
    82|    db = runtime / "data" / "finance.sqlite3"
    83|    _seed_db(db)
    84|
    85|    backup_rc = main(["backup-runtime-db"])
    86|    backup_out = capsys.readouterr().out
    87|    assert backup_rc == 0
    88|    assert "RUNTIME_DB_BACKUP" in backup_out
    89|    backup_file = next(part.split("=", 1)[1] for part in backup_out.split() if part.startswith("backup_file="))
    90|    assert str(runtime / "backups") in backup_file
    91|
    92|    verify_rc = main(["verify-backup", "--file", backup_file])
    93|    verify_out = capsys.readouterr().out
    94|    assert verify_rc == 0
    95|    assert "checksum_ok=true" in verify_out
    96|
    97|    conn = connect(db)
    98|    conn.execute("DELETE FROM crypto_assets")
    99|    conn.commit()
   100|    conn.close()
   101|    restore_rc = main(["restore-runtime-db", "--from", backup_file, "--yes"])
   102|    restore_out = capsys.readouterr().out
   103|    assert restore_rc == 0
   104|    assert "RUNTIME_DB_RESTORE" in restore_out
   105|
   106|    conn = connect(db)
   107|    count = conn.execute("SELECT COUNT(*) AS n FROM crypto_assets").fetchone()["n"]
   108|    conn.close()
   109|    assert count == 1
   110|


Explanation:
**Planning database tests**

I’m considering adding a test restore to a different test database—maybe one that's already been restored. That makes me think about the patch migrations that we’ll need to apply. It’s important to keep things organized, so I’ll ensure all the necessary changes are made. I want to make sure I’m following the right steps to keep everything working smoothly!


     1|from __future__ import annotations
     2|
     3|import hashlib
     4|from datetime import datetime, timezone
     5|from sqlite3 import Connection
     6|
     7|from .schema import INITIAL_SCHEMA_SQL
     8|
     9|MIGRATION_VERSION = 7
    10|MIGRATION_NAME = "007_broker_import_execution_plans"
    11|
    12|INSTRUMENT_OPTIONAL_COLUMNS = {
    13|    "position_category": "TEXT",
    14|    "ter": "TEXT",
    15|    "distribution_policy": "TEXT",
    16|    "index_name": "TEXT",
    17|    "fund_domicile": "TEXT",
    18|    "benchmark": "TEXT",
    19|}
    20|
    21|TEXT_AFFINITY_COLUMNS = {
    22|    "transactions": {"quantity", "price_original", "gross_amount_original", "fee_original", "tax_original", "net_amount_original", "fx_rate_to_chf", "gross_amount_chf", "fee_chf", "tax_chf", "net_amount_chf"},
    23|    "crypto_holdings": {"quantity", "legacy_snapshot_value_original", "legacy_snapshot_value_chf"},
    24|    "crypto_transactions": {"quantity", "price_original", "gross_amount_original", "fee_quantity", "fee_original", "fx_rate_to_chf", "amount_chf"},
    25|    "crypto_prices": {"price", "market_cap", "volume_24h", "change_24h_pct"},
    26|}
    27|
    28|
    29|def utc_now() -> str:
    30|    return datetime.now(timezone.utc).isoformat()
    31|
    32|
    33|def checksum_sql(sql: str) -> str:
    34|    return hashlib.sha256(sql.encode("utf-8")).hexdigest()
    35|
    36|
    37|def get_schema_version(conn: Connection) -> int:
    38|    row = conn.execute("SELECT MAX(version) AS version FROM schema_migrations").fetchone()
    39|    return int(row["version"] or 0) if row else 0
    40|
    41|
    42|def _table_columns(conn: Connection, table: str) -> dict[str, str]:
    43|    return {row["name"]: (row["type"] or "") for row in conn.execute(f"PRAGMA table_info({table})").fetchall()}
    44|
    45|
    46|def _add_missing_instrument_columns(conn: Connection) -> None:
    47|    existing = _table_columns(conn, "instruments")
    48|    for name, col_type in INSTRUMENT_OPTIONAL_COLUMNS.items():
    49|        if name not in existing:
    50|            conn.execute(f"ALTER TABLE instruments ADD COLUMN {name} {col_type}")
    51|
    52|
    53|def _rebuild_table_with_text_columns(conn: Connection, table: str, text_columns: set[str]) -> None:
    54|    cols = conn.execute(f"PRAGMA table_info({table})").fetchall()
    55|    if not cols:
    56|        return
    57|    needs_rebuild = any(row["name"] in text_columns and (row["type"] or "").upper() != "TEXT" for row in cols)
    58|    if not needs_rebuild:
    59|        return
    60|    tmp = f"{table}__text_migration"
    61|    col_defs: list[str] = []
    62|    pk_cols = [row["name"] for row in cols if row["pk"]]
    63|    for row in cols:
    64|        name = row["name"]
    65|        col_type = "TEXT" if name in text_columns else (row["type"] or "TEXT")
    66|        parts = [name, col_type]
    67|        if row["pk"] and len(pk_cols) == 1:
    68|            parts.append("PRIMARY KEY")
    69|        if row["notnull"]:
    70|            parts.append("NOT NULL")
    71|        if row["dflt_value"] is not None:
    72|            parts.append(f"DEFAULT {row['dflt_value']}")
    73|        col_defs.append(" ".join(parts))
    74|    if len(pk_cols) > 1:
    75|        col_defs.append("PRIMARY KEY (" + ", ".join(pk_cols) + ")")
    76|    conn.execute(f"DROP TABLE IF EXISTS {tmp}")
    77|    conn.execute(f"CREATE TABLE {tmp} (" + ", ".join(col_defs) + ")")
    78|    names = [row["name"] for row in cols]
    79|    select_exprs = [f"CAST({name} AS TEXT)" if name in text_columns else name for name in names]
    80|    conn.execute(f"INSERT INTO {tmp} ({', '.join(names)}) SELECT {', '.join(select_exprs)} FROM {table}")
    81|    conn.execute(f"DROP TABLE {table}")
    82|    conn.execute(f"ALTER TABLE {tmp} RENAME TO {table}")
    83|    if table == "crypto_holdings":
    84|        conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_crypto_holdings_asset_wallet ON crypto_holdings(asset_id, wallet_id)")
    85|
    86|
    87|def _add_missing_columns(conn: Connection, table: str, columns: dict[str, str]) -> None:
    88|    existing = _table_columns(conn, table)
    89|    for name, col_type in columns.items():
    90|        if name not in existing:
    91|            conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {col_type}")
    92|
    93|
    94|def _create_broker_bank_mapping_tables(conn: Connection) -> None:
    95|    conn.executescript(
    96|        """
    97|        CREATE TABLE IF NOT EXISTS instrument_mappings (
    98|            mapping_id TEXT PRIMARY KEY,
    99|            source_name TEXT,
   100|            source_platform TEXT NOT NULL,
   101|            source_account TEXT,
   102|            source_label TEXT NOT NULL,
   103|            normalized_name TEXT NOT NULL,
   104|            isin TEXT,
   105|            ticker TEXT,
   106|            exchange TEXT,
   107|            currency TEXT,
   108|            asset_class TEXT NOT NULL DEFAULT 'other',
   109|            source_instrument_name TEXT,
   110|            source_isin TEXT,
   111|            source_ticker TEXT,
   112|            source_exchange TEXT,
   113|            source_currency TEXT,
   114|            instrument_id TEXT REFERENCES instruments(instrument_id),
   115|            mapping_status TEXT NOT NULL DEFAULT 'needs_manual_review',
   116|            confidence TEXT NOT NULL DEFAULT '0',
   117|            quality_flags_json TEXT,
   118|            notes TEXT,
   119|            created_at TEXT NOT NULL,
   120|            updated_at TEXT
   121|        );
   122|
   123|        CREATE TABLE IF NOT EXISTS platform_account_mappings (
   124|            mapping_id TEXT PRIMARY KEY,
   125|            source_name TEXT,
   126|            source_platform TEXT NOT NULL,
   127|            source_platform_label TEXT,
   128|            source_account_label TEXT NOT NULL,
   129|            normalized_platform TEXT NOT NULL,
   130|            normalized_account_name TEXT NOT NULL,
   131|            source_account_type TEXT,
   132|            account_type TEXT NOT NULL DEFAULT 'other',
   133|            source_currency TEXT,
   134|            currency TEXT,
   135|            platform_id TEXT REFERENCES platforms(platform_id),
   136|            account_id TEXT REFERENCES accounts(account_id),
   137|            internal_platform_id TEXT REFERENCES platforms(platform_id),
   138|            internal_account_id TEXT REFERENCES accounts(account_id),
   139|            mapping_status TEXT NOT NULL DEFAULT 'needs_manual_review',
   140|            quality_flags_json TEXT,
   141|            notes TEXT,
   142|            created_at TEXT NOT NULL,
   143|            updated_at TEXT
   144|        );
   145|
   146|        CREATE TABLE IF NOT EXISTS broker_import_dry_runs (
   147|            dry_run_id TEXT PRIMARY KEY,
   148|            source_name TEXT,
   149|            source_platform TEXT NOT NULL,
   150|            source_file_label TEXT,
   151|            source_file_type TEXT NOT NULL,
   152|            source_hash TEXT,
   153|            source_filename_hash TEXT,
   154|            detected_snapshot_date TEXT,
   155|            snapshot_date_status TEXT NOT NULL DEFAULT 'missing',
   156|            detected_sections_json TEXT NOT NULL,
   157|            field_coverage_json TEXT,
   158|            rows_total INTEGER NOT NULL DEFAULT 0,
   159|            rows_position_candidates INTEGER NOT NULL DEFAULT 0,
   160|            rows_cash_candidates INTEGER NOT NULL DEFAULT 0,
   161|            candidate_positions INTEGER NOT NULL DEFAULT 0,
   162|            candidate_cash_rows INTEGER NOT NULL DEFAULT 0,
   163|            mapped_positions INTEGER NOT NULL DEFAULT 0,
   164|            blocked_positions INTEGER NOT NULL DEFAULT 0,
   165|            warnings_count INTEGER NOT NULL DEFAULT 0,
   166|            errors_count INTEGER NOT NULL DEFAULT 0,
   167|            quality_flags_json TEXT NOT NULL,
   168|            summary_json TEXT NOT NULL DEFAULT '{}',
   169|            created_at TEXT NOT NULL,
   170|            notes TEXT
   171|        );
   172|        """
   173|    )
   174|    _add_missing_columns(conn, "instrument_mappings", {
   175|        "source_name": "TEXT",
   176|        "source_platform": "TEXT",
   177|        "source_account": "TEXT",
   178|        "source_label": "TEXT",
   179|        "normalized_name": "TEXT",
   180|        "isin": "TEXT",
   181|        "ticker": "TEXT",
   182|        "exchange": "TEXT",
   183|        "currency": "TEXT",
   184|        "asset_class": "TEXT DEFAULT 'other'",
   185|        "confidence": "TEXT DEFAULT '0'",
   186|    })
   187|    _add_missing_columns(conn, "platform_account_mappings", {
   188|        "source_platform": "TEXT",
   189|        "normalized_platform": "TEXT",
   190|        "normalized_account_name": "TEXT",
   191|        "account_type": "TEXT DEFAULT 'other'",
   192|        "currency": "TEXT",
   193|        "internal_platform_id": "TEXT",
   194|        "internal_account_id": "TEXT",
   195|    })
   196|    _add_missing_columns(conn, "broker_import_dry_runs", {
   197|        "source_platform": "TEXT",
   198|        "source_filename_hash": "TEXT",
   199|        "detected_snapshot_date": "TEXT",
   200|        "snapshot_date_status": "TEXT DEFAULT 'missing'",
   201|        "candidate_positions": "INTEGER DEFAULT 0",
   202|        "candidate_cash_rows": "INTEGER DEFAULT 0",
   203|        "mapped_positions": "INTEGER DEFAULT 0",
   204|        "blocked_positions": "INTEGER DEFAULT 0",
   205|        "warnings_count": "INTEGER DEFAULT 0",
   206|        "errors_count": "INTEGER DEFAULT 0",
   207|        "summary_json": "TEXT DEFAULT '{}'",
   208|        "session_status": "TEXT DEFAULT 'active'",
   209|        "is_current": "INTEGER DEFAULT 0",
   210|        "archived_at": "TEXT",
   211|        "discarded_at": "TEXT",
   212|    })
   213|
   214|
   215|def _create_broker_import_review_items(conn: Connection) -> None:
   216|    conn.executescript(
   217|        """
   218|        CREATE TABLE IF NOT EXISTS broker_import_review_items (
   219|            review_item_id TEXT PRIMARY KEY,
   220|            dry_run_id TEXT NOT NULL REFERENCES broker_import_dry_runs(dry_run_id),
   221|            source_platform TEXT NOT NULL,
   222|            source_file_type TEXT NOT NULL,
   223|            source_row_ref TEXT NOT NULL,
   224|            row_hash TEXT NOT NULL,
   225|            source_label TEXT,
   226|            normalized_name TEXT,
   227|            detected_asset_class TEXT,
   228|            detected_currency TEXT,
   229|            detected_quantity_present INTEGER NOT NULL DEFAULT 0,
   230|            detected_market_value_present INTEGER NOT NULL DEFAULT 0,
   231|            isin TEXT,
   232|            ticker TEXT,
   233|            exchange TEXT,
   234|            mapped_instrument_id TEXT REFERENCES instruments(instrument_id),
   235|            mapped_account_id TEXT REFERENCES accounts(account_id),
   236|            quality_flags_json TEXT NOT NULL,
   237|            review_status TEXT NOT NULL DEFAULT 'open',
   238|            reviewer_note TEXT,
   239|            created_at TEXT NOT NULL,
   240|            updated_at TEXT
   241|        );
   242|        CREATE INDEX IF NOT EXISTS idx_broker_review_items_dry_run ON broker_import_review_items(dry_run_id);
   243|        CREATE INDEX IF NOT EXISTS idx_broker_review_items_status ON broker_import_review_items(review_status);
   244|        """
   245|    )
   246|    _add_missing_columns(conn, "broker_import_review_items", {
   247|        "import_readiness_status": "TEXT DEFAULT 'not_ready'",
   248|        "reviewer_confirmed": "INTEGER DEFAULT 0",
   249|        "snapshot_date_confirmed": "INTEGER DEFAULT 0",
   250|        "ticker_exchange_confirmed": "INTEGER DEFAULT 0",
   251|        "account_mapping_status": "TEXT DEFAULT 'missing'",
   252|    })
   253|
   254|def _create_broker_import_execution_plans(conn: Connection) -> None:
   255|    conn.executescript(
   256|        """
   257|        CREATE TABLE IF NOT EXISTS broker_import_execution_plans (
   258|            execution_plan_id TEXT PRIMARY KEY,
   259|            dry_run_id TEXT NOT NULL REFERENCES broker_import_dry_runs(dry_run_id),
   260|            review_item_id TEXT NOT NULL REFERENCES broker_import_review_items(review_item_id),
   261|            source_platform TEXT NOT NULL,
   262|            target_account_id TEXT NOT NULL REFERENCES accounts(account_id),
   263|            target_instrument_id TEXT NOT NULL REFERENCES instruments(instrument_id),
   264|            transaction_type TEXT NOT NULL DEFAULT 'initial_position_snapshot',
   265|            snapshot_date TEXT,
   266|            payload_status TEXT NOT NULL,
   267|            payload_quality_flags_json TEXT NOT NULL DEFAULT '[]',
   268|            source_row_hash TEXT NOT NULL,
   269|            planned_write_summary_json TEXT NOT NULL DEFAULT '{}',
   270|            execution_status TEXT NOT NULL DEFAULT 'planned',
   271|            transaction_id TEXT REFERENCES transactions(transaction_id),
   272|            created_at TEXT NOT NULL,
   273|            updated_at TEXT,
   274|            executed_at TEXT,
   275|            notes TEXT
   276|        );
   277|        CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_execution_plan_review_item ON broker_import_execution_plans(review_item_id);
   278|        CREATE UNIQUE INDEX IF NOT EXISTS idx_broker_execution_plan_source_hash ON broker_import_execution_plans(source_row_hash, target_account_id, target_instrument_id, transaction_type);
   279|        """
   280|    )
   281|
   282|def _apply_compat_migrations(conn: Connection) -> None:
   283|    _add_missing_instrument_columns(conn)
   284|    for table, text_columns in TEXT_AFFINITY_COLUMNS.items():
   285|        _rebuild_table_with_text_columns(conn, table, text_columns)
   286|    _create_broker_bank_mapping_tables(conn)
   287|    _create_broker_import_review_items(conn)
   288|    _create_broker_import_execution_plans(conn)
   289|
   290|
   291|def apply_migrations(conn: Connection) -> None:
   292|    conn.executescript(INITIAL_SCHEMA_SQL)
   293|    existing_initial = conn.execute("SELECT 1 FROM schema_migrations WHERE version = 1").fetchone()
   294|    if not existing_initial:
   295|        conn.execute(
   296|            "INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)",
   297|            (1, "001_initial_schema", utc_now(), checksum_sql(INITIAL_SCHEMA_SQL)),
   298|        )
   299|    _apply_compat_migrations(conn)
   300|    existing = conn.execute("SELECT 1 FROM schema_migrations WHERE version = ?", (MIGRATION_VERSION,)).fetchone()
   301|    if not existing:
   302|        conn.execute(
   303|            "INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)",
   304|            (MIGRATION_VERSION, MIGRATION_NAME, utc_now(), checksum_sql(MIGRATION_NAME)),
   305|        )
   306|    conn.commit()
   307|


patch

MIGRATION_VERSION = 7
MIGRATION_NAME = "007_broker_import_execution_plans"

MIGRATION_VERSION = 8
MIGRATION_NAME = "008_transaction_void_corrections"


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py
@@ -6,8 +6,8 @@

 from .schema import INITIAL_SCHEMA_SQL

-MIGRATION_VERSION = 7
-MIGRATION_NAME = "007_broker_import_execution_plans"
+MIGRATION_VERSION = 8
+MIGRATION_NAME = "008_transaction_void_corrections"

 INSTRUMENT_OPTIONAL_COLUMNS = {
     "position_category": "TEXT",


def _add_transaction_void_columns(conn: Connection) -> None:
    _add_missing_columns(conn, "transactions", {
        "is_voided": "INTEGER NOT NULL DEFAULT 0",
        "voided_at": "TEXT",
        "void_reason": "TEXT",
        "voided_by": "TEXT",
        "correction_of_transaction_id": "TEXT REFERENCES transactions(transaction_id)",
        "correction_reason": "TEXT",
    })
    conn.execute("CREATE INDEX IF NOT EXISTS idx_transactions_voided ON transactions(is_voided)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_transactions_correction_of ON transactions(correction_of_transaction_id)")

def _apply_compat_migrations(conn: Connection) -> None:
    _add_missing_instrument_columns(conn)
    for table, text_columns in TEXT_AFFINITY_COLUMNS.items():
        _rebuild_table_with_text_columns(conn, table, text_columns)
    _create_broker_bank_mapping_tables(conn)
    _create_broker_import_review_items(conn)
    _create_broker_import_execution_plans(conn)
    _add_transaction_void_columns(conn)


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/storage/migrations.py
@@ -279,6 +279,19 @@
         """
     )

+def _add_transaction_void_columns(conn: Connection) -> None:
+    _add_missing_columns(conn, "transactions", {
+        "is_voided": "INTEGER NOT NULL DEFAULT 0",
+        "voided_at": "TEXT",
+        "void_reason": "TEXT",
+        "voided_by": "TEXT",
+        "correction_of_transaction_id": "TEXT REFERENCES transactions(transaction_id)",
+        "correction_reason": "TEXT",
+    })
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_transactions_voided ON transactions(is_voided)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_transactions_correction_of ON transactions(correction_of_transaction_id)")
+
+
 def _apply_compat_migrations(conn: Connection) -> None:
     _add_missing_instrument_columns(conn)
     for table, text_columns in TEXT_AFFINITY_COLUMNS.items():
@@ -286,6 +299,7 @@
     _create_broker_bank_mapping_tables(conn)
     _create_broker_import_review_items(conn)
     _create_broker_import_execution_plans(conn)
+    _add_transaction_void_columns(conn)

 def apply_migrations(conn: Connection) -> None:


where = "WHERE instrument_id IS NOT NULL AND COALESCE(is_voided, 0) = 0"

where = "WHERE instrument_id IS NOT NULL"


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/ledger/positions.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/ledger/positions.py
@@ -93,7 +93,7 @@
     result = PositionCalculationResult(as_of_date=as_of_date)
     lots: dict[tuple[str, str], WeightedAverageLot] = {}
     params: tuple[object, ...] = ()
-    where = "WHERE instrument_id IS NOT NULL"
+    where = "WHERE instrument_id IS NOT NULL AND COALESCE(is_voided, 0) = 0"
     if as_of_date:
         where += " AND trade_date <= ?"
         params = (as_of_date,)


    where = "WHERE COALESCE(is_voided, 0) = 0"
    if as_of_date:
        where += " AND trade_date <= ?"
        params = (as_of_date,)

    where = ""
    if as_of_date:
        where = "WHERE trade_date <= ?"
        params = (as_of_date,)


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/ledger/cash.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/ledger/cash.py
@@ -65,9 +65,9 @@
 def calculate_cash_balances(conn: Connection, *, as_of_date: str | None = None) -> CashCalculationResult:
     result = CashCalculationResult()
     params: tuple[object, ...] = ()
-    where = ""
+    where = "WHERE COALESCE(is_voided, 0) = 0"
     if as_of_date:
-        where = "WHERE trade_date <= ?"
+        where += " AND trade_date <= ?"
         params = (as_of_date,)
     rows = conn.execute(
         f"""


Explanation:
**Organizing service file needs**

I need to create a service file for import execution. Specifically, I'll develop `jarvis_finance/imports/execution.py`, implementing `execute_execution_plan` for testing and future use. It's crucial to have a stable UUID and to incorporate duplicate checks. I should also make sure to understand the audit signature requirements. Overall, I'm focused on ensuring everything is organized and functional for smooth operation.


     1|from __future__ import annotations
     2|
     3|import json
     4|import uuid
     5|from datetime import datetime, timezone
     6|from sqlite3 import Connection
     7|from typing import Any
     8|
     9|
    10|def utc_now() -> str:
    11|    return datetime.now(timezone.utc).isoformat()
    12|
    13|
    14|def record_audit_event(
    15|    conn: Connection,
    16|    *,
    17|    source: str,
    18|    action: str,
    19|    entity_type: str,
    20|    entity_id: str,
    21|    old_values: dict[str, Any] | None = None,
    22|    new_values: dict[str, Any] | None = None,
    23|    user_text_note: str | None = None,
    24|    confirmed: bool = True,
    25|    created_by: str = "system",
    26|) -> str:
    27|    audit_id = str(uuid.uuid4())
    28|    now = utc_now()
    29|    conn.execute(
    30|        """
    31|        INSERT INTO audit_log(
    32|            audit_id, timestamp, source, action, entity_type, entity_id,
    33|            old_values_json, new_values_json, user_text_note, confirmed,
    34|            confirmation_timestamp, created_by, created_at
    35|        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    36|        """,
    37|        (
    38|            audit_id,
    39|            now,
    40|            source,
    41|            action,
    42|            entity_type,
    43|            entity_id,
    44|            json.dumps(old_values or {}, sort_keys=True),
    45|            json.dumps(new_values or {}, sort_keys=True),
    46|            user_text_note,
    47|            1 if confirmed else 0,
    48|            now if confirmed else None,
    49|            created_by,
    50|            now,
    51|        ),
    52|    )
    53|    return audit_id
    54|


from __future__ import annotations

import uuid
from sqlite3 import Connection

from jarvis_finance.audit.log import record_audit_event
from jarvis_finance.imports.common import utc_now
from jarvis_finance.quality.alerts import create_alert

def void_transaction(conn: Connection, *, transaction_id: str, note: str, created_by: str = "system") -> str:
    """Mark a productive transaction as voided without deleting it.

    The original row remains in the ledger for auditability. Position and cash
    calculations ignore rows with ``is_voided=1``.
    """
    if not note.strip():
        raise ValueError("void_transaction requires a note")
    row = conn.execute("SELECT * FROM transactions WHERE transaction_id=?", (transaction_id,)).fetchone()
    if row is None:
        raise ValueError(f"transaction not found: {transaction_id}")
    if int(row["is_voided"] or 0) == 1:
        raise ValueError(f"transaction is already voided: {transaction_id}")
    now = utc_now()
    conn.execute(
        """
        UPDATE transactions
        SET is_voided=1, voided_at=?, void_reason=?, voided_by=?, updated_at=?
        WHERE transaction_id=?
        """,
        (now, note, created_by, now, transaction_id),
    )
    audit_id = record_audit_event(
        conn,
        source="ledger_corrections",
        action="void_transaction",
        entity_type="transaction",
        entity_id=transaction_id,
        old_values={"is_voided": int(row["is_voided"] or 0), "quality_status": row["quality_status"]},
        new_values={"is_voided": 1, "voided_at": now, "void_reason": note},
        user_text_note=note,
        confirmed=True,
        created_by=created_by,
    )
    create_alert(
        conn,
        priority="warnung",
        category="ledger",
        entity_type="transaction",
        entity_id=transaction_id,
        rule_id="voided_transaction",
        message="A ledger transaction was voided and remains excluded from active position/cash calculations.",
        evidence={"transaction_id": transaction_id, "voided_at": now},
    )
    conn.commit()
    return audit_id

def record_correction_audit(
    conn: Connection,
    *,
    old_transaction_id: str,
    new_transaction_id: str,
    note: str,
    created_by: str = "system",
) -> str:
    """Link an existing transaction to its correction transaction via audit.

    Creation of the replacement transaction remains with the caller/importer so
    quantities and source-specific fields stay under that workflow's validation.
    """
    if not note.strip():
        raise ValueError("correction transaction requires a note")
    old_row = conn.execute("SELECT transaction_id FROM transactions WHERE transaction_id=?", (old_transaction_id,)).fetchone()
    new_row = conn.execute("SELECT transaction_id FROM transactions WHERE transaction_id=?", (new_transaction_id,)).fetchone()
    if old_row is None or new_row is None:
        raise ValueError("old and new transactions must both exist")
    now = utc_now()
    conn.execute(
        """
        UPDATE transactions
        SET correction_of_transaction_id=?, correction_reason=?, updated_at=?
        WHERE transaction_id=?
        """,
        (old_transaction_id, note, now, new_transaction_id),
    )
    audit_id = record_audit_event(
        conn,
        source="ledger_corrections",
        action="correction_transaction",
        entity_type="transaction",
        entity_id=new_transaction_id,
        old_values={"old_transaction_id": old_transaction_id},
        new_values={"new_transaction_id": new_transaction_id, "correction_of_transaction_id": old_transaction_id},
        user_text_note=note,
        confirmed=True,
        created_by=created_by,
    )
    conn.commit()
    return audit_id


from __future__ import annotations

import json
import uuid
from dataclasses import dataclass
from sqlite3 import Connection

from jarvis_finance.audit.log import record_audit_event
from jarvis_finance.imports.common import utc_now

@dataclass(frozen=True)
class ExecutionImportResult:
    transaction_id: str
    imported: bool

def execute_initial_snapshot_plan(conn: Connection, *, execution_plan_id: str, note: str, created_by: str = "system") -> ExecutionImportResult:
    if not note.strip():
        raise ValueError("execution import requires a note")
    plan = conn.execute("SELECT * FROM broker_import_execution_plans WHERE execution_plan_id=?", (execution_plan_id,)).fetchone()
    if plan is None:
        raise ValueError(f"execution plan not found: {execution_plan_id}")
    if plan["payload_status"] != "ready":
        raise ValueError(f"execution plan is not ready: {execution_plan_id}")
    if plan["execution_status"] == "imported" or plan["transaction_id"]:
        raise ValueError(f"execution plan already imported: {execution_plan_id}")
    existing = conn.execute(
        """
        SELECT transaction_id FROM transactions
        WHERE source_type='broker_import_reviewed_snapshot'
          AND (source_id=? OR external_transaction_id=? OR row_hash=?)
        LIMIT 1
        """,
        (plan["execution_plan_id"], plan["execution_plan_id"], plan["source_row_hash"]),
    ).fetchone()
    if existing:
        raise ValueError("execution plan or source row hash already imported")
    payload = json.loads(plan["planned_write_summary_json"] or "{}")
    required = ["account_id", "instrument_id", "trade_date", "quantity", "currency_original"]
    missing = [key for key in required if not payload.get(key)]
    if missing:
        raise ValueError("execution payload missing required fields: " + ", ".join(missing))
    tx_id = "tx-" + str(uuid.uuid4())
    now = utc_now()
    quality = "warning" if (payload.get("fx_status") == "missing" or "cost_basis_uncertain" in (plan["payload_quality_flags_json"] or "")) else "ok"
    conn.execute(
        """
        INSERT INTO transactions(
          transaction_id, transaction_type, account_id, instrument_id, trade_date, settlement_date,
          quantity, price_original, gross_amount_original, fee_original, tax_original, net_amount_original,
          currency_original, fx_rate_to_chf, fx_source, fx_status, gross_amount_chf, fee_chf, tax_chf, net_amount_chf,
          source_type, source_id, external_transaction_id, row_hash, is_confirmed, quality_status, notes, created_at, updated_at
        ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
        """,
        (
            tx_id, "initial_position_snapshot", payload["account_id"], payload["instrument_id"], payload["trade_date"], payload["trade_date"],
            payload["quantity"], payload.get("price_original"), None, None, None, None,
            payload.get("currency_original"), payload.get("fx_rate_to_chf"), payload.get("fx_source") or "missing_in_source", payload.get("fx_status", "missing"), payload.get("gross_amount_chf"), None, None, None,
            "broker_import_reviewed_snapshot", plan["execution_plan_id"], plan["execution_plan_id"], plan["source_row_hash"], 1, quality,
            note, now, now,
        ),
    )
    conn.execute(
        "UPDATE broker_import_execution_plans SET execution_status='imported', transaction_id=?, executed_at=?, updated_at=?, notes=? WHERE execution_plan_id=?",
        (tx_id, now, now, note, execution_plan_id),
    )
    conn.execute(
        "UPDATE broker_import_review_items SET review_status='imported', reviewer_note=?, updated_at=? WHERE review_item_id=?",
        (note, now, plan["review_item_id"]),
    )
    record_audit_event(
        conn,
        source="broker_import_execution_plan",
        action="execute_mini_import",
        entity_type="transaction",
        entity_id=tx_id,
        old_values={},
        new_values={"execution_plan_id": execution_plan_id, "review_item_id": plan["review_item_id"], "transaction_type": "initial_position_snapshot", "quality_status": quality},
        user_text_note=note,
        confirmed=True,
        created_by=created_by,
    )
    record_audit_event(
        conn,
        source="manual_review_queue",
        action="mark_review_item_imported",
        entity_type="broker_import_review_item",
        entity_id=plan["review_item_id"],
        old_values={"review_status": "resolved"},
        new_values={"review_status": "imported", "transaction_id": tx_id},
        user_text_note=note,
        confirmed=True,
        created_by=created_by,
    )
    conn.commit()
    return ExecutionImportResult(transaction_id=tx_id, imported=True)


Explanation:
**Generating tests and helpers**

I'm focusing on generating tests and setup helpers. It's important to ensure that the source row hash is checked for duplicates and that schema tests are in place as well. This will help maintain data integrity and functionality. So, I need to outline what specific tests are needed and how to set up those helpers. Having a solid testing foundation will really make things easier moving forward!


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
@@ -15,7 +15,7 @@
 def test_schema_version_recorded() -> None:
     conn = connect_memory()
     apply_migrations(conn)
-    assert get_schema_version(conn) == 7
+    assert get_schema_version(conn) == 8

 def test_decimal_sensitive_columns_use_text_affinity() -> None:


    execution_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_execution_plans)").fetchall()}
    assert {"execution_plan_id", "dry_run_id", "review_item_id", "source_platform", "target_account_id", "target_instrument_id", "transaction_type", "snapshot_date", "payload_status", "payload_quality_flags_json", "source_row_hash", "planned_write_summary_json", "execution_status", "transaction_id"}.issubset(execution_cols)
    transaction_cols = {row["name"] for row in conn.execute("PRAGMA table_info(transactions)").fetchall()}
    assert {"is_voided", "voided_at", "void_reason", "voided_by", "correction_of_transaction_id", "correction_reason"}.issubset(transaction_cols)


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_schema.py
@@ -44,3 +44,5 @@
     assert {"review_item_id", "dry_run_id", "source_platform", "source_row_ref", "row_hash", "source_label", "normalized_name", "detected_asset_class", "detected_currency", "quality_flags_json", "review_status", "import_readiness_status", "reviewer_confirmed", "snapshot_date_confirmed", "ticker_exchange_confirmed", "account_mapping_status"}.issubset(review_cols)
     execution_cols = {row["name"] for row in conn.execute("PRAGMA table_info(broker_import_execution_plans)").fetchall()}
     assert {"execution_plan_id", "dry_run_id", "review_item_id", "source_platform", "target_account_id", "target_instrument_id", "transaction_type", "snapshot_date", "payload_status", "payload_quality_flags_json", "source_row_hash", "planned_write_summary_json", "execution_status", "transaction_id"}.issubset(execution_cols)
+    transaction_cols = {row["name"] for row in conn.execute("PRAGMA table_info(transactions)").fetchall()}
+    assert {"is_voided", "voided_at", "void_reason", "voided_by", "correction_of_transaction_id", "correction_reason"}.issubset(transaction_cols)


from __future__ import annotations

import json
from decimal import Decimal

import pytest

from jarvis_finance.imports.execution import execute_initial_snapshot_plan
from jarvis_finance.ledger.corrections import record_correction_audit, void_transaction
from jarvis_finance.ledger.positions import calculate_positions
from jarvis_finance.storage.database import connect_memory
from jarvis_finance.storage.migrations import apply_migrations

def setup_conn():
    conn = connect_memory()
    apply_migrations(conn)
    conn.execute("INSERT INTO platforms(platform_id,name,platform_type,default_currency,created_at) VALUES('p1','True Wealth','broker','CHF','now')")
    conn.execute("INSERT INTO accounts(account_id,platform_id,account_name,account_type,currency,created_at) VALUES('a1','p1','Synthetic TW','brokerage','CHF','now')")
    conn.execute("INSERT INTO instruments(instrument_id,asset_class,name,ticker,isin,currency,created_at) VALUES('i1','ETF','Synthetic ETF','SETF','CH0000000001','USD','now')")
    conn.execute("""
        INSERT INTO broker_import_dry_runs(
            dry_run_id, source_platform, source_file_type, detected_sections_json, quality_flags_json, created_at
        ) VALUES('dry1','True Wealth','docx','{}','[]','now')
    """)
    conn.execute("""
        INSERT INTO broker_import_review_items(
            review_item_id, dry_run_id, source_platform, source_file_type, source_row_ref, row_hash,
            mapped_account_id, mapped_instrument_id, detected_quantity_present, quality_flags_json,
            review_status, import_readiness_status, reviewer_confirmed, snapshot_date_confirmed, ticker_exchange_confirmed, account_mapping_status, created_at
        ) VALUES('rev1','dry1','True Wealth','docx','row-1','hash-1','a1','i1',1,'[]','resolved','ready_for_import',1,1,1,'selected','now')
    """)
    payload = {
        "account_id": "a1",
        "instrument_id": "i1",
        "trade_date": "2025-12-31",
        "quantity": "1.234500",
        "currency_original": "USD",
        "fx_status": "missing",
    }
    conn.execute("""
        INSERT INTO broker_import_execution_plans(
            execution_plan_id, dry_run_id, review_item_id, source_platform, target_account_id, target_instrument_id,
            transaction_type, snapshot_date, payload_status, payload_quality_flags_json, source_row_hash,
            planned_write_summary_json, execution_status, created_at
        ) VALUES('plan1','dry1','rev1','True Wealth','a1','i1','initial_position_snapshot','2025-12-31','ready',?, 'hash-1', ?, 'planned','now')
    """, (json.dumps(["missing_fx", "cost_basis_uncertain", "snapshot_only"]), json.dumps(payload)))
    return conn

def test_execution_plan_import_blocks_same_execution_plan_id_and_row_hash() -> None:
    conn = setup_conn()
    result = execute_initial_snapshot_plan(conn, execution_plan_id="plan1", note="synthetic reviewed import")
    assert result.imported is True
    tx = conn.execute("SELECT * FROM transactions WHERE transaction_id=?", (result.transaction_id,)).fetchone()
    assert tx["source_id"] == "plan1"
    assert tx["row_hash"] == "hash-1"
    assert tx["quality_status"] == "warning"
    assert Decimal(str(tx["quantity"])) == Decimal("1.234500")

    with pytest.raises(ValueError, match="already imported"):
        execute_initial_snapshot_plan(conn, execution_plan_id="plan1", note="retry same plan")

    conn.execute("""
        INSERT INTO broker_import_review_items(
            review_item_id, dry_run_id, source_platform, source_file_type, source_row_ref, row_hash,
            mapped_account_id, mapped_instrument_id, detected_quantity_present, quality_flags_json,
            review_status, import_readiness_status, reviewer_confirmed, snapshot_date_confirmed, ticker_exchange_confirmed, account_mapping_status, created_at
        ) VALUES('rev2','dry1','True Wealth','docx','row-1-copy','hash-1','a1','i1',1,'[]','resolved','ready_for_import',1,1,1,'selected','now')
    """)
    conn.execute("""
        INSERT INTO broker_import_execution_plans(
            execution_plan_id, dry_run_id, review_item_id, source_platform, target_account_id, target_instrument_id,
            transaction_type, snapshot_date, payload_status, payload_quality_flags_json, source_row_hash,
            planned_write_summary_json, execution_status, created_at
        ) VALUES('plan2','dry1','rev2','True Wealth','a1','i1','initial_position_snapshot','2025-12-31','ready','[]','hash-1', ?, 'planned','now')
    """, (json.dumps({"account_id":"a1","instrument_id":"i1","trade_date":"2025-12-31","quantity":"2","currency_original":"USD","fx_status":"missing"}),))
    with pytest.raises(ValueError, match="row hash already imported|source row hash already imported|already imported"):
        execute_initial_snapshot_plan(conn, execution_plan_id="plan2", note="retry same row")

def test_void_transaction_requires_note_audits_and_excludes_from_position() -> None:
    conn = setup_conn()
    result = execute_initial_snapshot_plan(conn, execution_plan_id="plan1", note="synthetic reviewed import")
    assert calculate_positions(conn).positions[("a1", "i1")].quantity == Decimal("1.234500")

    with pytest.raises(ValueError, match="note"):
        void_transaction(conn, transaction_id=result.transaction_id, note="")

    audit_id = void_transaction(conn, transaction_id=result.transaction_id, note="wrong source row")
    assert audit_id
    assert conn.execute("SELECT is_voided FROM transactions WHERE transaction_id=?", (result.transaction_id,)).fetchone()["is_voided"] == 1
    assert conn.execute("SELECT COUNT(*) AS n FROM audit_log WHERE action='void_transaction'").fetchone()["n"] == 1
    assert conn.execute("SELECT COUNT(*) AS n FROM alerts WHERE rule_id='voided_transaction'").fetchone()["n"] == 1
    assert ("a1", "i1") not in calculate_positions(conn).positions

def test_correction_transaction_creates_audit_link() -> None:
    conn = setup_conn()
    result = execute_initial_snapshot_plan(conn, execution_plan_id="plan1", note="synthetic reviewed import")
    conn.execute("""
        INSERT INTO transactions(transaction_id, transaction_type, account_id, instrument_id, trade_date, quantity, currency_original, fx_status, source_type, is_confirmed, quality_status, notes, created_at)
        VALUES('tx-correction','initial_position_snapshot','a1','i1','2025-12-31','1.000000','USD','missing','manual_correction',1,'warning','synthetic correction','now')
    """)
    audit_id = record_correction_audit(conn, old_transaction_id=result.transaction_id, new_transaction_id="tx-correction", note="replace initial snapshot")
    assert audit_id
    row = conn.execute("SELECT correction_of_transaction_id FROM transactions WHERE transaction_id='tx-correction'").fetchone()
    assert row["correction_of_transaction_id"] == result.transaction_id
    assert conn.execute("SELECT COUNT(*) AS n FROM audit_log WHERE action='correction_transaction'").fetchone()["n"] == 1

def test_missing_fx_initial_snapshot_stays_incomplete_without_fake_total_return() -> None:
    conn = setup_conn()
    execute_initial_snapshot_plan(conn, execution_plan_id="plan1", note="synthetic reviewed import")
    pos = calculate_positions(conn).positions[("a1", "i1")]
    assert pos.data_quality_status == "incomplete"
    assert "missing_fx" in pos.quality_warnings
    assert "missing_market_price" in pos.quality_warnings
    assert pos.total_return_chf is None


Explanation:
**Evaluating message handling**

I'm considering whether I need to duplicate the message, since it seems like regex might not be matching correctly. I think it's important to check this out further. Maybe I'll back up the test just to be safe and add a restore mechanism to my test path. This way, if anything goes wrong, I can easily revert back and see what the issue might be. Let's keep it thorough!

def test_restore_runtime_db_requires_yes_and_restores_test_db(tmp_path: Path) -> None:
    repo = tmp_path / "repo"
    repo.mkdir()
    runtime = tmp_path / "runtime"
    db = runtime / "data" / "finance.sqlite3"
    _seed_db(db, marker="before")
    backup = backup_runtime_db(db_path=db, backups_dir=runtime / "backups", repo_root=repo)

    conn = connect(db)
    conn.execute("DELETE FROM crypto_assets")
    conn.commit()
    conn.close()

    with pytest.raises(PermissionError):
        restore_runtime_db(backup_file=backup.backup_file, db_path=db, repo_root=repo, yes=False)

    result = restore_runtime_db(backup_file=backup.backup_file, db_path=db, repo_root=repo, yes=True)
    assert result.verified is True
    conn = connect(db)
    count = conn.execute("SELECT COUNT(*) AS n FROM crypto_assets").fetchone()["n"]
    conn.close()
    assert count == 1

def test_restore_backup_to_separate_test_db_does_not_modify_productive_db(tmp_path: Path) -> None:
    repo = tmp_path / "repo"
    repo.mkdir()
    runtime = tmp_path / "runtime"
    productive_db = runtime / "data" / "finance.sqlite3"
    restore_db = runtime / "restore-tests" / "finance-restored.sqlite3"
    _seed_db(productive_db, marker="prod")
    backup = backup_runtime_db(db_path=productive_db, backups_dir=runtime / "backups", repo_root=repo)

    conn = connect(productive_db)
    create_crypto_asset(conn, coin_name="Synthetic After Backup", symbol="AFTER", coingecko_id="synthetic-after")
    conn.close()

    result = restore_runtime_db(backup_file=backup.backup_file, db_path=restore_db, repo_root=repo, yes=True)
    assert result.restored_to == restore_db.resolve()

    prod_conn = connect(productive_db)
    restored_conn = connect(restore_db)
    prod_count = prod_conn.execute("SELECT COUNT(*) AS n FROM crypto_assets").fetchone()["n"]
    restored_count = restored_conn.execute("SELECT COUNT(*) AS n FROM crypto_assets").fetchone()["n"]
    prod_conn.close()
    restored_conn.close()
    assert prod_count == 2
    assert restored_count == 1


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_runtime_backup.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_runtime_backup.py
@@ -76,6 +76,32 @@
     assert count == 1

+def test_restore_backup_to_separate_test_db_does_not_modify_productive_db(tmp_path: Path) -> None:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    runtime = tmp_path / "runtime"
+    productive_db = runtime / "data" / "finance.sqlite3"
+    restore_db = runtime / "restore-tests" / "finance-restored.sqlite3"
+    _seed_db(productive_db, marker="prod")
+    backup = backup_runtime_db(db_path=productive_db, backups_dir=runtime / "backups", repo_root=repo)
+
+    conn = connect(productive_db)
+    create_crypto_asset(conn, coin_name="Synthetic After Backup", symbol="AFTER", coingecko_id="synthetic-after")
+    conn.close()
+
+    result = restore_runtime_db(backup_file=backup.backup_file, db_path=restore_db, repo_root=repo, yes=True)
+    assert result.restored_to == restore_db.resolve()
+
+    prod_conn = connect(productive_db)
+    restored_conn = connect(restore_db)
+    prod_count = prod_conn.execute("SELECT COUNT(*) AS n FROM crypto_assets").fetchone()["n"]
+    restored_count = restored_conn.execute("SELECT COUNT(*) AS n FROM crypto_assets").fetchone()["n"]
+    prod_conn.close()
+    restored_conn.close()
+    assert prod_count == 2
+    assert restored_count == 1
+
+
 def test_backup_cli_verify_and_restore(monkeypatch, tmp_path: Path, capsys) -> None:
     runtime = tmp_path / "runtime"
     monkeypatch.setenv("JARVIS_FINANCE_RUNTIME_DIR", str(runtime))


Explanation:
**Running tests and updates**

I need to run some tests and also think about updating the REQUIRED_TABLES. I'm considering whether the broker_execution table is really necessary. It exists, but maybe it's not required? I guess it's fine to keep it for now. I’ll target the tests I want to run to make sure everything works as expected. It feels a little complex, but I’ll sort it out step by step!

terminal

set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src pytest -q tests/unit/test_schema.py tests/unit/test_import_execution_and_corrections.py tests/unit/test_runtime_backup.py tests/unit/test_ledger_cash_wac_positions.py


Output:
....F................                                                    [100%]
=================================== FAILURES ===================================
____ test_execution_plan_import_blocks_same_execution_plan_id_and_row_hash _____

    def test_execution_plan_import_blocks_same_execution_plan_id_and_row_hash() -> None:
        conn = setup_conn()
        result = execute_initial_snapshot_plan(conn, execution_plan_id="plan1", note="synthetic reviewed import")
        assert result.imported is True
        tx = conn.execute("SELECT * FROM transactions WHERE transaction_id=?", (result.transaction_id,)).fetchone()
        assert tx["source_id"] == "plan1"
        assert tx["row_hash"] == "hash-1"
        assert tx["quality_status"] == "warning"
        assert Decimal(str(tx["quantity"])) == Decimal("1.234500")

        with pytest.raises(ValueError, match="already imported"):
            execute_initial_snapshot_plan(conn, execution_plan_id="plan1", note="retry same plan")

        conn.execute("""
            INSERT INTO broker_import_review_items(
                review_item_id, dry_run_id, source_platform, source_file_type, source_row_ref, row_hash,
                mapped_account_id, mapped_instrument_id, detected_quantity_present, quality_flags_json,
                review_status, import_readiness_status, reviewer_confirmed, snapshot_date_confirmed, ticker_exchange_confirmed, account_mapping_status, created_at
            ) VALUES('rev2','dry1','True Wealth','docx','row-1-copy','hash-1','a1','i1',1,'[]','resolved','ready_for_import',1,1,1,'selected','now')
        """)
>       conn.execute("""
            INSERT INTO broker_import_execution_plans(
                execution_plan_id, dry_run_id, review_item_id, source_platform, target_account_id, target_instrument_id,
                transaction_type, snapshot_date, payload_status, payload_quality_flags_json, source_row_hash,
                planned_write_summary_json, execution_status, created_at
            ) VALUES('plan2','dry1','rev2','True Wealth','a1','i1','initial_position_snapshot','2025-12-31','ready','[]','hash-1', ?, 'planned','now')
        """, (json.dumps({"account_id":"a1","instrument_id":"i1","trade_date":"2025-12-31","quantity":"2","currency_original":"USD","fx_status":"missing"}),))
E       sqlite3.IntegrityError: UNIQUE constraint failed: broker_import_execution_plans.source_row_hash, broker_import_execution_plans.target_account_id, broker_import_execution_plans.target_instrument_id, broker_import_execution_plans.transaction_type

tests/unit/test_import_execution_and_corrections.py:71: IntegrityError
=========================== short test summary info ============================
FAILED tests/unit/test_import_execution_and_corrections.py::test_execution_plan_import_blocks_same_execution_plan_id_and_row_hash
1 failed, 20 passed in 4.54s


patch

    conn.execute("INSERT INTO instruments(instrument_id,asset_class,name,ticker,isin,currency,created_at) VALUES('i2','ETF','Synthetic ETF 2','SET2','CH0000000002','USD','now')")
    conn.execute("""
        INSERT INTO broker_import_execution_plans(
            execution_plan_id, dry_run_id, review_item_id, source_platform, target_account_id, target_instrument_id,
            transaction_type, snapshot_date, payload_status, payload_quality_flags_json, source_row_hash,
            planned_write_summary_json, execution_status, created_at
        ) VALUES('plan2','dry1','rev2','True Wealth','a1','i2','initial_position_snapshot','2025-12-31','ready','[]','hash-1', ?, 'planned','now')
    """, (json.dumps({"account_id":"a1","instrument_id":"i2","trade_date":"2025-12-31","quantity":"2","currency_original":"USD","fx_status":"missing"}),))

    conn.execute("""
        INSERT INTO broker_import_execution_plans(
            execution_plan_id, dry_run_id, review_item_id, source_platform, target_account_id, target_instrument_id,
            transaction_type, snapshot_date, payload_status, payload_quality_flags_json, source_row_hash,
            planned_write_summary_json, execution_status, created_at
        ) VALUES('plan2','dry1','rev2','True Wealth','a1','i1','initial_position_snapshot','2025-12-31','ready','[]','hash-1', ?, 'planned','now')
    """, (json.dumps({"account_id":"a1","instrument_id":"i1","trade_date":"2025-12-31","quantity":"2","currency_original":"USD","fx_status":"missing"}),))


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_import_execution_and_corrections.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_import_execution_and_corrections.py
@@ -68,13 +68,14 @@
             review_status, import_readiness_status, reviewer_confirmed, snapshot_date_confirmed, ticker_exchange_confirmed, account_mapping_status, created_at
         ) VALUES('rev2','dry1','True Wealth','docx','row-1-copy','hash-1','a1','i1',1,'[]','resolved','ready_for_import',1,1,1,'selected','now')
     """)
+    conn.execute("INSERT INTO instruments(instrument_id,asset_class,name,ticker,isin,currency,created_at) VALUES('i2','ETF','Synthetic ETF 2','SET2','CH0000000002','USD','now')")
     conn.execute("""
         INSERT INTO broker_import_execution_plans(
             execution_plan_id, dry_run_id, review_item_id, source_platform, target_account_id, target_instrument_id,
             transaction_type, snapshot_date, payload_status, payload_quality_flags_json, source_row_hash,
             planned_write_summary_json, execution_status, created_at
-        ) VALUES('plan2','dry1','rev2','True Wealth','a1','i1','initial_position_snapshot','2025-12-31','ready','[]','hash-1', ?, 'planned','now')
-    """, (json.dumps({"account_id":"a1","instrument_id":"i1","trade_date":"2025-12-31","quantity":"2","currency_original":"USD","fx_status":"missing"}),))
+        ) VALUES('plan2','dry1','rev2','True Wealth','a1','i2','initial_position_snapshot','2025-12-31','ready','[]','hash-1', ?, 'planned','now')
+    """, (json.dumps({"account_id":"a1","instrument_id":"i2","trade_date":"2025-12-31","quantity":"2","currency_original":"USD","fx_status":"missing"}),))
     with pytest.raises(ValueError, match="row hash already imported|source row hash already imported|already imported"):
         execute_initial_snapshot_plan(conn, execution_plan_id="plan2", note="retry same row")


terminal

Output:
.....................                                                    [100%]
21 passed in 4.26s


Explanation:
**Planning the migration steps**

I’m considering the process for applying migrations after updating the code. First, I need to run regression tests and back up the system before restoring it. There’s also an optional second mini-import that I might need to do. But before that, I really should evaluate if I need a new service or if I could just use the existing plan. It definitely seems like migrating the runtime is essential to get everything set up properly!

todo

Import Regression und Position/Data Quality Check aggregiert prüfen

Safe Correction/Void Datenmodell+Service+Tests implementieren

Backup/Restore Regression auf Test-DB prüfen


Optional zweiten Mini-Import genau 1 Item nach Regression durchführen

Compile/pytest/Git-Safety/Git Status, commit+push bei Codeänderungen


terminal

set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python -m jarvis_finance.cli.main migrate
# backup current runtime and restore to separate temp runtime test DB only
PYTHONPATH=src python -m jarvis_finance.cli.main backup-runtime-db > /tmp/b6c_backup_out.txt
python - <<'PY'
import re, subprocess, json, tempfile
from pathlib import Path
text=Path('/tmp/b6c_backup_out.txt').read_text()
m=re.search(r'backup_file=([^\s]+)', text)
if not m:
    raise SystemExit('backup_file not found')
backup=Path(m.group(1)).resolve()
repo=Path('/home/agent/.hermes/repos/FinanceManager')
restore_db=Path('/home/agent/jarvis_runtime/finance-system/restore-tests/b6c-restore-test.sqlite3').resolve()
restore_db.unlink(missing_ok=True)
verify=subprocess.run(['python','-m','jarvis_finance.cli.main','verify-backup','--file',str(backup)], cwd=repo, env={**__import__('os').environ,'PYTHONPATH':'src'}, text=True, capture_output=True, check=True)
from jarvis_finance.runtime.backup import restore_runtime_db
res=restore_runtime_db(backup_file=backup, db_path=restore_db, repo_root=repo, yes=True)
prod_db=Path('/home/agent/jarvis_runtime/finance-system/data/finance.sqlite3').resolve()
print(json.dumps({'backup_created':backup.exists(), 'backup_dir': str(backup.parent), 'verify_checksum_ok': 'checksum_ok=true' in verify.stdout, 'restore_test_db_exists': restore_db.exists(), 'restore_is_separate_from_prod': restore_db != prod_db, 'restore_verified': res.verified}, sort_keys=True))
PY


Output:
SCHEMA_VERSION=8
Traceback (most recent call last):
  File "<stdin>", line 12, in <module>
ModuleNotFoundError: No module named 'jarvis_finance'


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python - <<'PY'
import re, subprocess, json, os
from pathlib import Path
from jarvis_finance.runtime.backup import restore_runtime_db
text=Path('/tmp/b6c_backup_out.txt').read_text()
m=re.search(r'backup_file=([^\s]+)', text)
if not m:
    raise SystemExit('backup_file not found')
backup=Path(m.group(1)).resolve()
repo=Path('/home/agent/.hermes/repos/FinanceManager')
restore_db=Path('/home/agent/jarvis_runtime/finance-system/restore-tests/b6c-restore-test.sqlite3').resolve()
restore_db.unlink(missing_ok=True)
verify=subprocess.run(['python','-m','jarvis_finance.cli.main','verify-backup','--file',str(backup)], cwd=repo, env={**os.environ,'PYTHONPATH':'src'}, text=True, capture_output=True, check=True)
res=restore_runtime_db(backup_file=backup, db_path=restore_db, repo_root=repo, yes=True)
prod_db=Path('/home/agent/jarvis_runtime/finance-system/data/finance.sqlite3').resolve()
print(json.dumps({'backup_created':backup.exists(), 'backup_dir': str(backup.parent), 'verify_checksum_ok': 'checksum_ok=true' in verify.stdout, 'restore_test_db_exists': restore_db.exists(), 'restore_is_separate_from_prod': restore_db != prod_db, 'restore_verified': res.verified}, sort_keys=True))
PY


Output: {"backup_created": true, "backup_dir": "/home/agent/jarvis_runtime/finance-system/backups", "restore_is_separate_from_prod": true, "restore_test_db_exists": true, "restore_verified": true, "verify_checksum_ok": true}


Explanation:
**Preparing for backup**

I'm thinking about needing an optional second mini-import and making sure I have a backup ready. It should be okay, but I want to double-check that the payload was created earlier. I’m considering if the planned_write_summary_json has the correct payload keys. Yes, I’ll need the backup and the import, along with any notes. Let's make sure I’m using the service effectively!


set -euo pipefail
cd /home/agent/.hermes/repos/FinanceManager
PYTHONPATH=src python -m jarvis_finance.cli.main backup-runtime-db > /tmp/b6c_second_backup_out.txt
PYTHONPATH=src python - <<'PY'
import re, subprocess, json, os
from pathlib import Path
text=Path('/tmp/b6c_second_backup_out.txt').read_text(); m=re.search(r'backup_file=([^\s]+)', text)
if not m: raise SystemExit('backup_file not found')
backup=Path(m.group(1)).resolve(); repo=Path('/home/agent/.hermes/repos/FinanceManager')
verify=subprocess.run(['python','-m','jarvis_finance.cli.main','verify-backup','--file',str(backup)], cwd=repo, env={**os.environ,'PYTHONPATH':'src'}, text=True, capture_output=True, check=True)
print(json.dumps({'backup_created':backup.exists(), 'backup_verified':'checksum_ok=true' in verify.stdout, 'backup_dir':str(backup.parent)}, sort_keys=True))
PY
PYTHONPATH=src python - <<'PY'
import json
from jarvis_finance.config.settings import load_settings
from jarvis_finance.storage.database import connect
from jarvis_finance.storage.migrations import apply_migrations
from jarvis_finance.imports.execution import execute_initial_snapshot_plan
conn=connect(load_settings().db_path); apply_migrations(conn)
before_tx=conn.execute("SELECT COUNT(*) n FROM transactions WHERE source_type='broker_import_reviewed_snapshot' AND transaction_type='initial_position_snapshot'").fetchone()['n']
plan=conn.execute("SELECT execution_plan_id FROM broker_import_execution_plans WHERE source_platform='True Wealth' AND payload_status='ready' AND execution_status='planned' ORDER BY created_at, execution_plan_id LIMIT 1").fetchone()
if not plan:
    raise SystemExit('no remaining planned ready payload')
result=execute_initial_snapshot_plan(conn, execution_plan_id=plan['execution_plan_id'], note='B6c second controlled mini-import from reviewed True Wealth execution plan; no mass import.', created_by='phase_b6c')
after_tx=conn.execute("SELECT COUNT(*) n FROM transactions WHERE source_type='broker_import_reviewed_snapshot' AND transaction_type='initial_position_snapshot'").fetchone()['n']
remaining=conn.execute("SELECT COUNT(*) n FROM broker_import_execution_plans WHERE source_platform='True Wealth' AND payload_status='ready' AND execution_status='planned'").fetchone()['n']
print(json.dumps({'second_mini_import_performed':result.imported, 'new_import_transactions_delta': after_tx-before_tx, 'total_tw_initial_snapshots': after_tx, 'remaining_ready_planned_payloads': remaining}, sort_keys=True))
PY


Output:
{"backup_created": true, "backup_dir": "/home/agent/jarvis_runtime/finance-system/backups", "backup_verified": true}
{"new_import_transactions_delta": 1, "remaining_ready_planned_payloads": 1, "second_mini_import_performed": true, "total_tw_initial_snapshots": 2}


Explanation:
**Checking data quality**

I need to do some final checks to ensure everything is in order. I'll definitely consider data quality counts as an important factor. It’s essential that everything is accurately tested before finalizing the process. I want to make sure all aspects have gone through full testing to avoid any blunders later on. So, I'll approach this methodically and carefully — checking off each part as I go along!