     1|from __future__ import annotations
     2|
     3|import hashlib
     4|import json
     5|import re
     6|from collections import defaultdict
     7|from datetime import datetime, timezone
     8|from decimal import Decimal, InvalidOperation
     9|from pathlib import Path
    10|from sqlite3 import Connection
    11|from typing import Any
    12|from urllib.parse import urlparse
    13|
    14|from jarvis_finance.config.settings import find_repo_root, load_settings
    15|from jarvis_finance.services.budget_common import new_id, row_to_dict
    16|
    17|SAFE_RETAILERS = {"Coop", "Aldi Suisse", "Lidl", "Denner", "Otto's", "Migros"}
    18|SAFE_MATCH_TYPES = {"exact_match", "close_match", "cheaper_private_label", "manual_match"}
    19|SAFE_FLAGS = {"exact_match", "close_match", "cheaper_private_label", "manual_match"}
    20|UNCERTAIN_FLAGS = {"different_pack_size", "needs_review", "bio_vs_non_bio", "brand_vs_private_label"}
    21|
    22|
    23|def _now() -> str:
    24|    return datetime.now(timezone.utc).isoformat()
    25|
    26|
    27|def _money(value: Any) -> Decimal:
    28|    try:
    29|        text = str(value or "0").replace("CHF", "").replace("'", "").replace("’", "").strip().replace(",", ".")
    30|        return Decimal(text or "0").quantize(Decimal("0.01"))
    31|    except (InvalidOperation, ValueError):
    32|        return Decimal("0.00")
    33|
    34|
    35|def _fmt(value: Decimal | Any) -> str:
    36|    return f"{_money(value):.2f}"
    37|
    38|
    39|def normalize_product_name(raw_product_name: str) -> dict[str, str]:
    40|    raw = raw_product_name
    41|    text = re.sub(r"\b(aktion|rabatt|bonus|cumulus|hit|promo)\b", " ", raw, flags=re.I)
    42|    quantity = re.search(r"\b\d+(?:[.,]\d+)?\s*(?:kg|g|l|ml|stk|stück|x)\b", text, flags=re.I)
    43|    brand = "M-Budget" if re.search(r"m[- ]?budget", text, flags=re.I) else None
    44|    text = re.sub(r"m[- ]?budget|bio", " ", text, flags=re.I)
    45|    text = re.sub(r"\b\d+(?:[.,]\d+)?\s*(?:kg|g|l|ml|stk|stück|x)\b", " ", text, flags=re.I)
    46|    words = [w.lower() for w in re.findall(r"[A-Za-zÄÖÜäöüß]+", text) if len(w) > 1]
    47|    core_map = {"milch": "milch", "eier": "eier", "ei": "eier", "reis": "reis", "poulet": "poulet", "joghurt": "joghurt", "yogurt": "joghurt", "käse": "käse", "kaese": "käse", "waschmittel": "waschmittel", "katzenfutter": "katzenfutter"}
    48|    core = next((core_map[w] for w in words if w in core_map), " ".join(words[:3]).strip() or raw.strip().lower())
    49|    return {"raw_product_name": raw, "normalized_product_name": core, "brand_hint": brand or "", "quantity_hint": quantity.group(0).replace(" ", "") if quantity else "", "confidence": "0.82" if core else "0.50"}
    50|
    51|
    52|def add_demo_migros_receipt(conn: Connection, *, purchase_date: str, store_name: str, items: list[dict[str, Any]], receipt_id: str | None = None) -> dict[str, Any]:
    53|    rid = receipt_id or new_id("mreceipt")
    54|    out_items: list[dict[str, Any]] = []
    55|    for idx, item in enumerate(items, start=1):
    56|        norm = normalize_product_name(str(item.get("raw_product_name") or ""))
    57|        pid = new_id("gitem")
    58|        total = _fmt(item.get("total_price_text") or item.get("total") or "0")
    59|        conn.execute(
    60|            """INSERT INTO grocery_product_items(product_item_id,source_type,receipt_id,receipt_key,source_line_id,purchase_date,store_name,raw_product_name,normalized_product_name,normalization_confidence,brand_hint,quantity_hint,quantity_text,unit,unit_price_text,total_price_text,currency,action_label,include_in_analysis,notes,health_analysis_status,health_flags_json,health_notes,created_at)
    61|               VALUES (?, 'migros_receipt', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'CHF', ?, 1, ?, 'prepared_not_run', '{}', ?, ?)""",
    62|            (pid, rid, f"{purchase_date}:{store_name}", str(idx), purchase_date, store_name, item.get("raw_product_name"), norm["normalized_product_name"], norm["confidence"], norm["brand_hint"], norm["quantity_hint"], item.get("quantity_text"), item.get("unit"), item.get("unit_price_text"), total, item.get("action_label"), item.get("notes"), "Gesundheitsanalyse später", _now()),
    63|        )
    64|        row = row_to_dict(conn.execute("SELECT * FROM grocery_product_items WHERE product_item_id=?", (pid,)).fetchone())
    65|        out_items.append(row)
    66|    conn.commit()
    67|    return {"receipt_id": rid, "purchase_date": purchase_date, "store_name": store_name, "items": out_items}
    68|
    69|
    70|def _items_for_receipt(conn: Connection, receipt_id: str) -> list[dict[str, Any]]:
    71|    rows = conn.execute("SELECT * FROM grocery_product_items WHERE receipt_id=? ORDER BY source_line_id, product_item_id", (receipt_id,)).fetchall()
    72|    return [row_to_dict(r) for r in rows]
    73|
    74|
    75|def get_grocery_optimizer_dashboard(conn: Connection) -> dict[str, Any]:
    76|    rows = conn.execute("""SELECT receipt_id, purchase_date, store_name, COUNT(*) AS article_count, COALESCE(SUM(CAST(total_price_text AS REAL)),0) AS total_value FROM grocery_product_items GROUP BY receipt_id,purchase_date,store_name ORDER BY purchase_date DESC, receipt_id DESC""").fetchall()
    77|    receipts = []
    78|    for r in rows:
    79|        items = _items_for_receipt(conn, r["receipt_id"])
    80|        for item in items:
    81|            item["known_mappings"] = known_mappings_for_item(conn, item, include_rejected=True)
    82|        receipts.append({"receipt_id": r["receipt_id"], "purchase_date": r["purchase_date"], "store_name": r["store_name"], "article_count": int(r["article_count"] or 0), "total_text": _fmt(r["total_value"]), "items": items})
    83|    return {"purpose": "migros_product_savings_grocery_optimizer_v1", "receipts": receipts, "supported_retailers": sorted(SAFE_RETAILERS - {"Migros"}), "health_analysis_status": "prepared_not_run", "disclaimer": "Keine medizinische Bewertung; keine automatische Bestellung."}
    84|
    85|
    86|def _validate_product_url(url: str) -> None:
    87|    parsed = urlparse(url)
    88|    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
    89|        raise ValueError("invalid_product_url")
    90|    host = parsed.hostname or ""
    91|    if host in {"localhost", "127.0.0.1", "0.0.0.0"} or host.startswith("10.") or host.startswith("192.168.") or host.startswith("172.16."):
    92|        raise ValueError("unsafe_product_url")
    93|
    94|
    95|def upsert_product_match(conn: Connection, product_item_id: str, *, retailer: str, candidate_product_name: str, candidate_url: str, candidate_price_text: str | None, candidate_unit_price_text: str | None = None, quality_flags: list[str] | None = None, match_confidence: str = "0.60", status: str = "suggested", source: str = "manual", candidate_brand: str | None = None, candidate_package_size: str | None = None, candidate_unit: str | None = None) -> dict[str, Any]:
    96|    if retailer not in SAFE_RETAILERS:
    97|        raise ValueError("unsupported_retailer")
    98|    _validate_product_url(candidate_url)
    99|    mid = new_id("gmatch")
   100|    flags = quality_flags or ["needs_review"]
   101|    fetched = _now()
   102|    source_hash = hashlib.sha256((retailer + candidate_url + candidate_product_name + str(candidate_price_text)).encode()).hexdigest()
   103|    conn.execute(
   104|        """INSERT INTO grocery_product_matches(match_id,product_item_id,retailer,candidate_product_name,candidate_url,candidate_brand,candidate_package_size,candidate_unit,candidate_price_text,candidate_unit_price_text,price_currency,match_confidence,match_reason,quality_flags_json,fetched_at,source,status)
   105|           VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'CHF', ?, ?, ?, ?, ?, ?)""",
   106|        (mid, product_item_id, retailer, candidate_product_name, candidate_url, candidate_brand, candidate_package_size, candidate_unit, _fmt(candidate_price_text) if candidate_price_text is not None else None, candidate_unit_price_text, match_confidence, "reviewable sourced candidate", json.dumps(flags), fetched, source, status),
   107|    )
   108|    if source not in {"web_fetch", "cache"}:
   109|        conn.execute(
   110|            """INSERT OR REPLACE INTO grocery_product_details_cache(detail_id,retailer,product_url,product_name,price_text,unit_price_text,package_size,ingredients_text,nutrition_json,fetched_at,cache_status,source_hash)
   111|               VALUES (COALESCE((SELECT detail_id FROM grocery_product_details_cache WHERE retailer=? AND product_url=?), ?), ?, ?, ?, ?, ?, ?, NULL, '{}', ?, 'cached', ?)""",
   112|            (retailer, candidate_url, new_id("gdetail"), retailer, candidate_url, candidate_product_name, _fmt(candidate_price_text) if candidate_price_text is not None else None, candidate_unit_price_text, candidate_package_size, fetched, source_hash),
   113|        )
   114|    conn.commit()
   115|    return row_to_dict(conn.execute("SELECT * FROM grocery_product_matches WHERE match_id=?", (mid,)).fetchone())
   116|
   117|
   118|def _matches(conn: Connection, product_item_id: str) -> list[dict[str, Any]]:
   119|    return [row_to_dict(r) for r in conn.execute("SELECT * FROM grocery_product_matches WHERE product_item_id=? ORDER BY CAST(match_confidence AS REAL) DESC", (product_item_id,)).fetchall()]
   120|
   121|
   122|def _cache_status_for_url(conn: Connection, retailer: str, url: str) -> dict[str, Any]:
   123|    row = conn.execute("SELECT * FROM grocery_product_details_cache WHERE retailer=? AND product_url=? ORDER BY fetched_at DESC LIMIT 1", (retailer, url)).fetchone()
   124|    if not row:
   125|        return {"cache_status": "fehlt", "last_price_checked_at": None}
   126|    data = row_to_dict(row)
   127|    try:
   128|        fetched = datetime.fromisoformat(str(data.get("fetched_at")))
   129|        if fetched.tzinfo is None:
   130|            fetched = fetched.replace(tzinfo=timezone.utc)
   131|        age = (datetime.now(timezone.utc) - fetched).total_seconds()
   132|        status = "frisch" if age <= 86400 else "veraltet"
   133|    except Exception:
   134|        status = "fehler"
   135|    return {"cache_status": status, "last_price_checked_at": data.get("fetched_at"), "price_text": data.get("price_text"), "unit_price_text": data.get("unit_price_text")}
   136|
   137|
   138|def known_mappings_for_item(conn: Connection, item: dict[str, Any], include_rejected: bool = False) -> list[dict[str, Any]]:
   139|    statuses = ("accepted", "needs_review", "rejected") if include_rejected else ("accepted", "needs_review")
   140|    rows = conn.execute(
   141|        f"SELECT * FROM grocery_product_mappings WHERE source_retailer='Migros' AND source_product_normalized_name=? AND status IN ({','.join('?' for _ in statuses)}) ORDER BY status='accepted' DESC, CAST(confidence AS REAL) DESC, updated_at DESC",
   142|        (item.get("normalized_product_name"), *statuses),
   143|    ).fetchall()
   144|    mappings = []
   145|    for row in rows:
   146|        d = row_to_dict(row)
   147|        d.update(_cache_status_for_url(conn, d["target_retailer"], d["target_product_url"]))
   148|        mappings.append(d)
   149|    return mappings
   150|
   151|
   152|def _insert_mapping(conn: Connection, *, item: dict[str, Any], target_retailer: str, target_product_name: str, target_product_url: str, match_type: str, status: str, confidence: str = "0.80", user_note: str | None = None, source_match_id: str | None = None, target_brand: str | None = None, target_package_size: str | None = None, target_unit: str | None = None, target_price_text: str | None = None, target_unit_price_text: str | None = None, quality_flags: list[str] | None = None, last_price_checked_at: str | None = None) -> dict[str, Any]:
   153|    if target_retailer not in SAFE_RETAILERS:
   154|        raise ValueError("unsupported_retailer")
   155|    if match_type not in SAFE_MATCH_TYPES:
   156|        raise ValueError("unsupported_match_type")
   157|    _validate_product_url(target_product_url)
   158|    now = _now()
   159|    mapping_id = new_id("gmap")
   160|    conn.execute(
   161|        """INSERT INTO grocery_product_mappings(mapping_id,source_product_normalized_name,source_product_raw_name,source_retailer,target_retailer,target_product_name,target_product_url,target_brand,target_package_size,target_unit,target_price_text,target_unit_price_text,price_currency,match_type,status,confidence,user_note,source_match_id,quality_flags_json,health_analysis_status,health_flags_json,health_notes,created_at,updated_at,last_price_checked_at)
   162|           VALUES (?, ?, ?, 'Migros', ?, ?, ?, ?, ?, ?, ?, ?, 'CHF', ?, ?, ?, ?, ?, ?, 'prepared_not_run', '{}', ?, ?, ?, ?)""",
   163|        (mapping_id, item.get("normalized_product_name"), item.get("raw_product_name"), target_retailer, target_product_name, target_product_url, target_brand, target_package_size, target_unit, _fmt(target_price_text) if target_price_text else None, target_unit_price_text, match_type, status, confidence, user_note, source_match_id, json.dumps(quality_flags or [match_type]), "Gesundheitsanalyse später", now, now, last_price_checked_at),
   164|    )
   165|    conn.commit()
   166|    return row_to_dict(conn.execute("SELECT * FROM grocery_product_mappings WHERE mapping_id=?", (mapping_id,)).fetchone())
   167|
   168|
   169|def accept_grocery_product_match(conn: Connection, match_id: str, *, user_note: str | None = None, save_as_preferred: bool = True) -> dict[str, Any]:
   170|    row = conn.execute("SELECT * FROM grocery_product_matches WHERE match_id=?", (match_id,)).fetchone()
   171|    if not row:
   172|        raise ValueError("match_not_found")
   173|    match = row_to_dict(row)
   174|    item = row_to_dict(conn.execute("SELECT * FROM grocery_product_items WHERE product_item_id=?", (match["product_item_id"],)).fetchone())
   175|    conn.execute("UPDATE grocery_product_matches SET status='accepted' WHERE match_id=?", (match_id,))
   176|    mapping = _insert_mapping(conn, item=item, target_retailer=match["retailer"], target_product_name=match["candidate_product_name"], target_product_url=match["candidate_url"], target_brand=match.get("candidate_brand"), target_package_size=match.get("candidate_package_size"), target_unit=match.get("candidate_unit"), target_price_text=match.get("candidate_price_text"), target_unit_price_text=match.get("candidate_unit_price_text"), match_type=(json.loads(match.get("quality_flags_json") or "[]") or ["manual_match"])[0], status="accepted" if save_as_preferred else "needs_review", confidence=match.get("match_confidence") or "0.80", user_note=user_note, source_match_id=match_id, quality_flags=json.loads(match.get("quality_flags_json") or "[]"), last_price_checked_at=match.get("fetched_at"))
   177|    return mapping
   178|
   179|
   180|def reject_grocery_product_match(conn: Connection, match_id: str, *, user_note: str | None = None) -> dict[str, Any]:
   181|    row = conn.execute("SELECT * FROM grocery_product_matches WHERE match_id=?", (match_id,)).fetchone()
   182|    if not row:
   183|        raise ValueError("match_not_found")
   184|    match = row_to_dict(row)
   185|    item = row_to_dict(conn.execute("SELECT * FROM grocery_product_items WHERE product_item_id=?", (match["product_item_id"],)).fetchone())
   186|    conn.execute("UPDATE grocery_product_matches SET status='rejected' WHERE match_id=?", (match_id,))
   187|    mapping = _insert_mapping(conn, item=item, target_retailer=match["retailer"], target_product_name=match["candidate_product_name"], target_product_url=match["candidate_url"], target_brand=match.get("candidate_brand"), target_package_size=match.get("candidate_package_size"), target_unit=match.get("candidate_unit"), target_price_text=match.get("candidate_price_text"), target_unit_price_text=match.get("candidate_unit_price_text"), match_type="manual_match", status="rejected", confidence=match.get("match_confidence") or "0.50", user_note=user_note, source_match_id=match_id, quality_flags=json.loads(match.get("quality_flags_json") or "[]"), last_price_checked_at=match.get("fetched_at"))
   188|    return mapping
   189|
   190|
   191|def create_manual_grocery_mapping(conn: Connection, *, product_item_id: str, target_retailer: str, target_product_name: str, target_product_url: str, match_type: str = "manual_match", candidate_price_text: str | None = None, candidate_unit_price_text: str | None = None, target_brand: str | None = None, target_package_size: str | None = None, target_unit: str | None = None, user_note: str | None = None, confidence: str = "0.95") -> dict[str, Any]:
   192|    item = row_to_dict(conn.execute("SELECT * FROM grocery_product_items WHERE product_item_id=?", (product_item_id,)).fetchone())
   193|    mapping = _insert_mapping(conn, item=item, target_retailer=target_retailer, target_product_name=target_product_name, target_product_url=target_product_url, target_brand=target_brand, target_package_size=target_package_size, target_unit=target_unit, target_price_text=candidate_price_text, target_unit_price_text=candidate_unit_price_text, match_type=match_type, status="accepted", confidence=confidence, user_note=user_note, quality_flags=[match_type], last_price_checked_at=_now() if candidate_price_text else None)
   194|    if candidate_price_text:
   195|        upsert_product_match(conn, product_item_id, retailer=target_retailer, candidate_product_name=target_product_name, candidate_url=target_product_url, candidate_price_text=candidate_price_text, candidate_unit_price_text=candidate_unit_price_text, quality_flags=[match_type], match_confidence=confidence, status="accepted", source="manual_mapping", candidate_brand=target_brand, candidate_package_size=target_package_size, candidate_unit=target_unit)
   196|    return mapping
   197|
   198|
   199|def refresh_grocery_mapping_price(conn: Connection, mapping_id: str) -> dict[str, Any]:
   200|    row = conn.execute("SELECT * FROM grocery_product_mappings WHERE mapping_id=?", (mapping_id,)).fetchone()
   201|    if not row:
   202|        raise ValueError("mapping_not_found")
   203|    mapping = row_to_dict(row)
   204|    now = _now()
   205|    conn.execute("UPDATE grocery_product_mappings SET last_price_checked_at=?, updated_at=? WHERE mapping_id=?", (now, now, mapping_id))
   206|    conn.commit()
   207|    mapping.update({"last_price_checked_at": now, "cache_status": _cache_status_for_url(conn, mapping["target_retailer"], mapping["target_product_url"])["cache_status"]})
   208|    return mapping
   209|
   210|
   211|def _safe_match(match: dict[str, Any]) -> bool:
   212|    flags = set(json.loads(match.get("quality_flags_json") or "[]"))
   213|    return match.get("status") in {"suggested", "accepted"} and flags <= SAFE_FLAGS and bool(flags & SAFE_FLAGS)
   214|
   215|
   216|def run_grocery_optimization(conn: Connection, receipt_id: str, *, selected_retailers: list[str], max_store_count: int = 3, included_product_item_ids: list[str] | None = None) -> dict[str, Any]:
   217|    max_store_count = min(max(1, int(max_store_count or 3)), 3)
   218|    selected = [r for r in selected_retailers if r in SAFE_RETAILERS]
   219|    items = _items_for_receipt(conn, receipt_id)
   220|    if included_product_item_ids is not None:
   221|        include = set(included_product_item_ids)
   222|        items = [i for i in items if i["product_item_id"] in include]
   223|    else:
   224|        items = [i for i in items if int(i.get("include_in_analysis") or 0) == 1]
   225|    original = sum((_money(i["total_price_text"]) for i in items), Decimal("0.00"))
   226|
   227|    # First pass: determine each item's best safe alternative and potential saving.
   228|    item_options: list[dict[str, Any]] = []
   229|    retailer_savings: dict[str, Decimal] = defaultdict(lambda: Decimal("0.00"))
   230|    uncertain = 0
   231|    for item in items:
   232|        item_price = _money(item["total_price_text"])
   233|        candidates = [m for m in _matches(conn, item["product_item_id"]) if m["retailer"] in selected]
   234|        for mapping in known_mappings_for_item(conn, item, include_rejected=False):
   235|            if mapping["status"] == "accepted" and mapping["target_retailer"] in selected and mapping.get("target_price_text"):
   236|                candidates.append({"match_id": mapping["mapping_id"], "product_item_id": item["product_item_id"], "retailer": mapping["target_retailer"], "candidate_product_name": mapping["target_product_name"], "candidate_url": mapping["target_product_url"], "candidate_brand": mapping.get("target_brand"), "candidate_package_size": mapping.get("target_package_size"), "candidate_unit": mapping.get("target_unit"), "candidate_price_text": mapping.get("target_price_text"), "candidate_unit_price_text": mapping.get("target_unit_price_text"), "match_confidence": mapping.get("confidence") or "0.95", "quality_flags_json": mapping.get("quality_flags_json") or json.dumps([mapping.get("match_type") or "manual_match"]), "status": "accepted", "source": "known_mapping"})
   237|        rejected_mappings = [m for m in conn.execute("SELECT * FROM grocery_product_mappings WHERE source_product_normalized_name=? AND status='rejected'", (item.get("normalized_product_name"),)).fetchall()]
   238|        safe = [m for m in candidates if _safe_match(m)]
   239|        best = min(safe, key=lambda m: _money(m["candidate_price_text"]), default=None)
   240|        if best:
   241|            retailer_savings[best["retailer"]] += max(Decimal("0.00"), item_price - _money(best["candidate_price_text"]))
   242|        elif candidates:
   243|            uncertain += 1
   244|        item_options.append({"item": item, "item_price": item_price, "candidates": candidates, "safe": safe, "best": best, "has_rejected": bool(rejected_mappings)})
   245|
   246|    allowed_retailers = set(r for r, _saving in sorted(retailer_savings.items(), key=lambda kv: kv[1], reverse=True)[:max_store_count])
   247|    comparisons = []
   248|    by_store: dict[str, list[dict[str, Any]]] = defaultdict(list)
   249|    optimized = Decimal("0.00")
   250|    replaceable = 0
   251|    for option in item_options:
   252|        item = option["item"]; item_price = option["item_price"]
   253|        safe_in_allowed = [m for m in option["safe"] if m["retailer"] in allowed_retailers]
   254|        best = min(safe_in_allowed, key=lambda m: _money(m["candidate_price_text"]), default=None)
   255|        original_best = option["best"]
   256|        status = "not_found"
   257|        savings = Decimal("0.00")
   258|        flags: list[str] = []
   259|        if best:
   260|            alt = _money(best["candidate_price_text"])
   261|            savings = max(Decimal("0.00"), item_price - alt)
   262|            optimized += alt
   263|            replaceable += 1
   264|            status = "suggested"
   265|            flags = json.loads(best.get("quality_flags_json") or "[]")
   266|            by_store[best["retailer"]].append({"product": item["raw_product_name"], "candidate_product_name": best["candidate_product_name"], "price_text": _fmt(alt), "source_url": best["candidate_url"]})
   267|        else:
   268|            optimized += item_price
   269|            if original_best and original_best.get("retailer") not in allowed_retailers:
   270|                status = "not_selected_due_to_store_limit"
   271|                flags = json.loads(original_best.get("quality_flags_json") or "[]")
   272|            elif option.get("has_rejected"):
   273|                status = "rejected"
   274|            elif option["candidates"]:
   275|                status = "needs_review"
   276|                flags = json.loads(option["candidates"][0].get("quality_flags_json") or "[]")
   277|        comparisons.append({"product_item_id": item["product_item_id"], "migros_product": item["raw_product_name"], "migros_price_text": _fmt(item_price), "best_alternative": best["candidate_product_name"] if best else None, "retailer": best["retailer"] if best else None, "alternative_price_text": _fmt(best["candidate_price_text"]) if best else None, "unit_price_text": best.get("candidate_unit_price_text") if best else None, "savings_text": _fmt(savings), "confidence": best.get("match_confidence") if best else "0", "status": status, "quality_flags": flags, "source_url": best.get("candidate_url") if best else None})
   278|    stores = sorted(by_store.items(), key=lambda kv: len(kv[1]), reverse=True)
   279|    optimized_lists = [{"variant": "max_3_stores", "store_count": len(stores), "stores": [{"retailer": retailer, "items": rows} for retailer, rows in stores], "not_found_products": [c["migros_product"] for c in comparisons if c["status"] == "not_found"], "needs_review_products": [c["migros_product"] for c in comparisons if c["status"] == "needs_review"], "rejected_products": [c["migros_product"] for c in comparisons if c["status"] == "rejected"], "store_limit_omitted_products": [c["migros_product"] for c in comparisons if c["status"] == "not_selected_due_to_store_limit"]}]
   280|    savings_total = max(Decimal("0.00"), original - optimized)
   281|

Use offset=281 to continue reading (showing 1-280 of 322 lines)


   280|    if original_unit_price_text and candidate_unit_price_text and _decimal_text(original_unit_price_text) and _decimal_text(candidate_unit_price_text):
   281|        raw_savings = _money(original_unit_price_text) - _money(candidate_unit_price_text)
   282|        if raw_savings <= Decimal("0.00"):
   283|            if "not_cheaper" not in flags:
   284|                flags.append("not_cheaper")
   285|            return {"can_calculate_savings": False, "savings_text": "0.00", "basis": "unit_price", "quality_flags": flags}
   286|        return {"can_calculate_savings": True, "savings_text": f"{raw_savings:.2f}", "basis": "unit_price", "quality_flags": flags}
   287|    if not any(f in flags for f in ["different_pack_size", "different_quality", "organic_mismatch"]):
   288|        raw_savings = _money(original_price_text) - _money(candidate_price_text)
   289|        if raw_savings <= Decimal("0.00"):
   290|            if "not_cheaper" not in flags:
   291|                flags.append("not_cheaper")
   292|            return {"can_calculate_savings": False, "savings_text": "0.00", "basis": "package_price", "quality_flags": flags}
   293|        if "package_price_fallback" not in flags:
   294|            flags.append("package_price_fallback")
   295|        return {"can_calculate_savings": True, "savings_text": f"{raw_savings:.2f}", "basis": "package_price", "quality_flags": flags}
   296|    return {"can_calculate_savings": False, "savings_text": "0.00", "basis": "none", "quality_flags": flags}
   297|
   298|
   299|def search_and_store_product_matches(conn: Connection, receipt_id: str, *, providers: list[GroceryProductProvider], included_product_item_ids: list[str] | None = None, max_products: int = 5, use_cache: bool = True, only_safe_matches: bool = True, prefer_known_mappings: bool = True) -> dict[str, Any]:
   300|    items = _items_for_receipt(conn, receipt_id)
   301|    if included_product_item_ids is not None:
   302|        allowed = set(included_product_item_ids)
   303|        items = [item for item in items if item["product_item_id"] in allowed]
   304|    items = items[:max_products]
   305|    matches: list[dict[str, Any]] = []
   306|    provider_calls = 0
   307|    cache_hits = 0
   308|    known_mapping_hits = 0
   309|    for item in items:
   310|        query = item.get("normalized_product_name") or item.get("raw_product_name")
   311|        if prefer_known_mappings:
   312|            accepted = [m for m in known_mappings_for_item(conn, item, include_rejected=False) if m.get("status") == "accepted"]
   313|            if accepted:
   314|                for m in accepted:
   315|                    matches.append({"mapping_id": m["mapping_id"], "product_item_id": item["product_item_id"], "retailer": m["target_retailer"], "product_name": m["target_product_name"], "product_url": m["target_product_url"], "price_text": m.get("target_price_text"), "unit_price_text": m.get("target_unit_price_text"), "source_fetched_at": m.get("last_price_checked_at"), "fetched_at": m.get("updated_at"), "quality_flags": json.loads(m.get("quality_flags_json") or "[]"), "confidence": m.get("confidence"), "status": "suggested" if m.get("target_price_text") else "needs_review", "source": "known_mapping", "cache_status": m.get("cache_status")})
   316|                    known_mapping_hits += 1
   317|                continue
   318|            rejected = conn.execute("SELECT 1 FROM grocery_product_mappings WHERE source_product_normalized_name=? AND status='rejected' LIMIT 1", (item.get("normalized_product_name"),)).fetchone()
   319|            if rejected:
   320|                matches.append({"product_item_id": item["product_item_id"], "retailer": None, "product_name": item.get("raw_product_name"), "status": "rejected", "quality_flags": ["rejected"], "source": "known_mapping"})
   321|                known_mapping_hits += 1
   322|                continue
   323|        for provider in providers:
   324|            before = provider.request_count
   325|            results = provider.search_products(conn, query, use_cache=use_cache, max_results=3)
   326|            provider_calls += max(0, provider.request_count - before)
   327|            for result in results:
   328|                if result.get("source") == "cache":
   329|                    cache_hits += 1
   330|                if not result.get("product_url") or not result.get("product_name"):
   331|                    matches.append({"product_item_id": item["product_item_id"], **result})
   332|                    continue
   333|                flags = quality_flags_for_match(item.get("normalized_product_name") or item.get("raw_product_name"), result)
   334|                comparison = compare_product_prices(original_price_text=str(item.get("total_price_text") or "0"), original_unit_price_text=item.get("unit_price_text"), candidate_price_text=result.get("price_decimal_text") or result.get("price_text"), candidate_unit_price_text=result.get("unit_price_decimal_text") or result.get("unit_price_text"), currency=result.get("currency") or "CHF", quality_flags=flags)
   335|                status = "suggested" if comparison["can_calculate_savings"] and (not only_safe_matches or not (set(flags) & UNSAFE_FLAGS)) else "needs_review"
   336|                if not comparison["can_calculate_savings"] or not result.get("price_decimal_text"):
   337|                    matches.append({"product_item_id": item["product_item_id"], "retailer": result.get("retailer"), "product_name": result.get("product_name"), "product_url": result.get("product_url"), "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": result.get("fetched_at"), "quality_flags": comparison["quality_flags"], "confidence": result.get("confidence"), "status": status, "source": result.get("source")})
   338|                    continue
   339|                stored = upsert_product_match(conn, item["product_item_id"], retailer=result["retailer"], candidate_product_name=result["product_name"], candidate_url=result["product_url"], candidate_price_text=result.get("price_decimal_text") or result.get("price_text") or "0", candidate_unit_price_text=result.get("unit_price_text"), quality_flags=comparison["quality_flags"], match_confidence=result.get("confidence") or "0.60", status=status, source=result.get("source") or "web_fetch", candidate_brand=result.get("brand"), candidate_package_size=result.get("package_size"), candidate_unit=result.get("unit"))
   340|                matches.append({"match_id": stored.get("match_id"), "product_item_id": item["product_item_id"], "retailer": result["retailer"], "product_name": result["product_name"], "product_url": result["product_url"], "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": stored.get("fetched_at"), "quality_flags": comparison["quality_flags"], "confidence": result.get("confidence"), "status": status, "source": result.get("source")})
   341|    return {"receipt_id": receipt_id, "provider_calls": provider_calls, "cache_hits": cache_hits, "known_mapping_hits": known_mapping_hits, "matches": matches}
   342|


Ich schließe die Review-Blocker jetzt konkret: Refresh wird nicht mehr künstlich als Preischeck markiert, manuelle Preise gelten nicht als provider-geprüft, unsichere/no-price Kandidaten bekommen `match_id`, und Rejections blockieren nur exakt denselben Zieltreffer statt den ganzen Produkttyp.

patch

    mapping = _insert_mapping(conn, item=item, target_retailer=target_retailer, target_product_name=target_product_name, target_product_url=target_product_url, target_brand=target_brand, target_package_size=target_package_size, target_unit=target_unit, target_price_text=candidate_price_text, target_unit_price_text=candidate_unit_price_text, match_type=match_type, status="accepted", confidence=confidence, user_note=user_note, quality_flags=[match_type], last_price_checked_at=_now() if candidate_price_text else None)
    if candidate_price_text:
        upsert_product_match(conn, product_item_id, retailer=target_retailer, candidate_product_name=target_product_name, candidate_url=target_product_url, candidate_price_text=candidate_price_text, candidate_unit_price_text=candidate_unit_price_text, quality_flags=[match_type], match_confidence=confidence, status="accepted", source="manual_mapping", candidate_brand=target_brand, candidate_package_size=target_package_size, candidate_unit=target_unit)
    return mapping

    mapping = _insert_mapping(conn, item=item, target_retailer=target_retailer, target_product_name=target_product_name, target_product_url=target_product_url, target_brand=target_brand, target_package_size=target_package_size, target_unit=target_unit, target_price_text=candidate_price_text, target_unit_price_text=candidate_unit_price_text, match_type=match_type, status="accepted", confidence=confidence, user_note=user_note, quality_flags=[match_type, "manual_price" if candidate_price_text else "no_price"], last_price_checked_at=None)
    return mapping


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
@@ -190,9 +190,7 @@

 def create_manual_grocery_mapping(conn: Connection, *, product_item_id: str, target_retailer: str, target_product_name: str, target_product_url: str, match_type: str = "manual_match", candidate_price_text: str | None = None, candidate_unit_price_text: str | None = None, target_brand: str | None = None, target_package_size: str | None = None, target_unit: str | None = None, user_note: str | None = None, confidence: str = "0.95") -> dict[str, Any]:
     item = row_to_dict(conn.execute("SELECT * FROM grocery_product_items WHERE product_item_id=?", (product_item_id,)).fetchone())
-    mapping = _insert_mapping(conn, item=item, target_retailer=target_retailer, target_product_name=target_product_name, target_product_url=target_product_url, target_brand=target_brand, target_package_size=target_package_size, target_unit=target_unit, target_price_text=candidate_price_text, target_unit_price_text=candidate_unit_price_text, match_type=match_type, status="accepted", confidence=confidence, user_note=user_note, quality_flags=[match_type], last_price_checked_at=_now() if candidate_price_text else None)
-    if candidate_price_text:
-        upsert_product_match(conn, product_item_id, retailer=target_retailer, candidate_product_name=target_product_name, candidate_url=target_product_url, candidate_price_text=candidate_price_text, candidate_unit_price_text=candidate_unit_price_text, quality_flags=[match_type], match_confidence=confidence, status="accepted", source="manual_mapping", candidate_brand=target_brand, candidate_package_size=target_package_size, candidate_unit=target_unit)
+    mapping = _insert_mapping(conn, item=item, target_retailer=target_retailer, target_product_name=target_product_name, target_product_url=target_product_url, target_brand=target_brand, target_package_size=target_package_size, target_unit=target_unit, target_price_text=candidate_price_text, target_unit_price_text=candidate_unit_price_text, match_type=match_type, status="accepted", confidence=confidence, user_note=user_note, quality_flags=[match_type, "manual_price" if candidate_price_text else "no_price"], last_price_checked_at=None)
     return mapping

/home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py was last read with offset/limit pagination (partial view). Re-read the whole file before overwriting it.


def refresh_grocery_mapping_price(conn: Connection, mapping_id: str) -> dict[str, Any]:
    row = conn.execute("SELECT * FROM grocery_product_mappings WHERE mapping_id=?", (mapping_id,)).fetchone()
    if not row:
        raise ValueError("mapping_not_found")
    mapping = row_to_dict(row)
    cache = _cache_status_for_url(conn, mapping["target_retailer"], mapping["target_product_url"])
    if cache.get("price_text"):
        conn.execute(
            """UPDATE grocery_product_mappings
               SET target_price_text=?, target_unit_price_text=?, last_price_checked_at=?, updated_at=?
               WHERE mapping_id=?""",
            (cache.get("price_text"), cache.get("unit_price_text"), cache.get("last_price_checked_at"), _now(), mapping_id),
        )
        conn.commit()
        mapping.update({"target_price_text": cache.get("price_text"), "target_unit_price_text": cache.get("unit_price_text"), "last_price_checked_at": cache.get("last_price_checked_at")})
    mapping.update({"cache_status": cache["cache_status"], "price_refresh_result": "cache_price_applied" if cache.get("price_text") else "no_sourced_price_available"})
    return mapping

def refresh_grocery_mapping_price(conn: Connection, mapping_id: str) -> dict[str, Any]:
    row = conn.execute("SELECT * FROM grocery_product_mappings WHERE mapping_id=?", (mapping_id,)).fetchone()
    if not row:
        raise ValueError("mapping_not_found")
    mapping = row_to_dict(row)
    now = _now()
    conn.execute("UPDATE grocery_product_mappings SET last_price_checked_at=?, updated_at=? WHERE mapping_id=?", (now, now, mapping_id))
    conn.commit()
    mapping.update({"last_price_checked_at": now, "cache_status": _cache_status_for_url(conn, mapping["target_retailer"], mapping["target_product_url"])["cache_status"]})
    return mapping


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
@@ -199,10 +199,17 @@
     if not row:
         raise ValueError("mapping_not_found")
     mapping = row_to_dict(row)
-    now = _now()
-    conn.execute("UPDATE grocery_product_mappings SET last_price_checked_at=?, updated_at=? WHERE mapping_id=?", (now, now, mapping_id))
-    conn.commit()
-    mapping.update({"last_price_checked_at": now, "cache_status": _cache_status_for_url(conn, mapping["target_retailer"], mapping["target_product_url"])["cache_status"]})
+    cache = _cache_status_for_url(conn, mapping["target_retailer"], mapping["target_product_url"])
+    if cache.get("price_text"):
+        conn.execute(
+            """UPDATE grocery_product_mappings
+               SET target_price_text=?, target_unit_price_text=?, last_price_checked_at=?, updated_at=?
+               WHERE mapping_id=?""",
+            (cache.get("price_text"), cache.get("unit_price_text"), cache.get("last_price_checked_at"), _now(), mapping_id),
+        )
+        conn.commit()
+        mapping.update({"target_price_text": cache.get("price_text"), "target_unit_price_text": cache.get("unit_price_text"), "last_price_checked_at": cache.get("last_price_checked_at")})
+    mapping.update({"cache_status": cache["cache_status"], "price_refresh_result": "cache_price_applied" if cache.get("price_text") else "no_sourced_price_available"})
     return mapping


def _safe_match(match: dict[str, Any]) -> bool:
    flags = set(json.loads(match.get("quality_flags_json") or "[]"))
    if "manual_price" in flags or "no_price" in flags:
        return False
    return match.get("status") in {"suggested", "accepted"} and flags <= SAFE_FLAGS and bool(flags & SAFE_FLAGS) and bool(match.get("candidate_price_text"))

def _safe_match(match: dict[str, Any]) -> bool:
    flags = set(json.loads(match.get("quality_flags_json") or "[]"))
    return match.get("status") in {"suggested", "accepted"} and flags <= SAFE_FLAGS and bool(flags & SAFE_FLAGS)


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
@@ -215,7 +215,9 @@

 def _safe_match(match: dict[str, Any]) -> bool:
     flags = set(json.loads(match.get("quality_flags_json") or "[]"))
-    return match.get("status") in {"suggested", "accepted"} and flags <= SAFE_FLAGS and bool(flags & SAFE_FLAGS)
+    if "manual_price" in flags or "no_price" in flags:
+        return False
+    return match.get("status") in {"suggested", "accepted"} and flags <= SAFE_FLAGS and bool(flags & SAFE_FLAGS) and bool(match.get("candidate_price_text"))

 def run_grocery_optimization(conn: Connection, receipt_id: str, *, selected_retailers: list[str], max_store_count: int = 3, included_product_item_ids: list[str] | None = None) -> dict[str, Any]:


        for mapping in known_mappings_for_item(conn, item, include_rejected=False):
            mapping_flags = set(json.loads(mapping.get("quality_flags_json") or "[]"))
            if mapping["status"] == "accepted" and mapping["target_retailer"] in selected and mapping.get("target_price_text") and mapping.get("last_price_checked_at") and "manual_price" not in mapping_flags:
                candidates.append({"match_id": mapping["mapping_id"], "product_item_id": item["product_item_id"], "retailer": mapping["target_retailer"], "candidate_product_name": mapping["target_product_name"], "candidate_url": mapping["target_product_url"], "candidate_brand": mapping.get("target_brand"), "candidate_package_size": mapping.get("target_package_size"), "candidate_unit": mapping.get("target_unit"), "candidate_price_text": mapping.get("target_price_text"), "candidate_unit_price_text": mapping.get("target_unit_price_text"), "match_confidence": mapping.get("confidence") or "0.95", "quality_flags_json": mapping.get("quality_flags_json") or json.dumps([mapping.get("match_type") or "manual_match"]), "status": "accepted", "source": "known_mapping"})

        for mapping in known_mappings_for_item(conn, item, include_rejected=False):
            if mapping["status"] == "accepted" and mapping["target_retailer"] in selected and mapping.get("target_price_text"):
                candidates.append({"match_id": mapping["mapping_id"], "product_item_id": item["product_item_id"], "retailer": mapping["target_retailer"], "candidate_product_name": mapping["target_product_name"], "candidate_url": mapping["target_product_url"], "candidate_brand": mapping.get("target_brand"), "candidate_package_size": mapping.get("target_package_size"), "candidate_unit": mapping.get("target_unit"), "candidate_price_text": mapping.get("target_price_text"), "candidate_unit_price_text": mapping.get("target_unit_price_text"), "match_confidence": mapping.get("confidence") or "0.95", "quality_flags_json": mapping.get("quality_flags_json") or json.dumps([mapping.get("match_type") or "manual_match"]), "status": "accepted", "source": "known_mapping"})


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
@@ -239,7 +239,8 @@
         item_price = _money(item["total_price_text"])
         candidates = [m for m in _matches(conn, item["product_item_id"]) if m["retailer"] in selected]
         for mapping in known_mappings_for_item(conn, item, include_rejected=False):
-            if mapping["status"] == "accepted" and mapping["target_retailer"] in selected and mapping.get("target_price_text"):
+            mapping_flags = set(json.loads(mapping.get("quality_flags_json") or "[]"))
+            if mapping["status"] == "accepted" and mapping["target_retailer"] in selected and mapping.get("target_price_text") and mapping.get("last_price_checked_at") and "manual_price" not in mapping_flags:
                 candidates.append({"match_id": mapping["mapping_id"], "product_item_id": item["product_item_id"], "retailer": mapping["target_retailer"], "candidate_product_name": mapping["target_product_name"], "candidate_url": mapping["target_product_url"], "candidate_brand": mapping.get("target_brand"), "candidate_package_size": mapping.get("target_package_size"), "candidate_unit": mapping.get("target_unit"), "candidate_price_text": mapping.get("target_price_text"), "candidate_unit_price_text": mapping.get("target_unit_price_text"), "match_confidence": mapping.get("confidence") or "0.95", "quality_flags_json": mapping.get("quality_flags_json") or json.dumps([mapping.get("match_type") or "manual_match"]), "status": "accepted", "source": "known_mapping"})
         rejected_mappings = [m for m in conn.execute("SELECT * FROM grocery_product_mappings WHERE source_product_normalized_name=? AND status='rejected'", (item.get("normalized_product_name"),)).fetchall()]
         safe = [m for m in candidates if _safe_match(m)]


        if prefer_known_mappings:
            accepted = [m for m in known_mappings_for_item(conn, item, include_rejected=False) if m.get("status") == "accepted"]
            accepted_with_sourced_price = False
            for m in accepted:
                flags = set(json.loads(m.get("quality_flags_json") or "[]"))
                has_sourced_price = bool(m.get("target_price_text") and m.get("last_price_checked_at") and "manual_price" not in flags)
                matches.append({"mapping_id": m["mapping_id"], "product_item_id": item["product_item_id"], "retailer": m["target_retailer"], "product_name": m["target_product_name"], "product_url": m["target_product_url"], "price_text": m.get("target_price_text"), "unit_price_text": m.get("target_unit_price_text"), "source_fetched_at": m.get("last_price_checked_at"), "fetched_at": m.get("updated_at"), "quality_flags": json.loads(m.get("quality_flags_json") or "[]"), "confidence": m.get("confidence"), "status": "suggested" if has_sourced_price else "needs_review", "source": "known_mapping", "cache_status": m.get("cache_status")})
                known_mapping_hits += 1
                accepted_with_sourced_price = accepted_with_sourced_price or has_sourced_price
            if accepted_with_sourced_price:
                continue
            rejected_targets = [row_to_dict(r) for r in conn.execute("SELECT target_retailer, target_product_url, target_product_name FROM grocery_product_mappings WHERE source_product_normalized_name=? AND status='rejected'", (item.get("normalized_product_name"),)).fetchall()]
        else:
            rejected_targets = []
        for provider in providers:

        if prefer_known_mappings:
            accepted = [m for m in known_mappings_for_item(conn, item, include_rejected=False) if m.get("status") == "accepted"]
            if accepted:
                for m in accepted:
                    matches.append({"mapping_id": m["mapping_id"], "product_item_id": item["product_item_id"], "retailer": m["target_retailer"], "product_name": m["target_product_name"], "product_url": m["target_product_url"], "price_text": m.get("target_price_text"), "unit_price_text": m.get("target_unit_price_text"), "source_fetched_at": m.get("last_price_checked_at"), "fetched_at": m.get("updated_at"), "quality_flags": json.loads(m.get("quality_flags_json") or "[]"), "confidence": m.get("confidence"), "status": "suggested" if m.get("target_price_text") else "needs_review", "source": "known_mapping", "cache_status": m.get("cache_status")})
                    known_mapping_hits += 1
                continue
            rejected = conn.execute("SELECT 1 FROM grocery_product_mappings WHERE source_product_normalized_name=? AND status='rejected' LIMIT 1", (item.get("normalized_product_name"),)).fetchone()
            if rejected:
                matches.append({"product_item_id": item["product_item_id"], "retailer": None, "product_name": item.get("raw_product_name"), "status": "rejected", "quality_flags": ["rejected"], "source": "known_mapping"})
                known_mapping_hits += 1
                continue
        for provider in providers:


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_price_providers.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_price_providers.py
@@ -310,16 +310,18 @@
         query = item.get("normalized_product_name") or item.get("raw_product_name")
         if prefer_known_mappings:
             accepted = [m for m in known_mappings_for_item(conn, item, include_rejected=False) if m.get("status") == "accepted"]
-            if accepted:
-                for m in accepted:
-                    matches.append({"mapping_id": m["mapping_id"], "product_item_id": item["product_item_id"], "retailer": m["target_retailer"], "product_name": m["target_product_name"], "product_url": m["target_product_url"], "price_text": m.get("target_price_text"), "unit_price_text": m.get("target_unit_price_text"), "source_fetched_at": m.get("last_price_checked_at"), "fetched_at": m.get("updated_at"), "quality_flags": json.loads(m.get("quality_flags_json") or "[]"), "confidence": m.get("confidence"), "status": "suggested" if m.get("target_price_text") else "needs_review", "source": "known_mapping", "cache_status": m.get("cache_status")})
-                    known_mapping_hits += 1
+            accepted_with_sourced_price = False
+            for m in accepted:
+                flags = set(json.loads(m.get("quality_flags_json") or "[]"))
+                has_sourced_price = bool(m.get("target_price_text") and m.get("last_price_checked_at") and "manual_price" not in flags)
+                matches.append({"mapping_id": m["mapping_id"], "product_item_id": item["product_item_id"], "retailer": m["target_retailer"], "product_name": m["target_product_name"], "product_url": m["target_product_url"], "price_text": m.get("target_price_text"), "unit_price_text": m.get("target_unit_price_text"), "source_fetched_at": m.get("last_price_checked_at"), "fetched_at": m.get("updated_at"), "quality_flags": json.loads(m.get("quality_flags_json") or "[]"), "confidence": m.get("confidence"), "status": "suggested" if has_sourced_price else "needs_review", "source": "known_mapping", "cache_status": m.get("cache_status")})
+                known_mapping_hits += 1
+                accepted_with_sourced_price = accepted_with_sourced_price or has_sourced_price
+            if accepted_with_sourced_price:
                 continue
-            rejected = conn.execute("SELECT 1 FROM grocery_product_mappings WHERE source_product_normalized_name=? AND status='rejected' LIMIT 1", (item.get("normalized_product_name"),)).fetchone()
-            if rejected:
-                matches.append({"product_item_id": item["product_item_id"], "retailer": None, "product_name": item.get("raw_product_name"), "status": "rejected", "quality_flags": ["rejected"], "source": "known_mapping"})
-                known_mapping_hits += 1
-                continue
+            rejected_targets = [row_to_dict(r) for r in conn.execute("SELECT target_retailer, target_product_url, target_product_name FROM grocery_product_mappings WHERE source_product_normalized_name=? AND status='rejected'", (item.get("normalized_product_name"),)).fetchall()]
+        else:
+            rejected_targets = []
         for provider in providers:
             before = provider.request_count
             results = provider.search_products(conn, query, use_cache=use_cache, max_results=3)

/home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_price_providers.py was last read with offset/limit pagination (partial view). Re-read the whole file before overwriting it.


                flags = quality_flags_for_match(item.get("normalized_product_name") or item.get("raw_product_name"), result)
                if any(rt.get("target_retailer") == result.get("retailer") and (rt.get("target_product_url") == result.get("product_url") or rt.get("target_product_name") == result.get("product_name")) for rt in rejected_targets):
                    stored = upsert_product_match(conn, item["product_item_id"], retailer=result["retailer"], candidate_product_name=result["product_name"], candidate_url=result["product_url"], candidate_price_text=result.get("price_decimal_text") or result.get("price_text"), candidate_unit_price_text=result.get("unit_price_text"), quality_flags=[*flags, "rejected"], match_confidence=result.get("confidence") or "0.40", status="rejected", source=result.get("source") or "web_fetch", candidate_brand=result.get("brand"), candidate_package_size=result.get("package_size"), candidate_unit=result.get("unit"))
                    matches.append({"match_id": stored.get("match_id"), "product_item_id": item["product_item_id"], "retailer": result.get("retailer"), "product_name": result.get("product_name"), "product_url": result.get("product_url"), "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": stored.get("fetched_at"), "quality_flags": [*flags, "rejected"], "confidence": result.get("confidence"), "status": "rejected", "source": result.get("source")})
                    continue
                comparison = compare_product_prices

                flags = quality_flags_for_match(item.get("normalized_product_name") or item.get("raw_product_name"), result)
                comparison = compare_product_prices


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_price_providers.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_price_providers.py
@@ -333,6 +333,10 @@
                     matches.append({"product_item_id": item["product_item_id"], **result})
                     continue
                 flags = quality_flags_for_match(item.get("normalized_product_name") or item.get("raw_product_name"), result)
+                if any(rt.get("target_retailer") == result.get("retailer") and (rt.get("target_product_url") == result.get("product_url") or rt.get("target_product_name") == result.get("product_name")) for rt in rejected_targets):
+                    stored = upsert_product_match(conn, item["product_item_id"], retailer=result["retailer"], candidate_product_name=result["product_name"], candidate_url=result["product_url"], candidate_price_text=result.get("price_decimal_text") or result.get("price_text"), candidate_unit_price_text=result.get("unit_price_text"), quality_flags=[*flags, "rejected"], match_confidence=result.get("confidence") or "0.40", status="rejected", source=result.get("source") or "web_fetch", candidate_brand=result.get("brand"), candidate_package_size=result.get("package_size"), candidate_unit=result.get("unit"))
+                    matches.append({"match_id": stored.get("match_id"), "product_item_id": item["product_item_id"], "retailer": result.get("retailer"), "product_name": result.get("product_name"), "product_url": result.get("product_url"), "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": stored.get("fetched_at"), "quality_flags": [*flags, "rejected"], "confidence": result.get("confidence"), "status": "rejected", "source": result.get("source")})
+                    continue
                 comparison = compare_product_prices(original_price_text=str(item.get("total_price_text") or "0"), original_unit_price_text=item.get("unit_price_text"), candidate_price_text=result.get("price_decimal_text") or result.get("price_text"), candidate_unit_price_text=result.get("unit_price_decimal_text") or result.get("unit_price_text"), currency=result.get("currency") or "CHF", quality_flags=flags)
                 status = "suggested" if comparison["can_calculate_savings"] and (not only_safe_matches or not (set(flags) & UNSAFE_FLAGS)) else "needs_review"
                 if not comparison["can_calculate_savings"] or not result.get("price_decimal_text"):


                if not comparison["can_calculate_savings"] or not result.get("price_decimal_text"):
                    stored = upsert_product_match(conn, item["product_item_id"], retailer=result["retailer"], candidate_product_name=result["product_name"], candidate_url=result["product_url"], candidate_price_text=result.get("price_decimal_text") or result.get("price_text"), candidate_unit_price_text=result.get("unit_price_text"), quality_flags=comparison["quality_flags"], match_confidence=result.get("confidence") or "0.40", status="needs_review", source=result.get("source") or "web_fetch", candidate_brand=result.get("brand"), candidate_package_size=result.get("package_size"), candidate_unit=result.get("unit"))
                    matches.append({"match_id": stored.get("match_id"), "product_item_id": item["product_item_id"], "retailer": result.get("retailer"), "product_name": result.get("product_name"), "product_url": result.get("product_url"), "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": stored.get("fetched_at"), "quality_flags": comparison["quality_flags"], "confidence": result.get("confidence"), "status": "needs_review", "source": result.get("source")})
                    continue

                if not comparison["can_calculate_savings"] or not result.get("price_decimal_text"):
                    matches.append({"product_item_id": item["product_item_id"], "retailer": result.get("retailer"), "product_name": result.get("product_name"), "product_url": result.get("product_url"), "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": result.get("fetched_at"), "quality_flags": comparison["quality_flags"], "confidence": result.get("confidence"), "status": status, "source": result.get("source")})
                    continue


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_price_providers.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_price_providers.py
@@ -340,7 +340,8 @@
                 comparison = compare_product_prices(original_price_text=str(item.get("total_price_text") or "0"), original_unit_price_text=item.get("unit_price_text"), candidate_price_text=result.get("price_decimal_text") or result.get("price_text"), candidate_unit_price_text=result.get("unit_price_decimal_text") or result.get("unit_price_text"), currency=result.get("currency") or "CHF", quality_flags=flags)
                 status = "suggested" if comparison["can_calculate_savings"] and (not only_safe_matches or not (set(flags) & UNSAFE_FLAGS)) else "needs_review"
                 if not comparison["can_calculate_savings"] or not result.get("price_decimal_text"):
-                    matches.append({"product_item_id": item["product_item_id"], "retailer": result.get("retailer"), "product_name": result.get("product_name"), "product_url": result.get("product_url"), "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": result.get("fetched_at"), "quality_flags": comparison["quality_flags"], "confidence": result.get("confidence"), "status": status, "source": result.get("source")})
+                    stored = upsert_product_match(conn, item["product_item_id"], retailer=result["retailer"], candidate_product_name=result["product_name"], candidate_url=result["product_url"], candidate_price_text=result.get("price_decimal_text") or result.get("price_text"), candidate_unit_price_text=result.get("unit_price_text"), quality_flags=comparison["quality_flags"], match_confidence=result.get("confidence") or "0.40", status="needs_review", source=result.get("source") or "web_fetch", candidate_brand=result.get("brand"), candidate_package_size=result.get("package_size"), candidate_unit=result.get("unit"))
+                    matches.append({"match_id": stored.get("match_id"), "product_item_id": item["product_item_id"], "retailer": result.get("retailer"), "product_name": result.get("product_name"), "product_url": result.get("product_url"), "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": stored.get("fetched_at"), "quality_flags": comparison["quality_flags"], "confidence": result.get("confidence"), "status": "needs_review", "source": result.get("source")})
                     continue
                 stored = upsert_product_match(conn, item["product_item_id"], retailer=result["retailer"], candidate_product_name=result["product_name"], candidate_url=result["product_url"], candidate_price_text=result.get("price_decimal_text") or result.get("price_text") or "0", candidate_unit_price_text=result.get("unit_price_text"), quality_flags=comparison["quality_flags"], match_confidence=result.get("confidence") or "0.60", status=status, source=result.get("source") or "web_fetch", candidate_brand=result.get("brand"), candidate_package_size=result.get("package_size"), candidate_unit=result.get("unit"))
                 matches.append({"match_id": stored.get("match_id"), "product_item_id": item["product_item_id"], "retailer": result["retailer"], "product_name": result["product_name"], "product_url": result["product_url"], "price_text": result.get("price_decimal_text") or result.get("price_text"), "unit_price_text": result.get("unit_price_text"), "source_fetched_at": result.get("fetched_at"), "fetched_at": stored.get("fetched_at"), "quality_flags": comparison["quality_flags"], "confidence": result.get("confidence"), "status": status, "source": result.get("source")})


Explanation:
**Figuring out potential errors**

I'm considering the possibility of a KeyError when a retailer doesn't have a price if the product URL or name is invalid. It seems like the retailer might not always be present, which could lead to issues. I think I need to run some tests to identify any blockers before moving forward. So, I'll open the test file and get started on that to ensure everything's working smoothly!


1|from __future__ import annotations
     2|
     3|import sqlite3
     4|from pathlib import Path
     5|
     6|import pytest
     7|
     8|from jarvis_finance.storage.migrations import apply_migrations, get_schema_version
     9|from jarvis_finance.services.grocery_optimizer import (
    10|    add_demo_migros_receipt,
    11|    accept_grocery_product_match,
    12|    reject_grocery_product_match,
    13|    create_manual_grocery_mapping,
    14|    get_grocery_optimizer_dashboard,
    15|    run_grocery_optimization,
    16|    generate_grocery_optimization_report,
    17|)
    18|from jarvis_finance.services.grocery_price_providers import search_and_store_product_matches, CoopProvider
    19|
    20|
    21|def db() -> sqlite3.Connection:
    22|    conn = sqlite3.connect(':memory:')
    23|    conn.row_factory = sqlite3.Row
    24|    apply_migrations(conn)
    25|    assert get_schema_version(conn) == 34
    26|    return conn
    27|
    28|
    29|def seed_receipt(conn: sqlite3.Connection, raw='Milch 1L', receipt_id='r1') -> dict:
    30|    return add_demo_migros_receipt(conn, purchase_date='2026-05-19', store_name='Migros', receipt_id=receipt_id, items=[{'raw_product_name': raw, 'quantity_text': '1L', 'unit': 'l', 'unit_price_text': '1.50/l', 'total_price_text': '1.50'}])
    31|
    32|
    33|def test_accept_match_creates_preferred_mapping_and_next_receipt_reuses_without_provider_call() -> None:
    34|    conn = db()
    35|    first = seed_receipt(conn, receipt_id='r1')
    36|    provider = CoopProvider(fetcher=lambda _url: '
Prix Garantie Milch 1LCHF 1.201.20/l
')
    37|    found = search_and_store_product_matches(conn, 'r1', providers=[provider], included_product_item_ids=[first['items'][0]['product_item_id']], use_cache=False)
    38|    accepted = accept_grocery_product_match(conn, found['matches'][0]['match_id'], user_note='passt')
    39|    assert accepted['status'] == 'accepted'
    40|    assert accepted['target_retailer'] == 'Coop'
    41|
    42|    second = seed_receipt(conn, receipt_id='r2')
    43|    fail_provider = CoopProvider(fetcher=lambda _url: (_ for _ in ()).throw(AssertionError('provider must not be called')))
    44|    reused = search_and_store_product_matches(conn, 'r2', providers=[fail_provider], included_product_item_ids=[second['items'][0]['product_item_id']], prefer_known_mappings=True, use_cache=True)
    45|    assert reused['provider_calls'] == 0
    46|    assert reused['known_mapping_hits'] == 1
    47|    assert reused['matches'][0]['source'] == 'known_mapping'
    48|    run = run_grocery_optimization(conn, 'r2', selected_retailers=['Coop'], included_product_item_ids=[second['items'][0]['product_item_id']])
    49|    assert run['summary']['replaceable_product_count'] == 1
    50|
    51|
    52|def test_rejected_match_is_persisted_and_not_recommended_again() -> None:
    53|    conn = db()
    54|    receipt = seed_receipt(conn)
    55|    provider = CoopProvider(fetcher=lambda _url: '
Coop Milch 1LCHF 1.201.20/l
')
    56|    found = search_and_store_product_matches(conn, 'r1', providers=[provider], included_product_item_ids=[receipt['items'][0]['product_item_id']], use_cache=False)
    57|    rejected = reject_grocery_product_match(conn, found['matches'][0]['match_id'], user_note='falsch')
    58|    assert rejected['status'] == 'rejected'
    59|
    60|    second = seed_receipt(conn, receipt_id='r2')
    61|    reused = search_and_store_product_matches(conn, 'r2', providers=[provider], included_product_item_ids=[second['items'][0]['product_item_id']], prefer_known_mappings=True, use_cache=True)
    62|    assert all(m.get('status') != 'suggested' for m in reused['matches'])
    63|    run = run_grocery_optimization(conn, 'r2', selected_retailers=['Coop'], included_product_item_ids=[second['items'][0]['product_item_id']])
    64|    assert run['summary']['replaceable_product_count'] == 0
    65|    assert 'Milch 1L' in run['optimized_lists'][0]['rejected_products']
    66|
    67|
    68|def test_manual_mapping_can_be_saved_and_used_with_optional_source_price() -> None:
    69|    conn = db()
    70|    receipt = seed_receipt(conn)
    71|    mapping = create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Denner', target_product_name='Denner Milch 1L', target_product_url='https://www.denner.ch/milch', target_brand='Denner', target_package_size='1L', target_unit='l', match_type='manual_match', candidate_price_text='1.10', candidate_unit_price_text='1.10/l', user_note='manuell geprüft')
    72|    assert mapping['status'] == 'accepted'
    73|    assert mapping['match_type'] == 'manual_match'
    74|    run = run_grocery_optimization(conn, 'r1', selected_retailers=['Denner'], included_product_item_ids=[receipt['items'][0]['product_item_id']])
    75|    assert run['optimized_total_text'] == '1.10'
    76|
    77|
    78|def test_dashboard_surfaces_known_mappings_and_cache_status() -> None:
    79|    conn = db()
    80|    receipt = seed_receipt(conn)
    81|    mapping = create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Coop', target_product_name='Coop Milch 1L', target_product_url='https://coop.example/milch', match_type='manual_match', candidate_price_text='1.20')
    82|    dash = get_grocery_optimizer_dashboard(conn)
    83|    item = dash['receipts'][0]['items'][0]
    84|    assert item['known_mappings'][0]['mapping_id'] == mapping['mapping_id']
    85|    assert item['known_mappings'][0]['cache_status'] in {'frisch', 'veraltet', 'fehlt', 'fehler'}
    86|
    87|
    88|def test_report_contains_known_mappings_sources_and_disclaimer_runtime_only(tmp_path: Path) -> None:
    89|    conn = db()
    90|    receipt = seed_receipt(conn)
    91|    create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Coop', target_product_name='Coop Milch 1L', target_product_url='https://coop.example/milch', match_type='manual_match', candidate_price_text='1.20')
    92|    run = run_grocery_optimization(conn, 'r1', selected_retailers=['Coop'], included_product_item_ids=[receipt['items'][0]['product_item_id']])
    93|    report = generate_grocery_optimization_report(conn, run['run_id'], reports_dir=tmp_path / 'runtime-reports', repo_root=tmp_path / 'repo')
    94|    text = Path(report['report_path']).read_text()
    95|    assert report['runtime_only'] is True
    96|    assert 'Bekannte Mappings' in text
    97|    assert 'https://coop.example/milch' in text
    98|    assert 'keine medizinische Empfehlung' in text
    99|


     1|from __future__ import annotations
     2|
     3|import hashlib
     4|import html
     5|import json
     6|import re
     7|from dataclasses import dataclass
     8|from datetime import datetime, timezone, timedelta
     9|from decimal import Decimal, InvalidOperation
    10|from sqlite3 import Connection
    11|from typing import Any, Callable
    12|from urllib.parse import quote_plus, urljoin
    13|from urllib.request import Request, urlopen
    14|
    15|from jarvis_finance.services.budget_common import new_id, row_to_dict
    16|from jarvis_finance.services.grocery_optimizer import _items_for_receipt, upsert_product_match, known_mappings_for_item
    17|
    18|SAFE_MATCH_FLAGS = {"exact_match", "close_match", "cheaper_private_label"}
    19|UNSAFE_FLAGS = {"needs_review", "no_price", "no_unit_price", "different_pack_size", "different_quality", "organic_mismatch"}
    20|
    21|
    22|def _now() -> str:
    23|    return datetime.now(timezone.utc).isoformat()
    24|
    25|
    26|def _decimal_text(value: Any) -> str | None:
    27|    if value is None:
    28|        return None
    29|    text = str(value).replace("CHF", "").replace("'", "").replace("’", "").strip().replace(",", ".")
    30|    match = re.search(r"-?\d+(?:\.\d+)?", text)
    31|    if not match:
    32|        return None
    33|    try:
    34|        return f"{Decimal(match.group(0)).quantize(Decimal('0.01')):.2f}"
    35|    except InvalidOperation:
    36|        return None
    37|
    38|
    39|def _money(value: Any) -> Decimal:
    40|    text = _decimal_text(value)
    41|    return Decimal(text or "0.00")
    42|
    43|
    44|def _abs_url(base: str, url: str) -> str:
    45|    return urljoin(base, html.unescape(url or ""))
    46|
    47|
    48|class GroceryProviderRateLimitError(RuntimeError):
    49|    pass
    50|
    51|
    52|@dataclass
    53|class GroceryProviderResult:
    54|    retailer: str
    55|    product_name: str
    56|    product_url: str
    57|    price_text: str | None = None
    58|    unit_price_text: str | None = None
    59|    brand: str | None = None
    60|    image_url: str | None = None
    61|    package_size: str | None = None
    62|    unit: str | None = None
    63|    currency: str = "CHF"
    64|    availability_status: str = "unknown"
    65|    promotion_text: str | None = None
    66|    source: str = "web_fetch"
    67|    confidence: str = "0.60"
    68|    quality_flags: list[str] | None = None
    69|    status: str = "suggested"
    70|    fetched_at: str | None = None
    71|    raw_result: dict[str, Any] | None = None
    72|
    73|    def as_dict(self) -> dict[str, Any]:
    74|        price_decimal = _decimal_text(self.price_text)
    75|        unit_decimal = _decimal_text(self.unit_price_text)
    76|        flags = list(self.quality_flags or [])
    77|        if not price_decimal and "no_price" not in flags:
    78|            flags.append("no_price")
    79|        if not unit_decimal and "no_unit_price" not in flags:
    80|            flags.append("no_unit_price")
    81|        if not flags:
    82|            flags = ["needs_review"]
    83|        return {
    84|            "retailer": self.retailer,
    85|            "product_name": self.product_name,
    86|            "brand": self.brand,
    87|            "product_url": self.product_url,
    88|            "image_url": self.image_url,
    89|            "price_text": price_decimal or self.price_text,
    90|            "price_decimal_text": price_decimal,
    91|

Use offset=91 to continue reading (showing 1-90 of 348 lines)


Explanation:
**Considering code testing**

I’m thinking about whether to add a test with the GroceryProductProvider subclass. It doesn't seem strictly necessary at this point, and I need to compile first. I should focus on patching the tests to make sure everything is running smoothly before I dive into adding new tests. I wonder if this approach will streamline the process and avoid unnecessary complications. Let’s keep it simple for now!

    create_manual_grocery_mapping,
    refresh_grocery_mapping_price,
    get_grocery_optimizer_dashboard,

    create_manual_grocery_mapping,
    get_grocery_optimizer_dashboard,


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
@@ -11,6 +11,7 @@
     accept_grocery_product_match,
     reject_grocery_product_match,
     create_manual_grocery_mapping,
+    refresh_grocery_mapping_price,
     get_grocery_optimizer_dashboard,
     run_grocery_optimization,
     generate_grocery_optimization_report,


from jarvis_finance.services.grocery_price_providers import search_and_store_product_matches, CoopProvider, GroceryProductProvider, GroceryProviderResult


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
@@ -16,7 +16,7 @@
     run_grocery_optimization,
     generate_grocery_optimization_report,
 )
-from jarvis_finance.services.grocery_price_providers import search_and_store_product_matches, CoopProvider
+from jarvis_finance.services.grocery_price_providers import search_and_store_product_matches, CoopProvider, GroceryProductProvider, GroceryProviderResult

 def db() -> sqlite3.Connection:


def test_manual_mapping_can_be_saved_and_used_after_sourced_cache_refresh() -> None:
    conn = db()
    receipt = seed_receipt(conn)
    mapping = create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Denner', target_product_name='Denner Milch 1L', target_product_url='https://www.denner.ch/milch', target_brand='Denner', target_package_size='1L', target_unit='l', match_type='manual_match', candidate_price_text='1.10', candidate_unit_price_text='1.10/l', user_note='manuell notiert')
    assert mapping['status'] == 'accepted'
    assert mapping['match_type'] == 'manual_match'
    assert mapping['last_price_checked_at'] is None
    run_before = run_grocery_optimization(conn, 'r1', selected_retailers=['Denner'], included_product_item_ids=[receipt['items'][0]['product_item_id']])
    assert run_before['summary']['replaceable_product_count'] == 0
    conn.execute("INSERT INTO grocery_product_details_cache(detail_id,retailer,product_url,product_name,price_text,unit_price_text,package_size,ingredients_text,nutrition_json,fetched_at,cache_status,source_hash) VALUES ('d1','Denner','https://www.denner.ch/milch','Denner Milch 1L','1.10','1.10/l','1L',NULL,'{}','2026-05-19T10:00:00+00:00','cached','hash')")
    refreshed = refresh_grocery_mapping_price(conn, mapping['mapping_id'])
    assert refreshed['last_price_checked_at'] == '2026-05-19T10:00:00+00:00'
    run_after = run_grocery_optimization(conn, 'r1', selected_retailers=['Denner'], included_product_item_ids=[receipt['items'][0]['product_item_id']])
    assert run_after['optimized_total_text'] == '1.10'

def test_manual_mapping_rejects_unsafe_url_and_match_type() -> None:
    conn = db()
    receipt = seed_receipt(conn)
    with pytest.raises(ValueError):
        create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Coop', target_product_name='Bad', target_product_url='javascript:alert(1)')
    with pytest.raises(ValueError):
        create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Coop', target_product_name='Bad', target_product_url='https://coop.example/bad', match_type='evil_match')

def test_manual_mapping_can_be_saved_and_used_with_optional_source_price() -> None:
    conn = db()
    receipt = seed_receipt(conn)
    mapping = create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Denner', target_product_name='Denner Milch 1L', target_product_url='https://www.denner.ch/milch', target_brand='Denner', target_package_size='1L', target_unit='l', match_type='manual_match', candidate_price_text='1.10', candidate_unit_price_text='1.10/l', user_note='manuell geprüft')
    assert mapping['status'] == 'accepted'
    assert mapping['match_type'] == 'manual_match'
    run = run_grocery_optimization(conn, 'r1', selected_retailers=['Denner'], included_product_item_ids=[receipt['items'][0]['product_item_id']])
    assert run['optimized_total_text'] == '1.10'


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
@@ -66,14 +66,29 @@
     assert 'Milch 1L' in run['optimized_lists'][0]['rejected_products']

-def test_manual_mapping_can_be_saved_and_used_with_optional_source_price() -> None:
+def test_manual_mapping_can_be_saved_and_used_after_sourced_cache_refresh() -> None:
     conn = db()
     receipt = seed_receipt(conn)
-    mapping = create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Denner', target_product_name='Denner Milch 1L', target_product_url='https://www.denner.ch/milch', target_brand='Denner', target_package_size='1L', target_unit='l', match_type='manual_match', candidate_price_text='1.10', candidate_unit_price_text='1.10/l', user_note='manuell geprüft')
+    mapping = create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Denner', target_product_name='Denner Milch 1L', target_product_url='https://www.denner.ch/milch', target_brand='Denner', target_package_size='1L', target_unit='l', match_type='manual_match', candidate_price_text='1.10', candidate_unit_price_text='1.10/l', user_note='manuell notiert')
     assert mapping['status'] == 'accepted'
     assert mapping['match_type'] == 'manual_match'
-    run = run_grocery_optimization(conn, 'r1', selected_retailers=['Denner'], included_product_item_ids=[receipt['items'][0]['product_item_id']])
-    assert run['optimized_total_text'] == '1.10'
+    assert mapping['last_price_checked_at'] is None
+    run_before = run_grocery_optimization(conn, 'r1', selected_retailers=['Denner'], included_product_item_ids=[receipt['items'][0]['product_item_id']])
+    assert run_before['summary']['replaceable_product_count'] == 0
+    conn.execute("INSERT INTO grocery_product_details_cache(detail_id,retailer,product_url,product_name,price_text,unit_price_text,package_size,ingredients_text,nutrition_json,fetched_at,cache_status,source_hash) VALUES ('d1','Denner','https://www.denner.ch/milch','Denner Milch 1L','1.10','1.10/l','1L',NULL,'{}','2026-05-19T10:00:00+00:00','cached','hash')")
+    refreshed = refresh_grocery_mapping_price(conn, mapping['mapping_id'])
+    assert refreshed['last_price_checked_at'] == '2026-05-19T10:00:00+00:00'
+    run_after = run_grocery_optimization(conn, 'r1', selected_retailers=['Denner'], included_product_item_ids=[receipt['items'][0]['product_item_id']])
+    assert run_after['optimized_total_text'] == '1.10'
+
+
+def test_manual_mapping_rejects_unsafe_url_and_match_type() -> None:
+    conn = db()
+    receipt = seed_receipt(conn)
+    with pytest.raises(ValueError):
+        create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Coop', target_product_name='Bad', target_product_url='javascript:alert(1)')
+    with pytest.raises(ValueError):
+        create_manual_grocery_mapping(conn, product_item_id=receipt['items'][0]['product_item_id'], target_retailer='Coop', target_product_name='Bad', target_product_url='https://coop.example/bad', match_type='evil_match')

 def test_dashboard_surfaces_known_mappings_and_cache_status() -> None:


Explanation:
**Evaluating manual flags**

I'm analyzing the manual flags, specifically noting that `manual_price` is marked as unsafe due to not being a subset. Even after a refresh, it still shows up in the quality flags, which means the run won't utilize it. I think I need to refresh and potentially remove `manual_price` when applying the sourced cache and consider replacing the flags. Adding an update to `quality_flags_json` seems necessary to remove `manual_price` and add `manual_match` instead.

        flags = [f for f in json.loads(mapping.get("quality_flags_json") or "[]") if f != "manual_price"]
        if "manual_match" not in flags:
            flags.append("manual_match")
        conn.execute(
            """UPDATE grocery_product_mappings
               SET target_price_text=?, target_unit_price_text=?, quality_flags_json=?, last_price_checked_at=?, updated_at=?
               WHERE mapping_id=?""",
            (cache.get("price_text"), cache.get("unit_price_text"), json.dumps(flags), cache.get("last_price_checked_at"), _now(), mapping_id),
        )

        conn.execute(
            """UPDATE grocery_product_mappings
               SET target_price_text=?, target_unit_price_text=?, last_price_checked_at=?, updated_at=?
               WHERE mapping_id=?""",
            (cache.get("price_text"), cache.get("unit_price_text"), cache.get("last_price_checked_at"), _now(), mapping_id),
        )


--- a//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
+++ b//home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_optimizer.py
@@ -201,11 +201,14 @@
     mapping = row_to_dict(row)
     cache = _cache_status_for_url(conn, mapping["target_retailer"], mapping["target_product_url"])
     if cache.get("price_text"):
+        flags = [f for f in json.loads(mapping.get("quality_flags_json") or "[]") if f != "manual_price"]
+        if "manual_match" not in flags:
+            flags.append("manual_match")
         conn.execute(
             """UPDATE grocery_product_mappings
-               SET target_price_text=?, target_unit_price_text=?, last_price_checked_at=?, updated_at=?
+               SET target_price_text=?, target_unit_price_text=?, quality_flags_json=?, last_price_checked_at=?, updated_at=?
                WHERE mapping_id=?""",
-            (cache.get("price_text"), cache.get("unit_price_text"), cache.get("last_price_checked_at"), _now(), mapping_id),
+            (cache.get("price_text"), cache.get("unit_price_text"), json.dumps(flags), cache.get("last_price_checked_at"), _now(), mapping_id),
         )
         conn.commit()
         mapping.update({"target_price_text": cache.get("price_text"), "target_unit_price_text": cache.get("unit_price_text"), "last_price_checked_at": cache.get("last_price_checked_at")})


Explanation:
**Creating tests and providers**

I need to add tests for cases where there’s no price match ID, ensuring that rejection doesn’t suppress other functionalities. This seems important for reliability. I’m considering the need for a DummyProvider too, which will help in testing. Oh, and maybe I can simplify things by appending the class after the seed stage. That could streamline the process a bit. Let’s get started on that!

def seed_receipt(conn: sqlite3.Connection, raw='Milch 1L', receipt_id='r1') -> dict:
    return add_demo_migros_receipt(conn, purchase_date='2026-05-19', store_name='Migros', receipt_id=receipt_id, items=[{'raw_product_name': raw, 'quantity_text': '1L', 'unit': 'l', 'unit_price_text': '1.50/l', 'total_price_text': '1.50'}])

class StaticProvider(GroceryProductProvider):
    def __init__(self, retailer: str, results: list[GroceryProviderResult]) -> None:
        super().__init__(retailer=retailer, base_url=f'https://{retailer.lower().replace(" ", "-")}.example', search_path='/search')
        self._results = results

    def _fetch_search_results(self, query: str) -> list[GroceryProviderResult]:
        return self._results


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
@@ -29,6 +29,15 @@

 def seed_receipt(conn: sqlite3.Connection, raw='Milch 1L', receipt_id='r1') -> dict:
     return add_demo_migros_receipt(conn, purchase_date='2026-05-19', store_name='Migros', receipt_id=receipt_id, items=[{'raw_product_name': raw, 'quantity_text': '1L', 'unit': 'l', 'unit_price_text': '1.50/l', 'total_price_text': '1.50'}])
+
+
+class StaticProvider(GroceryProductProvider):
+    def __init__(self, retailer: str, results: list[GroceryProviderResult]) -> None:
+        super().__init__(retailer=retailer, base_url=f'https://{retailer.lower().replace(" ", "-")}.example', search_path='/search')
+        self._results = results
+
+    def _fetch_search_results(self, query: str) -> list[GroceryProviderResult]:
+        return self._results

 def test_accept_match_creates_preferred_mapping_and_next_receipt_reuses_without_provider_call() -> None:

LSP diagnostics introduced by this edit:
<diagnostics file="/home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py">
ERROR [36:26] No parameter named "retailer" [reportCallIssue] (Pyright)
ERROR [36:45] No parameter named "base_url" [reportCallIssue] (Pyright)
ERROR [36:111] No parameter named "search_path" [reportCallIssue] (Pyright)
</diagnostics>


{"total_count": 11, "matches": [{"path": "/home/agent/.hermes/repos/FinanceManager/src/jarvis_finance/services/grocery_price_providers.py", "line": 102, "content": "            \"status\": self.status if self.status else (\"suggested\" if set(flags) & SAFE_MATCH_FLAGS else \"needs_review\"),"}], "truncated": true}

[Hint: Results truncated. Use offset=1 to see more, or narrow with a more specific pattern or file_glob.]


104|        }
   105|
   106|
   107|class GroceryProductProvider:
   108|    retailer = "generic"
   109|    base_url = "https://example.invalid/"
   110|    future_status: str | None = None
   111|
   112|    def __init__(self, fetcher: Callable[[str], str] | None = None, max_requests: int = 5):
   113|        self.fetcher = fetcher or self._default_fetcher
   114|        self.max_requests = max_requests
   115|        self.request_count = 0
   116|
   117|    def search_url(self, query: str, locale: str = "de-CH") -> str:
   118|        return self.base_url + "search?q=" + quote_plus(query)
   119|
   120|    def _default_fetcher(self, url: str) -> str:
   121|        if self.future_status:
   122|            raise NotImplementedError(self.future_status)
   123|        req = Request(url, headers={"User-Agent": "JarvisFinanceBot/1.0 low-volume product price lookup"})
   124|        with urlopen(req, timeout=8) as response:  # noqa: S310 - explicit user-triggered low-volume public lookup
   125|            if response.status == 429:
   126|                raise GroceryProviderRateLimitError("rate_limited")
   127|            return response.read().decode("utf-8", errors="ignore")
   128|
   129|    def _fetch(self, url: str) -> str:
   130|        self.request_count += 1
   131|        if self.request_count > self.max_requests:
   132|            raise GroceryProviderRateLimitError("rate_limited")
   133|        return self.fetcher(url)
   134|
   135|    def normalize_result(self, raw_result: dict[str, Any]) -> GroceryProviderResult:
   136|        return GroceryProviderResult(retailer=self.retailer, **raw_result)
   137|
   138|    def parse_search_results(self, html_text: str) -> list[dict[str, Any]]:
   139|        chunks = re.findall(r"
]*data-product[^>]*>(.*?)
", html_text, flags=re.I | re.S) or [html_text]
   140|        results: list[dict[str, Any]] = []
   141|        for chunk in chunks[:5]:
   142|            link = re.search(r"]+href=[\"']([^\"']+)[\"'][^>]*>(.*?)", chunk, flags=re.I | re.S)
   143|            price = re.search(r"class=[\"'][^\"']*price[^\"']*[\"'][^>]*>(.*?)</", chunk, flags=re.I | re.S)
   144|            unit_price = re.search(r"class=[\"'][^\"']*unit-price[^\"']*[\"'][^>]*>(.*?)</", chunk, flags=re.I | re.S)
   145|            brand = re.search(r"class=[\"'][^\"']*brand[^\"']*[\"'][^>]*>(.*?)</", chunk, flags=re.I | re.S)
   146|            if not link:
   147|                continue
   148|            name = re.sub(r"<[^>]+>", "", link.group(2)).strip()
   149|            results.append({
   150|                "product_name": html.unescape(name),
   151|                "product_url": _abs_url(self.base_url, link.group(1)),
   152|                "price_text": re.sub(r"<[^>]+>", "", price.group(1)).strip() if price else None,
   153|                "unit_price_text": re.sub(r"<[^>]+>", "", unit_price.group(1)).strip() if unit_price else None,
   154|                "brand": re.sub(r"<[^>]+>", "", brand.group(1)).strip() if brand else None,
   155|                "raw_result": {"source": "html_article"},
   156|            })
   157|        return results
   158|
   159|    def _cache_key(self, query: str) -> str:
   160|        return f"provider-search://{self.retailer}/{quote_plus(query.lower().strip())}"
   161|
   162|    def get_cached_result(self, conn: Connection, query: str, max_results: int = 5, max_age_seconds: int = 86400) -> list[dict[str, Any]]:
   163|        rows = conn.execute("SELECT * FROM grocery_product_details_cache WHERE retailer=? AND source_hash=? AND cache_status='cached' ORDER BY fetched_at DESC LIMIT ?", (self.retailer, self._cache_key(query), max_results)).fetchall()
   164|        output = []
   165|        cutoff = datetime.now(timezone.utc) - timedelta(seconds=max_age_seconds)
   166|        for row in rows:
   167|            d = row_to_dict(row)
   168|            try:
   169|                fetched_dt = datetime.fromisoformat(str(d.get("fetched_at")))
   170|                if fetched_dt.tzinfo is None:
   171|                    fetched_dt = fetched_dt.replace(tzinfo=timezone.utc)
   172|                if fetched_dt < cutoff:
   173|                    continue
   174|            except Exception:
   175|                continue
   176|            output.append({"retailer": d["retailer"], "product_name": d.get("product_name"), "brand": d.get("brand"), "product_url": d.get("product_url"), "image_url": d.get("image_url"), "price_text": d.get("price_text"), "price_decimal_text": d.get("price_decimal_text"), "currency": d.get("currency") or "CHF", "package_size": d.get("package_size"), "unit": d.get("unit"), "unit_price_text": d.get("unit_price_text"), "unit_price_decimal_text": d.get("unit_price_decimal_text"), "availability_status": d.get("availability_status"), "promotion_text": d.get("promotion_text"), "fetched_at": d.get("fetched_at"), "source": "cache", "confidence": d.get("confidence") or "0", "quality_flags": json.loads(d.get("quality_flags_json") or "[]"), "status": "suggested"})
   177|        return output
   178|
   179|    def store_cache(self, conn: Connection, query: str, result: dict[str, Any]) -> None:
   180|        detail_id = new_id("gdetail")
   181|        conn.execute("""INSERT OR REPLACE INTO grocery_product_details_cache(detail_id,retailer,product_url,product_name,price_text,unit_price_text,package_size,ingredients_text,nutrition_json,fetched_at,cache_status,source_hash,brand,image_url,price_decimal_text,currency,unit,unit_price_decimal_text,availability_status,promotion_text,source,confidence,quality_flags_json,raw_result_json)
   182|            VALUES (COALESCE((SELECT detail_id FROM grocery_product_details_cache WHERE retailer=? AND product_url=?), ?), ?, ?, ?, ?, ?, ?, NULL, '{}', ?, 'cached', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
   183|            (self.retailer, result["product_url"], detail_id, self.retailer, result["product_url"], result.get("product_name"), result.get("price_text"), result.get("unit_price_text"), result.get("package_size"), result.get("fetched_at") or _now(), self._cache_key(query), result.get("brand"), result.get("image_url"), result.get("price_decimal_text"), result.get("currency") or "CHF", result.get("unit"), result.get("unit_price_decimal_text"), result.get("availability_status"), result.get("promotion_text"), result.get("source"), result.get("confidence"), json.dumps(result.get("quality_flags") or []), json.dumps(result.get("raw_result") or {})))
   184|

Use offset=184 to continue reading (showing 104-183 of 348 lines)


   184|        conn.commit()
   185|
   186|    def search_products(self, conn: Connection, query: str, retailer: str | None = None, locale: str = "de-CH", use_cache: bool = True, max_results: int = 5, max_age_seconds: int = 86400) -> list[dict[str, Any]]:
   187|        if self.future_status:
   188|            return [{"retailer": self.retailer, "query": query, "status": "needs_review", "future_status": self.future_status, "quality_flags": ["needs_review"], "source": "skeleton"}]
   189|        if use_cache:
   190|            cached = self.get_cached_result(conn, query, max_results=max_results, max_age_seconds=max_age_seconds)
   191|            if cached:
   192|                return cached
   193|        try:
   194|            html_text = self._fetch(self.search_url(query, locale=locale))
   195|            raw_rows = self.parse_search_results(html_text)[:max_results]
   196|            results = [self.normalize_result(raw).as_dict() for raw in raw_rows]
   197|            for result in results:
   198|                self.store_cache(conn, query, result)
   199|            return results
   200|        except GroceryProviderRateLimitError:
   201|            return [{"retailer": self.retailer, "query": query, "status": "needs_review", "quality_flags": ["rate_limited", "needs_review"], "source": "web_fetch", "fetched_at": _now()}]
   202|        except Exception as exc:
   203|            return [{"retailer": self.retailer, "query": query, "status": "needs_review", "quality_flags": ["provider_error", "needs_review"], "error": type(exc).__name__, "source": "web_fetch", "fetched_at": _now()}]
   204|
   205|    def fetch_product_detail(self, url: str) -> dict[str, Any]:
   206|        html_text = self._fetch(url)
   207|        parsed = self.parse_search_results(html_text)
   208|        return parsed[0] if parsed else {"product_url": url, "status": "needs_review"}
   209|
   210|
   211|class MigrosProvider(GroceryProductProvider):
   212|    retailer = "Migros"
   213|    base_url = "https://www.migros.ch/de/search?query="
   214|
   215|    def search_url(self, query: str, locale: str = "de-CH") -> str:
   216|        return self.base_url + quote_plus(query)
   217|
   218|
   219|class CoopProvider(GroceryProductProvider):
   220|    retailer = "Coop"
   221|    base_url = "https://www.coop.ch/de/search/?text="
   222|
   223|    def search_url(self, query: str, locale: str = "de-CH") -> str:
   224|

Use offset=224 to continue reading (showing 184-223 of 348 lines)


Explanation:
**Simplifying class methods**

I realize I need to ensure that the static subclass correctly sets the class attributes. I think overriding the `search_products` method could be made simpler, perhaps by using `super()` without arguments to avoid complexities. This way, I can streamline the code and make it more efficient. It's a small adjustment, but I believe it will lead to cleaner, more maintainable code overall.

class StaticProvider(GroceryProductProvider):
    def __init__(self, retailer: str, results: list[GroceryProviderResult]) -> None:
        super().__init__(fetcher=lambda _url: "")
        self.retailer = retailer
        self.base_url = f'https://{retailer.lower().replace(" ", "-")}.example/'
        self._results = results

    def search_products(self, conn: sqlite3.Connection, query: str, retailer: str | None = None, locale: str = 'de-CH', use_cache: bool = True, max_results: int = 5, max_age_seconds: int = 86400) -> list[dict]:
        self.request_count += 1
        return [r.as_dict() for r in self._results[:max_results]]

class StaticProvider(GroceryProductProvider):
    def __init__(self, retailer: str, results: list[GroceryProviderResult]) -> None:
        super().__init__(retailer=retailer, base_url=f'https://{retailer.lower().replace(" ", "-")}.example', search_path='/search')
        self._results = results

    def _fetch_search_results(self, query: str) -> list[GroceryProviderResult]:
        return self._results


--- a//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
+++ b//home/agent/.hermes/repos/FinanceManager/tests/unit/test_grocery_matching_learning_v2.py
@@ -33,11 +33,14 @@

 class StaticProvider(GroceryProductProvider):
     def __init__(self, retailer: str, results: list[GroceryProviderResult]) -> None:
-        super().__init__(retailer=retailer, base_url=f'https://{retailer.lower().replace(" ", "-")}.example', search_path='/search')
+        super().__init__(fetcher=lambda _url: "")
+        self.retailer = retailer
+        self.base_url = f'https://{retailer.lower().replace(" ", "-")}.example/'
         self._results = results

-    def _fetch_search_results(self, query: str) -> list[GroceryProviderResult]:
-        return self._results
+    def search_products(self, conn: sqlite3.Connection, query: str, retailer: str | None = None, locale: str = 'de-CH', use_cache: bool = True, max_results: int = 5, max_age_seconds: int = 86400) -> list[dict]:
+        self.request_count += 1
+        return [r.as_dict() for r in self._results[:max_results]]

 def test_accept_match_creates_preferred_mapping_and_next_receipt_reuses_without_provider_call() -> None: