"""Hermes update pipeline — extracted from ``hermes_cli/main.py``. Mechanical move (main.py decomposition): ``_cmd_update_impl``, ``_cmd_update_check`` and every module-level helper used only by the update path, plus the update-only constants they read. Function bodies are lifted verbatim; the only mechanical change is that references to helpers/constants that STAY in ``hermes_cli.main`` (and to moved-but-test-patched siblings) are routed through ``_m()`` — a lazy ``hermes_cli.main`` reference — so existing call sites and test monkeypatches that target ``hermes_cli.main.`` (``PROJECT_ROOT``, ``_is_windows``, ``_run_pre_update_backup``, ...) keep working unchanged. ``main.py`` re-imports every public-ish name from here (``# noqa: F401``) so the argparse wiring and the test-patch surface still resolve on ``hermes_cli.main``. Three self-contained closures nested inside ``_cmd_update_impl`` (``_print_items``, ``_wait_for_service_active``, ``_service_restart_sec``) were hoisted to module level; they capture no enclosing state (verified via ``symtable``). ``_restart_one_systemd_gateway_unit``, ``_resolve_manage_cmd`` and ``_on_unit_timeout`` DO capture enclosing locals and stay nested, byte-identical. Imports are one-way: ``hermes_cli.main`` imports this module, never the reverse at import time (``_m()`` resolves lazily at call time, when main.py is fully loaded, so there is no import cycle). """ import hashlib import json import logging import os import shlex import shutil import subprocess import sys import time as _time from datetime import datetime from pathlib import Path from typing import Optional from hermes_cli.config import get_hermes_home from hermes_constants import venv_python_path logger = logging.getLogger(__name__) def _m(): """Lazy ``hermes_cli.main`` reference. Lets callers keep patching ``hermes_cli.main.`` (the historical test surface) and have those patches reach this code path, and defers the import so ``hermes_cli.main`` -> ``hermes_cli.update_cmd`` stays one-way at import time. """ from hermes_cli import main return main _UPDATE_RUNTIME_RELOAD_MODULES = ( "hermes_constants", "tools.environments.local", "tools.lazy_deps", ) #: Package prefixes whose cached modules become stale the moment the checkout #: changes under this process. Purged (not reloaded) by #: ``_purge_stale_hermes_modules`` so any LATER import chain resolves against #: fresh on-disk source only. _STALE_PURGE_PREFIXES = ( "hermes_cli", "gateway", "tools", "tui_gateway", "agent", ) #: Modules that must survive the purge: they are (or are referenced by) the #: code currently EXECUTING the update, so evicting them buys nothing — the #: running frames keep their module objects alive regardless — and reloading #: them mid-flight is the one genuinely unsafe move. _STALE_PURGE_PROTECTED = frozenset( { "hermes_cli", "hermes_cli.main", "hermes_cli.update_cmd", "hermes_cli.hermes_logging", } ) def _purge_stale_hermes_modules() -> None: """Evict every cached Hermes module after the checkout changed in-place. ``hermes update`` keeps running in the pre-pull Python process. The gateway auto-restart phase that follows does function-level ``from hermes_cli.gateway import ...`` — executing NEW source inside an OLD ``sys.modules`` world. The moment new source references a symbol that was added to an already-cached module, the import dies (2026-08-20 field failure: freshly-pulled ``hermes_cli.gateway`` does ``from hermes_cli.cli_output import line_input``, but ``cli_output`` was cached from before d0132b582 which introduced ``line_input`` → the whole restart phase aborted and the gateway kept serving pre-update code). ``_UPDATE_RUNTIME_RELOAD_MODULES`` handled this per-symptom — three hardcoded module names, re-fixed every time a new module grew a new export. This is the class fix: drop EVERY cached module under the Hermes package prefixes so subsequent lazy imports rebuild a self-consistent, all-new module graph from the updated checkout. Old module objects referenced by the running updater frames stay alive and functional (a purge only removes the ``sys.modules`` cache entry); only genuinely executing modules are exempted, because reloading-in-place — not purging — is the operation that can pull code out from under a running frame. Best-effort: never raises. """ try: import importlib importlib.invalidate_caches() purged = [] for name in list(_m().sys.modules): if name in _STALE_PURGE_PROTECTED: continue if not name.startswith(_STALE_PURGE_PREFIXES): continue root = name.split(".", 1)[0] if root not in _STALE_PURGE_PREFIXES: # Prefix-string match caught an unrelated package # (e.g. ``gateway_foo``) — leave it alone. continue if _m().sys.modules.pop(name, None) is not None: purged.append(name) if purged: logger.debug( "Purged %d stale Hermes module(s) after checkout update", len(purged) ) except Exception as exc: logger.debug("Could not purge stale Hermes modules: %s", exc) def _reload_updated_runtime_modules() -> None: """Reload update-sensitive modules after the checkout changes in-place. ``hermes update`` keeps running in the pre-pull Python process. After a large update, modules already present in ``sys.modules`` can still expose old symbols even though their source files on disk are new. Refresh the small module set used by lazy-backend refresh before that step imports newly-updated code paths. """ try: import importlib importlib.invalidate_caches() for module_name in _UPDATE_RUNTIME_RELOAD_MODULES: module = _m().sys.modules.get(module_name) if module is None: continue try: importlib.reload(module) except Exception as exc: logger.debug("Could not reload updated module %s: %s", module_name, exc) except Exception as exc: logger.debug("Could not refresh update runtime modules: %s", exc) def _reload_config_modules() -> None: """Force-reload modules from disk after git pull. ``hermes update`` runs in the PRE-pull Python process. After ``git pull`` updates the source files on disk, modules already in ``sys.modules`` still hold the OLD code. Function-level imports return the cached module, so ``DEFAULT_CONFIG["_config_version"]`` is the OLD value and ``check_config_version()`` reports ``(33, 33)`` — "up to date" — even though the freshly-pulled code has v34 with a migration to run. This function force-reloads ``hermes_cli.config_defaults``, ``hermes_cli.config``, and ``hermes_cli.config_migrations`` from disk so subsequent imports read the UPDATED code. It also reloads ``hermes_cli._subprocess_compat`` and ``hermes_cli.dashboard_procs`` so that post-update dashboard cleanup (``_finish_dashboard_update_cleanup`` → ``_scan_dashboard_processes``) uses the freshly-pulled code. Without this, a new symbol added to ``_subprocess_compat`` (e.g. ``bounded_probe_run``) is invisible to the cached module object, causing ``ImportError`` during the cleanup step that runs later in the same process. """ import importlib importlib.invalidate_caches() for mod_name in ( "hermes_cli.config_defaults", "hermes_cli.config", "hermes_cli.config_migrations", "hermes_cli._subprocess_compat", "hermes_cli.dashboard_procs", ): mod = sys.modules.get(mod_name) if mod is not None: try: importlib.reload(mod) except Exception as exc: logger.debug("Could not reload %s for fresh post-update code: %s", mod_name, exc) def _run_config_check_fresh() -> tuple: """Check config version using freshly-reloaded modules. See ``_reload_config_modules`` for why this is necessary. Returns ``(current_ver, latest_ver)``. """ _reload_config_modules() from hermes_cli.config import check_config_version return check_config_version() def _run_migrate_config_fresh(*, interactive: bool = False, quiet: bool = False) -> dict: """Run config migration using freshly-reloaded modules. See ``_reload_config_modules`` for why this is necessary. Returns the migration results dict. """ _reload_config_modules() from hermes_cli.config import migrate_config return migrate_config(interactive=interactive, quiet=quiet) # Critical files that Hermes must be able to import immediately after an # update/install. Most are imported on every CLI startup; ``web_server.py`` # is the desktop/dashboard backend path that a fresh Windows install launches # right away. If any of these fail to parse after a pull, the user can be # left with a bricked CLI or desktop backend. The post-pull syntax guard # validates these and auto-rolls-back on failure. _UPDATE_CRITICAL_FILES = ( "hermes_cli/main.py", "hermes_cli/config.py", "hermes_cli/__init__.py", "hermes_cli/web_server.py", "cli.py", "run_agent.py", "model_tools.py", "toolsets.py", "hermes_constants.py", ) def _capture_head_sha(git_cmd, cwd) -> str | None: """Return the current HEAD SHA, or None if it can't be resolved.""" try: result = subprocess.run( git_cmd + ["rev-parse", "HEAD"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True, ) return result.stdout.strip() or None except (subprocess.CalledProcessError, OSError): return None # Files that define the editable install. A pull that touches none of them # cannot have invalidated it. _INSTALL_DEFINING_FILES = ( "pyproject.toml", "setup.py", "setup.cfg", "MANIFEST.in", "uv.lock", ) def _editable_install_is_current(git_cmd, cwd, pre_pull_sha: str | None) -> bool: """True when the pulled commits cannot have invalidated the editable install. ``uv pip install -e .`` never audits an editable target — it reinstalls on every invocation, and every reinstall rewrites the console-script shims. On Windows that rewrite is the only reason the running ``hermes.exe`` has to be quarantined, and a quarantine that loses its race is the whole ``os error 32`` family. Not reinstalling when the reinstall provably cannot change anything removes that risk outright for the common update, rather than trying to make the rename win more often. Skipping is safe because Hermes pins its editable finder to a *static* module list (``[tool.setuptools] py-modules`` plus ``packages.find.include``). The one source-only change that would stale that finder is a new top-level module or package, and it cannot land without a ``pyproject.toml`` diff. Dependencies and ``[project.scripts]`` live there too. New submodules inside an already-mapped package resolve through the real package directory and need no reinstall. Fails closed: an unresolvable pre-pull SHA (shallow checkout, ZIP swap) or a failed ``git diff`` returns False and the install runs as before. """ if not pre_pull_sha: return False try: result = subprocess.run( git_cmd + ["diff", "--name-only", f"{pre_pull_sha}..HEAD", "--"] + list(_INSTALL_DEFINING_FILES), cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) except OSError: return False if result.returncode != 0: return False return not result.stdout.strip() def _validate_critical_files_syntax(root) -> tuple[bool, str | None, str | None]: """Compile each file in ``_UPDATE_CRITICAL_FILES`` to catch SyntaxErrors. These are the files imported on every ``hermes`` startup; if any of them has a syntax error (orphan merge-conflict markers, bad ref to a name that no longer exists, etc.) the CLI can't bootstrap at all. We validate them after a successful ``git pull`` so we can auto-roll-back instead of leaving the user with a bricked install. The compiled ``.pyc`` is written to a temp directory rather than the source tree's ``__pycache__/`` so we don't race with concurrent test workers that walk the same dir, and so we don't leave a stale pyc behind in production if the next interpreter run picks a different Python version. The pyc is discarded on function return either way — we only care about the compile-or-not signal. Returns ``(ok, failing_path, error_message)``. ``ok=True`` means every file parsed cleanly. """ import py_compile import tempfile root = Path(root) with tempfile.TemporaryDirectory(prefix="hermes-syntax-check-") as tmpdir: for relpath in _UPDATE_CRITICAL_FILES: path = root / relpath if not path.exists(): # Missing file is suspicious but not necessarily fatal — a future # refactor may legitimately remove one of these. Skip and move on. continue # Mirror the relative path under the tmpdir so two different # files with the same basename don't collide on the cfile name. cfile = Path(tmpdir) / (relpath.replace("/", "__") + "c") try: py_compile.compile(str(path), cfile=str(cfile), doraise=True) except py_compile.PyCompileError as exc: return False, str(path), str(exc) except OSError as exc: return False, str(path), f"could not read: {exc}" return True, None, None # Modules imported on every agent startup. Unlike _UPDATE_CRITICAL_FILES (which # is only parsed), these are actually *imported* so that cross-module breakage # is caught — a file can be syntactically perfect and still fail to import # because a name it pulls from a sibling module no longer exists. _UPDATE_CRITICAL_MODULES = ( "hermes_cli.main", "run_agent", "model_tools", "toolsets", ) def _validate_critical_modules_import(root) -> tuple[bool, str | None, str | None]: """Import each module in ``_UPDATE_CRITICAL_MODULES`` in a subprocess. ``_validate_critical_files_syntax`` only *parses* files, so it cannot see cross-module breakage: a partially-updated tree where ``agent/`` is new but ``tools/`` is old parses perfectly and still dies at startup with ``ImportError: cannot import name 'TODO_INJECTION_HEADER' from 'tools.todo_tool'``. Every file is valid Python; the *combination* is not. That skew is reachable on the Windows ZIP-update path, whose copy loop walks top-level entries in ``os.listdir`` order and replaces each one independently — ``agent/`` lands long before ``tools/``, so a failure or interruption between them leaves exactly that mismatch on disk. Runs in a subprocess because importing these modules into the running updater would pollute ``sys.modules`` and execute import-time side effects against the half-updated tree. Costs ~0.4s. Uses the project venv's interpreter when there is one (matching ``_venv_core_imports_healthy``): ``hermes update`` can be driven by a different Python than the install's own, and probing the wrong interpreter would test a tree the user never runs. Returns ``(ok, failing_module, error_message)``. """ from hermes_constants import FIRST_PARTY_MODULE_ROOTS probe = ( "import importlib, sys\n" "for name in %r:\n" " try:\n" " importlib.import_module(name)\n" " except ModuleNotFoundError as exc:\n" # A missing *third-party* module means dependencies aren't installed # yet, not a skewed checkout. Only our own packages count as breakage. # The root set is injected from hermes_constants so this can't drift # from the hint the user is shown (they disagreed once already). " missing = (getattr(exc, 'name', '') or '').split('.')[0]\n" " if missing in %r or missing.startswith('hermes_'):\n" " sys.stdout.write(name + '\\n' + str(exc))\n" " raise SystemExit(3)\n" " except ImportError as exc:\n" " sys.stdout.write(name + '\\n' + str(exc))\n" " raise SystemExit(3)\n" " except Exception:\n" " pass\n" # non-import errors (config/env) aren't update breakage "raise SystemExit(0)\n" % (_UPDATE_CRITICAL_MODULES, tuple(sorted(FIRST_PARTY_MODULE_ROOTS))) ) try: interpreter = sys.executable try: venv_python = venv_python_path( Path(root) / "venv", windows=_m()._is_windows() ) if venv_python.exists(): interpreter = str(venv_python) except Exception: pass # fall back to the running interpreter result = subprocess.run( [interpreter, "-c", probe], cwd=str(root), capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=120, ) except (OSError, subprocess.SubprocessError): # Can't run the probe — don't block the update on our own tooling. return True, None, None if result.returncode == 3: parts = (result.stdout or "").split("\n", 1) module = parts[0].strip() or "unknown" detail = parts[1].strip() if len(parts) > 1 else "" return False, module, detail return True, None, None def _gateway_prompt(prompt_text: str, default: str = "", timeout: float = 300.0) -> str: """File-based IPC prompt for gateway mode. Writes a prompt marker file so the gateway can forward the question to the user, then polls for a response file. Falls back to *default* on timeout. Used by ``hermes update --gateway`` so interactive prompts (stash restore, config migration) are forwarded to the messenger instead of being silently skipped. """ import json as _json import uuid as _uuid from hermes_constants import get_hermes_home home = get_hermes_home() prompt_path = home / ".update_prompt.json" response_path = home / ".update_response" # Clean any stale response file response_path.unlink(missing_ok=True) payload = { "prompt": prompt_text, "default": default, "id": str(_uuid.uuid4()), } tmp = prompt_path.with_suffix(".tmp") tmp.write_text(_json.dumps(payload), encoding="utf-8") tmp.replace(prompt_path) # Poll for response deadline = _time.monotonic() + timeout while _time.monotonic() < deadline: if response_path.exists(): try: answer = response_path.read_text(encoding="utf-8").strip() response_path.unlink(missing_ok=True) prompt_path.unlink(missing_ok=True) return answer if answer else default except (OSError, ValueError): pass _time.sleep(0.5) # Timeout — clean up and use default prompt_path.unlink(missing_ok=True) response_path.unlink(missing_ok=True) print(f" (no response after {int(timeout)}s, using default: {default!r})") return default def _npm_bin_exists(bin_dir: Path, name: str) -> bool: """True when an npm bin shim for *name* exists (POSIX or Windows).""" return any( (bin_dir / candidate).exists() for candidate in (name, f"{name}.cmd", f"{name}.ps1", f"{name}.exe") ) def _web_build_toolchain_ready(*roots: Path) -> bool: """True when ``tsc`` and ``vite`` shims are reachable from any of *roots*. Callers must pass every root the build would search; checking only one reports a healthy tree as broken. """ bin_dirs = [ bin_dir for bin_dir in (root / "node_modules" / ".bin" for root in roots) if bin_dir.is_dir() ] return bool(bin_dirs) and all( any(_npm_bin_exists(bin_dir, tool) for bin_dir in bin_dirs) for tool in ("tsc", "vite") ) def _web_toolchain_roots(web_dir: Path) -> tuple[Path, ...]: """Roots whose ``node_modules/.bin`` can satisfy the web build. ``npm run build`` prepends ``node_modules/.bin`` for the package and each of its ancestors, so shims hoisted to the workspace root and shims nested under a package that owns its lockfile (#42973) are equally valid. """ return (web_dir, web_dir.parent) def _print_curator_first_run_notice() -> None: """Print a short heads-up about the skill curator after `hermes update`. Only fires when the curator is enabled AND has no recorded run yet, which is exactly the window where the gateway ticker used to fire Curator against a fresh skill library immediately after an update. We defer the first real pass by one ``interval_hours``; this notice tells the user how to preview or disable before then. Silent on steady state. """ try: from agent import curator except Exception: return try: if not curator.is_enabled(): return state = curator.load_state() except Exception: return if state.get("last_run_at"): # Curator has run before (real or already seeded) — no notice needed. return try: hours = curator.get_interval_hours() except Exception: hours = 24 * 7 days = max(1, hours // 24) print() print("ℹ Skill curator") print( f" Background skill maintenance is enabled. First pass is deferred " f"~{days}d after installation; only agent-created skills are in " f"scope and nothing is ever auto-deleted (archive is recoverable)." ) print(" Preview now: hermes curator run --dry-run") print(" Pause it: hermes curator pause") print( " Docs: https://hermes-agent.nousresearch.com/docs/user-guide/features/curator" ) def _print_fts_optimize_available_notice() -> None: """Advertise the opt-in v23 search-index optimization after `hermes update`. Only fires when the current profile's state.db is still on the legacy (pre-v23) inline FTS layout. Leads with the reclaimable-space figure and points at the exact command. Honors ``sessions.fts_optimize_notice``: ``advise`` (default) prints an advisory notice, ``require`` prints a firmer required-upgrade notice, ``off`` suppresses it. Silent for fresh/already-optimized installs. """ mode = "advise" try: from hermes_cli.config import load_config mode = str( ((load_config() or {}).get("sessions") or {}).get( "fts_optimize_notice", "advise" ) ).strip().lower() except Exception: mode = "advise" if mode == "off": return try: from hermes_constants import get_hermes_home from hermes_state import SessionDB except Exception: return db_path = get_hermes_home() / "state.db" if not db_path.exists(): return try: size_gb = db_path.stat().st_size / (1024 ** 3) except OSError: return # Skip the notice for trivially small DBs — the win isn't worth the nag. if size_gb < 0.5: return db = None interrupted = False try: db = SessionDB(db_path=db_path, read_only=True) # read_only opens skip schema init, so probe the layout directly. row = db._conn.execute( "SELECT sql FROM sqlite_master " "WHERE type = 'table' AND name = 'messages_fts'" ).fetchone() # An interrupted `optimize-storage` run: the table is already the # v23 shape, but backfill markers / demoted trash tables remain. # Offer the command again — re-running resumes and finishes it. interrupted = bool( db._conn.execute( "SELECT 1 FROM state_meta " "WHERE key = 'fts_rebuild_high_water' LIMIT 1" ).fetchone() or db._conn.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' " "AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1" ).fetchone() or db._conn.execute( "SELECT 1 FROM state_meta WHERE key IN " "('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1" ).fetchone() ) except Exception: return finally: if db is not None: try: db.close() except Exception: pass sql = (row[0] if row else "") or "" if not sql or ("tool_name" in sql and not interrupted): # v23 layout already present (fresh/optimized) — nothing to offer. return if interrupted: print() print("◆ Session database optimization incomplete") print( " A previous `hermes sessions optimize-storage` run was " "interrupted. Search still works; re-run the command to resume " "and finish reclaiming disk:" ) print(" hermes sessions optimize-storage") return # Concrete size framing — lead with the savings the user cares about. est_reclaim = size_gb * 0.6 print() if mode == "require": print("◆ Session database upgrade required") print( f" Your search index uses the OLD storage layout and should be " f"upgraded. The new layout typically frees ~60% of state.db " f"(≈{est_reclaim:.1f} GB of your current {size_gb:.1f} GB) and is " f"required for continued optimal operation." ) else: print("◆ Reclaim ~60% of your session database disk") print( f" Your search index uses the old storage layout. Upgrading it " f"typically frees ~60% of state.db — about {est_reclaim:.1f} GB " f"of your current {size_gb:.1f} GB." ) print(" Run when convenient: hermes sessions optimize-storage") print( " It runs in the foreground with a progress bar, is safe to " "interrupt/re-run, and never changes your conversations." ) def _print_curator_recent_run_notice() -> None: """Print the most recent curator run summary, exactly once. The curator runs in the background (gateway tick + CLI session start), so users learn about skill consolidations only by stumbling into a rename. ``hermes update`` is a high-attention surface — surface the most recent run's rename map here, once. Show-once: state stamps ``last_run_summary_shown_at`` after printing. Subsequent ``hermes update`` invocations skip the block until a newer curator run lands. Silent when the curator has never run, when the most recent summary has already been shown, or when the summary has no rename information to display (no archives). """ try: from agent import curator except Exception: return try: state = curator.load_state() except Exception: return last_run_at = state.get("last_run_at") if not last_run_at: return # no curator run yet — first-run notice handles this case if state.get("last_run_summary_shown_at") == last_run_at: return # already shown for this run summary = state.get("last_run_summary") or "" if not summary: return # Only print when there's something interesting to show — i.e. the # rename map block was appended (multi-line summary). A bare "auto: # no changes; llm: no change" doesn't warrant interrupting the # update flow. if "\n" not in summary: # Still stamp it shown so we don't reconsider it on every update. try: state["last_run_summary_shown_at"] = last_run_at curator.save_state(state) except Exception: pass return # Format the timestamp as "Xh ago" for readability. when = _format_time_ago(last_run_at) print() print(f"ℹ Skill curator — last run {when}") for line in summary.splitlines(): print(f" {line}") print( " (This message shows once per curator run. " "View anytime: hermes curator status)" ) # Stamp shown so we don't repeat on the next update. try: state["last_run_summary_shown_at"] = last_run_at curator.save_state(state) except Exception: pass def _format_time_ago(iso_ts: str) -> str: """Render an ISO timestamp as `Xh ago` / `Xd ago` / `Xm ago`. Best effort.""" try: from datetime import datetime, timezone ts = datetime.fromisoformat(iso_ts.replace("Z", "+00:00")) if ts.tzinfo is None: ts = ts.replace(tzinfo=timezone.utc) delta = datetime.now(timezone.utc) - ts secs = int(delta.total_seconds()) if secs < 60: return "just now" if secs < 3600: return f"{secs // 60}m ago" if secs < 86400: return f"{secs // 3600}h ago" return f"{secs // 86400}d ago" except Exception: return "recently" def _reload_process_scan_modules() -> None: """Force-reload the process-scan modules from disk after an update. ``_finish_dashboard_update_cleanup`` runs in the PRE-update Python process, but ``_scan_dashboard_processes`` does a function-level ``from hermes_cli._subprocess_compat import bounded_probe_run``. If the update added a new symbol to ``_subprocess_compat`` (as #87134 did with ``bounded_probe_run``), the cached OLD module object doesn't have it and the cleanup step crashes with ImportError — after the code update itself already succeeded. Reload dependency-first so ``dashboard_procs`` binds against the fresh ``_subprocess_compat``. Lives here (called from the cleanup entry point) rather than only in ``_reload_config_modules`` so EVERY caller — the git-update path, the Windows ZIP fallback path, and any future one — is covered. """ import importlib importlib.invalidate_caches() for mod_name in ( "hermes_cli._subprocess_compat", "hermes_cli.dashboard_procs", ): mod = sys.modules.get(mod_name) if mod is not None: try: importlib.reload(mod) except Exception as exc: # warning, not debug: a failed reload here surfaces seconds # later as an ImportError in the same process — leave a trail. logger.warning( "Could not reload %s for post-update cleanup: %s", mod_name, exc, ) def _finish_dashboard_update_cleanup( node_failures: list[str], already_restarted_units: "set[str] | None" = None ) -> None: """Refresh managed dashboards or stop stale manual ones after an update. *already_restarted_units* forwards the systemd unit names (no ``.service`` suffix) that the fleet-restart loop already restarted directly, so a Serve-only install's freshly restarted process isn't found and restarted a second time here (review on #83595). """ if node_failures: print() print(" ℹ Leaving running dashboard process(es) untouched because the") print(" Node.js dependency refresh did not complete.") return # The scan path lazy-imports symbols from _subprocess_compat; make sure # both modules reflect the freshly-updated source before touching them. _reload_process_scan_modules() stop_result = _m()._kill_stale_dashboard_processes( restart_managed=True, already_restarted_units=already_restarted_units ) if not stop_result.get("unrecovered"): return print() print( "⚠ A web dashboard/serve process was stopped during update and could " "not be auto-restarted." ) print(" Re-launch it when you want the web UI back:") print(" hermes dashboard --port ") def _atomic_replace_dir(src: str, dst: str) -> None: """Replace directory *dst* with *src* without leaving *dst* half-deleted. The naive ``rmtree(dst); copytree(src, dst)`` has a destructive window: if the copy fails partway (common on the Windows ZIP-update path, which only runs because file I/O is already flaky on that machine), the old directory is already gone and nothing replaced it — the install is left with a deleted tree (issue #49145, where ``ui-tui/`` vanished and broke the TUI). Now a thin single-entry alias over the two-phase helpers below, which generalise the same stage-then-swap discipline across every entry the ZIP update touches (#76104). Retained because it is part of the mechanical ``hermes_cli.main`` re-export surface and guards the #49145 regression. """ _commit_staged_replacements([(_stage_replacement(src, dst), dst)]) def _stage_replacement(src: str, dst: str) -> str: """Copy *src* to a sibling staging path for *dst*; return the staging path. Phase 1 of the two-phase replace. Handles both directories and plain files. Touches nothing live, so a failure here leaves the whole install untouched. """ staging = f"{dst}.hermes-update-staging" backup = f"{dst}.hermes-update-old" # A previous run may have died between "move dst aside" and "move staging # in" — leaving dst missing and the backup as the ONLY copy of that entry. # Restore it before clearing leftovers: deleting the backup first and then # failing to stage (disk exhaustion is likely right after writing a full # staging copy) would leave a hole in the install with nothing to roll # back to. The restore is a same-filesystem rename — instant and safe. if not os.path.exists(dst) and os.path.exists(backup): os.rename(backup, dst) for leftover in (staging, backup): if os.path.isdir(leftover): shutil.rmtree(leftover, ignore_errors=True) elif os.path.exists(leftover): os.remove(leftover) if os.path.isdir(src): shutil.copytree(src, staging) else: shutil.copy2(src, staging) return staging def _discard_staged(staged) -> None: """Remove staging paths for entries that were never committed. Without this a phase-1 failure (typically disk exhaustion) orphans one staging copy per entry already processed — up to a full second copy of the tree. The user then follows the "re-run `hermes update`" advice with *less* free space than before and the retry fails harder than the original attempt. """ for staging, _dst in staged: try: if os.path.isdir(staging): shutil.rmtree(staging, ignore_errors=True) elif os.path.exists(staging): os.remove(staging) except OSError as exc: # best-effort cleanup, never fatal logger.warning("could not remove staging path %s: %s", staging, exc) def _commit_staged_replacements(staged) -> None: """Phase 2: swap every staged entry into place, rolling back all on failure. ``_atomic_replace_dir`` makes each *individual* directory swap safe, but the ZIP update replaces ~90 top-level entries in a loop, and nothing made the loop atomic *as a whole*. A failure partway left some entries at the new version and the rest at the old one — every file valid Python, the combination unbootable (issue #76104; the ``ImportError`` in #76091 and the field report in #63717 are both this). This covers plain files as well as directories: the repo root holds 20 first-party modules (``run_agent.py``, ``cli.py``, ``hermes_constants.py`` …), so a files-only failure reproduces exactly the bug class we are closing. Every swap is an ``os.rename`` onto a path that was just moved aside — a same-filesystem rename is atomic on POSIX and NTFS alike, so a file swap can never leave a half-written module the way ``copy2`` onto a live path can. Splitting stage-all-then-swap-all shrinks the failure window from "the duration of a full tree copy" to "the duration of N renames", and makes the remaining window recoverable: if a swap fails we restore every entry already swapped, so the tree lands wholly new or wholly old. """ swapped: list[tuple[str, str]] = [] # (dst, backup) in swap order; "" = absent try: for staging, dst in staged: backup = f"{dst}.hermes-update-old" if os.path.exists(dst): os.rename(dst, backup) swapped.append((dst, backup)) else: swapped.append((dst, "")) os.rename(staging, dst) except OSError: # Undo every swap already made so the install stays self-consistent. for dst, backup in reversed(swapped): try: if os.path.isdir(dst): shutil.rmtree(dst, ignore_errors=True) elif os.path.exists(dst): os.remove(dst) if backup and os.path.exists(backup): os.rename(backup, dst) except OSError as exc: # Keep restoring the rest — a silent failure here is the one # thing that turns a recoverable rollback into a mixed tree, # so say so rather than swallowing it. logger.warning("rollback failed for %s: %s", dst, exc) raise # All swaps succeeded — drop the backups (best-effort, never fatal). for _dst, backup in swapped: if backup and os.path.isdir(backup): shutil.rmtree(backup, ignore_errors=True) elif backup and os.path.exists(backup): try: os.remove(backup) except OSError: pass def _branch_head_label(git_cmd=None, cwd=None) -> str | None: """``" @ "`` for the checkout, or None when unknown. Appended to the update summary lines so branch drift is visible at a glance (live incident 2026-08-17: a checkout parked on a stale feature branch got "✓ Update complete!" with nothing on the line saying WHERE the checkout actually sat). Never raises — summary decoration must not break an update. """ try: cmd = list(git_cmd) if git_cmd else ["git"] root = cwd if cwd is not None else _m().PROJECT_ROOT branch = subprocess.run( cmd + ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root, capture_output=True, text=True, encoding="utf-8", errors="replace", ) sha = subprocess.run( cmd + ["rev-parse", "--short", "HEAD"], cwd=root, capture_output=True, text=True, encoding="utf-8", errors="replace", ) branch_name = branch.stdout.strip() sha_text = sha.stdout.strip() if branch.returncode != 0 or sha.returncode != 0 or not sha_text: return None if not branch_name: return None label = "detached" if branch_name == "HEAD" else branch_name return f"{label} @ {sha_text}" except Exception: return None def _branch_head_suffix(git_cmd=None, cwd=None) -> str: """`` [ @ ]`` suffix for summary lines ("" when unknown).""" label = _branch_head_label(git_cmd, cwd) return f" [{label}]" if label else "" def _assess_parked_branch_switch( git_cmd: list[str], cwd: Path, current_branch: str, target_branch: str ) -> tuple[bool, str]: """Decide whether it is safe to auto-switch a parked feature branch back to the update target. Live incident (2026-08-17, Teknium's box): the source checkout sat on a stale feature branch left behind by earlier tooling; ``hermes update`` autostashed, ran its post-update steps and printed "✓ Code updated!" while the running code stayed days behind main. The guard's contract: - safe (True, "") only when the working tree + index are clean AND every commit on the parked branch is already contained in ``origin/`` (``git cherry`` reports no ``+`` lines). - anything else — dirty tree, unmerged commits, git errors, or the ``updates.auto_switch_parked_branch: false`` config opt-out — returns (False, ) and the caller must NOT touch the branch. Reasons: "disabled", "dirty", "unmerged:", "unverifiable". """ try: from hermes_cli.config import load_config _update_cfg = (load_config() or {}).get("updates", {}) if isinstance(_update_cfg, dict) and not bool( _update_cfg.get("auto_switch_parked_branch", True) ): return False, "disabled" except Exception as exc: # A config read failure must not disable the guard's safety checks — # fall through to them with the default (auto-switch allowed). logger.debug("Could not read updates.auto_switch_parked_branch: %s", exc) status = subprocess.run( git_cmd + ["status", "--porcelain"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if status.returncode != 0: return False, "unverifiable" if status.stdout.strip(): return False, "dirty" cherry = subprocess.run( git_cmd + ["cherry", f"origin/{target_branch}"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if cherry.returncode != 0: return False, "unverifiable" unmerged = [ line for line in cherry.stdout.splitlines() if line.startswith("+") ] if unmerged: return False, f"unmerged:{len(unmerged)}" return True, "" def _print_parked_branch_skip_warning( git_cmd: list[str], cwd: Path, current_branch: str, target_branch: str, reason: str, ) -> None: """LOUD block explaining why the code update was skipped on a parked branch, with the behind-count and the exact commands to resolve.""" behind = None try: behind_result = subprocess.run( git_cmd + ["rev-list", f"HEAD..origin/{target_branch}", "--count"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if behind_result.returncode == 0 and behind_result.stdout.strip(): behind = int(behind_result.stdout.strip()) except Exception: behind = None if reason == "dirty": why = "the working tree has uncommitted changes" elif reason.startswith("unmerged:"): count = reason.split(":", 1)[1] why = ( f"the branch has {count} commit(s) not merged into " f"origin/{target_branch}" ) elif reason == "disabled": why = "updates.auto_switch_parked_branch is set to false in config.yaml" else: why = ( f"the branch state could not be verified against " f"origin/{target_branch}" ) bar = "=" * 68 print() print(bar) print(f"⚠ CODE UPDATE SKIPPED — checkout is parked on '{current_branch}'") print(f" Not auto-switching to {target_branch}: {why}.") if behind is not None and behind > 0: print( f" This checkout is {behind} commit(s) BEHIND " f"origin/{target_branch} — the code you are running is stale." ) print() print(" To resolve, inspect the branch and switch back yourself:") print(f" git -C {cwd} status") print(f" git -C {cwd} checkout {target_branch} && hermes update") print( " (commit or stash your work on the branch first if you want to " "keep it)" ) print(bar) def _print_update_completion(message: str) -> None: """Print an update outcome plus, when the dashboard launched this run with an action id, a terminal receipt line the Desktop can match after the dashboard restarts (see #47359 / #58764). The outcome line carries the checkout's actual branch + HEAD short-sha so branch drift is visible at a glance (2026-08-17 parked-branch incident).""" print(f"{message}{_branch_head_suffix()}") action_id = os.environ.get("HERMES_ACTION_ID", "") if len(action_id) == 32 and all(char in "0123456789abcdef" for char in action_id): print(f"=== hermes-update completed {action_id} ===") def _read_project_version() -> str | None: """Read the ``version`` field from the checkout's pyproject.toml. Reads the on-disk file (not importlib.metadata) because after a git pull the installed distribution metadata still describes the OLD version; the file is the only source that reflects what was just pulled. Returns None on any failure — version reporting is cosmetic and must never break an update. """ try: import tomllib with open(_m().PROJECT_ROOT / "pyproject.toml", "rb") as fh: # windows-footgun: ok — binary mode, tomllib requires bytes version = tomllib.load(fh).get("project", {}).get("version") return str(version) if version else None except Exception: return None def _update_complete_message(pre_version: str | None) -> str: """Completion line with the version transition when it is known. Ported from PrimeIntellect-ai/prime-agent#630: after a successful self-update, show both versions (``v0.19.4 → v0.20.0``) so the user can see what they actually got. Falls back to the plain message when either side is unknown or the version did not change (e.g. several commits landed within one release). """ post_version = _read_project_version() if pre_version and post_version and pre_version != post_version: return f"✓ Update complete! (v{pre_version} → v{post_version})" if post_version: return f"✓ Update complete! (v{post_version})" return "✓ Update complete!" def _print_update_summary( *, node_failures: list, desktop_build_ok: bool, pre_update_version: str | None, ) -> None: """Final update banner. A failed Desktop rebuild is non-fatal for the Python side, but must not print ``✓ Update complete!`` (#88251).""" print() if node_failures or not desktop_build_ok: parts = [] if node_failures: parts.append( f"Node.js dependencies for {', '.join(node_failures)} did not refresh" ) if not desktop_build_ok: parts.append( "the desktop app was not rebuilt and is still on the previous build" ) print("⚠ Update partially complete — " + "; ".join(parts) + ".") if node_failures: print(" Code and Python deps are updated, but the dashboard/TUI may") print(" be in a mixed state until the Node deps are rebuilt.") if not desktop_build_ok: print(" Run `hermes desktop` to retry the desktop rebuild.") else: _print_update_completion(_update_complete_message(pre_update_version)) def _write_gateway_update_exit_code(ok: bool) -> None: path = get_hermes_home() / ".update_exit_code" try: path.write_text("0" if ok else "1", encoding="utf-8") except OSError: pass def _update_via_zip(args, *, had_desktop_app_before_update: bool = False) -> bool: """Update Hermes Agent by downloading a ZIP archive. Used on Windows when git file I/O is broken (antivirus, NTFS filter drivers causing 'Invalid argument' errors on file creation). Returns ``False`` when a Desktop rebuild ran and failed; ``True`` otherwise. """ active_tool_dependencies = _m()._capture_active_tool_dependencies() import tempfile import zipfile from urllib.request import urlretrieve # Snapshot the pre-update version before files are replaced so the # completion line can report the transition (prime-agent#630 port). pre_update_version = _read_project_version() # The ZIP fallback exists for Windows git-file-I/O breakage. It pulls a # static archive from GitHub, which is fine for the default "main" # channel but would silently ignore --branch and update from main even # if the user asked for something else — exactly the silent-divergence # bug --branch was added to prevent. Refuse to proceed in that case # rather than lie. branch = _m()._resolve_update_branch(args) if branch != "main": print( f"✗ --branch={branch} is not supported on the Windows ZIP-fallback " "update path." ) print( " This path runs when git file I/O is broken on the system. " "Either resolve the git-side breakage (typically an antivirus " "or NTFS filter holding files open) and rerun `hermes update " f"--branch {branch}`, or update against main with `hermes update`." ) _m().sys.exit(1) zip_url = ( f"https://github.com/NousResearch/hermes-agent/archive/refs/heads/{branch}.zip" ) print("→ Downloading latest version...") tmp_dir = tempfile.mkdtemp(prefix="hermes-update-") try: zip_path = os.path.join(tmp_dir, f"hermes-agent-{branch}.zip") urlretrieve(zip_url, zip_path) print("→ Extracting...") import stat as _stat with zipfile.ZipFile(zip_path, "r") as zf: # Validate paths to prevent zip-slip (path traversal) AND reject # symlink members. A GitHub source ZIP for hermes-agent itself # should never contain symlinks — they'd point outside the # extracted tree and let an attacker who can compromise the # update mirror plant arbitrary files via the update path. tmp_dir_real = os.path.realpath(tmp_dir) for member in zf.infolist(): member_path = os.path.realpath(os.path.join(tmp_dir, member.filename)) if ( not member_path.startswith(tmp_dir_real + os.sep) and member_path != tmp_dir_real ): raise ValueError( f"Zip-slip detected: {member.filename} escapes extraction directory" ) # Unix mode lives in the upper 16 bits of external_attr; # mask to the file-type bits. mode = (member.external_attr >> 16) & 0o170000 if _stat.S_ISLNK(mode): raise ValueError( f"ZIP contains unsupported symlink member: {member.filename}" ) zf.extractall(tmp_dir) # GitHub ZIPs extract to hermes-agent-/ extracted = os.path.join(tmp_dir, f"hermes-agent-{branch}") if not os.path.isdir(extracted): # Try to find it for d in os.listdir(tmp_dir): candidate = os.path.join(tmp_dir, d) if os.path.isdir(candidate) and d != "__MACOSX": extracted = candidate break # Copy updated files over existing installation, preserving venv/node_modules/.git preserve = {"venv", "node_modules", ".git", ".env"} entries = [i for i in os.listdir(extracted) if i not in preserve] # Two-phase replace (#76104). Phase 1 copies every entry — directories # AND top-level files — to a sibling staging path without touching # anything live; phase 2 swaps them all in with same-filesystem # renames and rolls back every swap if any one fails. Replacing # entries one-at-a-time (the previous shape) meant an interruption # partway left `agent/` new and `tools/` stale — all files valid, the # tree unbootable. Files matter as much as directories here: the repo # root holds 20 first-party modules (run_agent.py, cli.py, # hermes_constants.py, ...). # # Staging costs one extra copy of the tree on disk. Check up front so # we fail with a clear message instead of running out mid-copy. need = sum( os.path.getsize(os.path.join(dirpath, f)) for entry in entries for dirpath, _dirs, files in os.walk(os.path.join(extracted, entry)) for f in files ) + sum( os.path.getsize(os.path.join(extracted, e)) for e in entries if os.path.isfile(os.path.join(extracted, e)) ) # Only the staging copy is new — the live tree already occupies its # space and the swaps are renames, not copies. Ask for the staging # copy plus 20% headroom rather than a full 2x, which would block # updates that would have succeeded on exactly the space-constrained # machines most likely to hit this path. required = int(need * 1.2) free = shutil.disk_usage(str(_m().PROJECT_ROOT)).free if free < required: raise RuntimeError( f"not enough free disk space to stage the update safely " f"(need ~{required // (1024 * 1024)} MB, have " f"{free // (1024 * 1024)} MB)" ) staged: list[tuple[str, str]] = [] try: for item in entries: src = os.path.join(extracted, item) dst = os.path.join(str(_m().PROJECT_ROOT), item) staged.append((_stage_replacement(src, dst), dst)) except Exception: # Nothing is live yet; drop the partial staging copies so a retry # starts from the same free space this attempt did. _discard_staged(staged) raise try: _commit_staged_replacements(staged) except Exception: # The rollback already restored every swapped entry, but staging # copies for the not-yet-swapped entries (potentially most of a # full tree) are still on disk. Drop them, or the retry's # up-front free-space check — which runs BEFORE the lazy # per-entry leftover cleanup — fails on litter this attempt # left behind: the exact "retry fails harder" failure mode # _discard_staged exists to prevent. Safe post-rollback: swapped # entries' staging paths were renamed away, and _discard_staged # skips paths that no longer exist. _discard_staged(staged) raise update_count = len(staged) print(f"✓ Updated {update_count} items from ZIP") except Exception as e: print(f"✗ ZIP update failed: {e}") # The two-phase replace either commits every entry or rolls them all # back, so a failure here does not leave a mixed-version tree — don't # scare the user toward a reinstall they don't need. print(" Your existing install was left in place.") print( " Re-run `hermes update` to retry; if the agent won't start, " "reinstall from https://hermes-agent.nousresearch.com" ) _m().sys.exit(1) finally: shutil.rmtree(tmp_dir, ignore_errors=True) # Clear stale bytecode after ZIP extraction removed = _m()._clear_bytecode_cache(_m().PROJECT_ROOT) if removed: print( f" ✓ Cleared {removed} stale __pycache__ director{'y' if removed == 1 else 'ies'}" ) _m()._record_bytecode_fingerprint() _m()._refresh_bootstrap_cache_scripts(branch) # Reinstall Python dependencies. Prefer .[all], but if one optional extra # breaks on this machine, keep base deps and reinstall the remaining extras # individually so update does not silently strip working capabilities. # # Self-lock deferral (relocated preflight — #86735): the ZIP code swap # above is already committed; defer only the dependency sync when this # process holds a native extension the sync must rewrite. _m()._abort_dependency_sync_if_self_locked() print("→ Updating Python dependencies...") from hermes_cli.managed_uv import ensure_uv, update_managed_uv # Keep managed uv current — runs `uv self update` if we already have one. update_managed_uv() uv_bin = ensure_uv() pip_cmd = [_m().sys.executable, "-m", "pip"] if not uv_bin: uv_bin = _ensure_uv_for_termux(pip_cmd) if uv_bin: uv_env = {**os.environ, "VIRTUAL_ENV": str(_m().PROJECT_ROOT / "venv")} if _m()._is_termux_env(uv_env): uv_env.pop("PYTHONPATH", None) uv_env.pop("PYTHONHOME", None) _m()._install_python_dependencies_with_optional_fallback([uv_bin, "pip"], env=uv_env) else: # Use sys.executable to explicitly call the venv's pip module, # avoiding PEP 668 'externally-managed-environment' errors on Debian/Ubuntu. # Some environments lose pip inside the venv; bootstrap it back with # ensurepip before trying the editable install. try: subprocess.run( pip_cmd + ["--version"], cwd=_m().PROJECT_ROOT, check=True, capture_output=True, ) except subprocess.CalledProcessError: subprocess.run( [_m().sys.executable, "-m", "ensurepip", "--upgrade", "--default-pip"], cwd=_m().PROJECT_ROOT, check=True, ) _m()._install_python_dependencies_with_optional_fallback(pip_cmd) install_prefix = [uv_bin, "pip"] if uv_bin else pip_cmd install_env = uv_env if uv_bin else None _m()._restore_active_tool_dependencies( active_tool_dependencies, install_prefix, env=install_env, ) # ZIP path parity: heal the active memory provider's bridge packages # after the dependency reinstall, same as the git-pull path (#53272, # #70636). _m()._refresh_active_memory_provider_dependencies() # Now that dependencies are installed, verify the tree actually imports. # The copy loop above replaces top-level entries one at a time in # os.listdir order, so an interruption between (say) `agent/` and `tools/` # leaves a tree whose files all parse but cannot be imported together — # the ImportError-on-startup class this guard exists to catch. Deliberately # placed *after* the dependency reinstall so a genuinely-new third-party # requirement isn't misreported as a partial copy. There is no SHA to roll # back to here, so surface it with a concrete recovery step rather than # reporting a successful update over a bricked install. import_ok, failing_module, import_error = _validate_critical_modules_import( _m().PROJECT_ROOT ) if not import_ok: print() print("✗ Update left the install in an unimportable state:") print(f" {failing_module}: {import_error}") print() print(" This usually means the copy was interrupted partway through.") print(" Re-run `hermes update` to complete it.") _m().sys.exit(1) node_failures = _update_node_dependencies() _m()._build_web_ui(_m().PROJECT_ROOT / "web") desktop_build_ok = _rebuild_desktop_after_update( _m().PROJECT_ROOT / "apps" / "desktop", had_desktop_app_before_update=had_desktop_app_before_update, ) # Sync skills try: from tools.skills_sync import sync_skills print("→ Syncing bundled skills...") result = sync_skills(quiet=True) if result["copied"]: print(f" + {len(result['copied'])} new: {', '.join(result['copied'])}") if result.get("updated"): print( f" ↑ {len(result['updated'])} updated: {', '.join(result['updated'])}" ) if result.get("user_modified"): print(f" ~ {len(result['user_modified'])} user-modified (kept)") print( " → see them: hermes skills list-modified " "(diff/reset to resume updates)" ) if result.get("cleaned"): print(f" − {len(result['cleaned'])} removed from manifest") if result.get("relocated"): print( f" → {len(result['relocated'])} moved to new upstream paths: " f"{', '.join(result['relocated'])}" ) if not result["copied"] and not result.get("updated"): print(" ✓ Skills are up to date") except Exception: pass # Seed the model-catalog disk cache from the freshly-unpacked checkout # (same rationale as the git-pull path in _cmd_update_impl). Non-fatal. try: from hermes_cli.model_catalog import seed_cache_from_checkout if seed_cache_from_checkout(_m().PROJECT_ROOT): print(" ✓ Model catalog cache refreshed from checkout") except Exception as e: logger.debug("Model catalog seed during zip update failed: %s", e) # ── Post-update state.db integrity guard (#68474) ───────────────── # Same as the git-pull path: verify state.db survived the ZIP update # and auto-restore from the most recent pre-update snapshot if needed. try: from hermes_cli.backup import _quick_snapshot_root, verify_sqlite_integrity _state_path = get_hermes_home() / "state.db" if _state_path.exists(): _state_ok = verify_sqlite_integrity( _state_path, check_header=True, run_pragma=True ) if not _state_ok.get("valid"): print() print( "⚠ state.db is corrupted after update: " + _state_ok.get("message", "unknown error") ) _snap_root = _quick_snapshot_root(get_hermes_home()) if _snap_root.exists(): _snap_dirs = sorted( (d for d in _snap_root.iterdir() if d.is_dir()), reverse=True, ) for _snap_dir in _snap_dirs: _snap_state = _snap_dir / "state.db" if _snap_state.exists(): _snap_ok = verify_sqlite_integrity( _snap_state, check_header=True, run_pragma=True ) if _snap_ok.get("valid"): try: import shutil as _shutil _shutil.copy2(_snap_state, _state_path) _restored_ok = verify_sqlite_integrity( _state_path, check_header=True, run_pragma=True, ) if _restored_ok.get("valid"): print( " ✓ Auto-restored from snapshot " f"{_snap_dir.name}" ) else: print( " ✗ Auto-restore FAILED — restored " "copy also failed integrity" ) break except OSError as _exc: print( f" ✗ Auto-restore file copy failed: {_exc}" ) break except Exception as exc: logger.debug( "Post-update state.db integrity check (zip path) failed: %s", exc ) _print_update_summary( node_failures=node_failures, desktop_build_ok=desktop_build_ok, pre_update_version=pre_update_version, ) try: _print_curator_first_run_notice() except Exception as e: logger.debug("Curator first-run notice failed: %s", e) try: _print_curator_recent_run_notice() except Exception as e: logger.debug("Curator recent-run notice failed: %s", e) # Don't stop a working dashboard when the Node refresh failed — see the # git-update path for rationale (#30271). _finish_dashboard_update_cleanup(node_failures) try: from hermes_cli.update_receipt import finalize_update_receipt finalize_update_receipt( "success" if (desktop_build_ok and not node_failures) else "partial" ) except Exception as _receipt_exc: logger.debug("Update receipt finalize (zip path) failed: %s", _receipt_exc) return desktop_build_ok def _stash_local_changes_if_needed(git_cmd: list[str], cwd: Path) -> Optional[str]: status = subprocess.run( git_cmd + ["status", "--porcelain"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True, ) if not status.stdout.strip(): return None # If the index has unmerged entries (e.g. from an interrupted merge/rebase), # git stash will fail with "needs merge / could not write index". Clear the # conflict state with `git reset` so the stash can proceed. Working-tree # changes are preserved; only the index conflict markers are dropped. unmerged = subprocess.run( git_cmd + ["ls-files", "--unmerged"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if unmerged.stdout.strip(): print("→ Clearing unmerged index entries from a previous conflict...") subprocess.run(git_cmd + ["reset"], cwd=cwd, capture_output=True) from datetime import datetime, timezone stash_name = datetime.now(timezone.utc).strftime( "hermes-update-autostash-%Y%m%d-%H%M%S" ) print("→ Local changes detected — stashing before update...") prev_stash = subprocess.run( git_cmd + ["rev-parse", "--verify", "refs/stash"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ).stdout.strip() push = subprocess.run( git_cmd + ["stash", "push", "--include-untracked", "-m", stash_name], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if push.stdout.strip(): print(push.stdout.strip()) stash_probe = subprocess.run( git_cmd + ["rev-parse", "--verify", "refs/stash"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) stash_ref = stash_probe.stdout.strip() stash_created = ( stash_probe.returncode == 0 and bool(stash_ref) and stash_ref != prev_stash ) if push.returncode != 0: if stash_created: # git stash push exits non-zero when it saved everything but could # not delete some swept untracked files from the working tree # (e.g. a root-owned directory: "warning: failed to remove ...: # Permission denied"). The stash entry is complete — the changes # are safe — so this is not a failure. Leave the undeletable # files in place and continue the update. if push.stderr.strip(): print(push.stderr.strip()) print( " ⚠ Some untracked files could not be removed from the " "working tree (permission denied)." ) print( " They were still saved to the stash and were left in " "place — the update will continue." ) # A partially-failed stash push also aborts its working-tree # cleanup for TRACKED modifications — they are saved in the stash # but still dirty the tree, which would break the checkout/pull # that follows. Safe to reset: everything is in the stash entry. subprocess.run( git_cmd + ["reset", "--hard", "HEAD"], cwd=cwd, capture_output=True, ) else: # No stash entry was created: the changes were NOT saved. This # is a real failure — bail out before the update touches HEAD. print("✗ Could not stash local changes — update aborted.") if push.stderr.strip(): print(f" {push.stderr.strip().splitlines()[0]}") print( " Commit, stash, or clean up your local changes manually, " "then re-run `hermes update`." ) raise subprocess.CalledProcessError( push.returncode, push.args, output=push.stdout, stderr=push.stderr ) return stash_ref def _resolve_stash_selector( git_cmd: list[str], cwd: Path, stash_ref: str ) -> Optional[str]: stash_list = subprocess.run( git_cmd + ["stash", "list", "--format=%gd %H"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True, ) for line in stash_list.stdout.splitlines(): selector, _, commit = line.partition(" ") if commit.strip() == stash_ref: return selector.strip() return None def _print_stash_cleanup_guidance( stash_ref: str, stash_selector: Optional[str] = None ) -> None: print( " Check `git status` first so you don't accidentally reapply the same change twice." ) print(" Find the saved entry with: git stash list --format='%gd %H %s'") if stash_selector: print(f" Remove it with: git stash drop {stash_selector}") else: print( f" Look for commit {stash_ref}, then drop its selector with: git stash drop stash@{{N}}" ) def _stash_apply_failed_only_on_existing_untracked(stderr: str) -> bool: """True when a ``git stash apply`` failure is ONLY about untracked files that already exist in the working tree. This is the tail end of the permission-denied autostash class: ``git stash push --include-untracked`` swept undeletable files (e.g. a root-owned ``packaging/`` directory) into the stash but could not remove them from disk. On restore, git applies all tracked changes, then refuses to overwrite those still-present files (``already exists, no checkout`` / ``could not restore untracked files from stash``) and exits non-zero even though nothing was lost. Any other error line (e.g. ``would be overwritten by merge`` / ``Aborting``) means the tracked apply itself failed and this returns False. """ lines = [ln.strip() for ln in (stderr or "").splitlines() if ln.strip()] if not lines: return False saw_untracked_error = False for ln in lines: if "already exists, no checkout" in ln: saw_untracked_error = True elif "could not restore untracked files from stash" in ln: saw_untracked_error = True elif ln.startswith(("warning:", "hint:")): continue else: return False return saw_untracked_error def _park_stashed_changes(stash_ref: str) -> None: """Leave a pre-update autostash parked instead of re-applying it. Used by ``hermes update --keep-stash`` (the desktop updater's mode): the stash made the update possible on a dirty tree, but local source edits must never be silently re-applied onto the updated code. Nothing is lost — the entry stays in ``git stash`` with printed recovery guidance. """ print() print("ℹ️ Local changes were stashed before updating and were NOT re-applied (--keep-stash).") print(f" Stash ref: {stash_ref}") print(f" Restore manually with: git stash apply {stash_ref}") def _restore_stashed_changes( git_cmd: list[str], cwd: Path, stash_ref: str, prompt_user: bool = False, input_fn=None, ) -> bool: if prompt_user: print() print("⚠ Local changes were stashed before updating.") print( " Restoring them may reapply local customizations onto the updated codebase." ) print(" Review the result afterward if Hermes behaves unexpectedly.") print("Restore local changes now? [Y/n]") if input_fn is not None: response = input_fn("Restore local changes now? [Y/n]", "y") else: try: response = input().strip().lower() except (EOFError, UnicodeDecodeError): # Mirror the config-migration prompt's fix: don't let a # terminal-encoding issue or a closed stdin crash the # update mid-restore. Falls through to the existing # skip-restore path below, which already explains how to # restore manually from git stash. response = "n" if response not in {"", "y", "yes"}: print("Skipped restoring local changes.") print("Your changes are still preserved in git stash.") print(f"Restore manually with: git stash apply {stash_ref}") return False print("→ Restoring local changes...") restore = subprocess.run( git_cmd + ["stash", "apply", stash_ref], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) # Check for unmerged (conflicted) files — can happen even when returncode is 0 unmerged = subprocess.run( git_cmd + ["diff", "--name-only", "--diff-filter=U"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) has_conflicts = bool(unmerged.stdout.strip()) if restore.returncode != 0 and not has_conflicts and ( _stash_apply_failed_only_on_existing_untracked(restore.stderr) ): # Permission-denied autostash tail end: the tracked changes applied # cleanly; the only "failure" is untracked files that never left the # working tree (git could not delete them at stash time, so it now # refuses to overwrite them). Their content was never touched — # nothing is lost. Treat as restored. print( " ⚠ Some stashed untracked files already exist in the working " "tree and were kept as-is." ) elif restore.returncode != 0 or has_conflicts: print("✗ Update pulled new code, but restoring local changes hit conflicts.") if restore.stdout.strip(): print(restore.stdout.strip()) if restore.stderr.strip(): print(restore.stderr.strip()) # Show which files conflicted conflicted_files = unmerged.stdout.strip() if conflicted_files: print("\nConflicted files:") for f in conflicted_files.splitlines(): print(f" • {f}") print("\nYour stashed changes are preserved — nothing is lost.") print(f" Stash ref: {stash_ref}") # Always reset to clean state — leaving conflict markers in source # files makes hermes completely unrunnable (SyntaxError on import). # The user's changes are safe in the stash for manual recovery. subprocess.run( git_cmd + ["reset", "--hard", "HEAD"], cwd=cwd, capture_output=True, ) print("Working tree reset to clean state.") print(f"Restore your changes later with: git stash apply {stash_ref}") # Don't sys.exit — the code update itself succeeded, only the stash # restore had conflicts. Let cmd_update continue with pip install, # skill sync, and gateway restart. return False stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) if stash_selector is None: print( "⚠ Local changes were restored, but Hermes couldn't find the stash entry to drop." ) print( " The stash was left in place. You can remove it manually after checking the result." ) _print_stash_cleanup_guidance(stash_ref) else: drop = subprocess.run( git_cmd + ["stash", "drop", stash_selector], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if drop.returncode != 0: print( "⚠ Local changes were restored, but Hermes couldn't drop the saved stash entry." ) if drop.stdout.strip(): print(drop.stdout.strip()) if drop.stderr.strip(): print(drop.stderr.strip()) print( " The stash was left in place. You can remove it manually after checking the result." ) _print_stash_cleanup_guidance(stash_ref, stash_selector) print("⚠ Local changes were restored on top of the updated codebase.") print(" Review `git diff` / `git status` if Hermes behaves unexpectedly.") return True def _discard_stashed_changes( git_cmd: list[str], cwd: Path, stash_ref: str, ) -> bool: """Throw away a stash created before an update, without applying it. Used only on a NON-interactive update when the user has set ``updates.non_interactive_local_changes: discard`` — i.e. they've opted out of keeping local source edits on this machine. Drops the stash entry instead of re-applying it, so the working tree stays clean at the freshly pulled HEAD. Unlike ``git reset --hard`` + ``git clean -fd``, this only affects what was stashed (tracked changes + the untracked files we explicitly captured) — ignored paths like node_modules/venv/build outputs are never touched, since they were never stashed. Returns True if the stash was dropped, False on a git failure (in which case the stash is left in place for safety). """ stash_selector = _resolve_stash_selector(git_cmd, cwd, stash_ref) if stash_selector is None: print( "⚠ Configured to discard local changes on non-interactive update, " "but Hermes couldn't find the stash entry to drop." ) _print_stash_cleanup_guidance(stash_ref) return False drop = subprocess.run( git_cmd + ["stash", "drop", stash_selector], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if drop.returncode != 0: print( "⚠ Configured to discard local changes, but Hermes couldn't drop " "the saved stash entry." ) if drop.stderr.strip(): print(f" {drop.stderr.strip().splitlines()[0]}") _print_stash_cleanup_guidance(stash_ref, stash_selector) return False print("→ Discarded local source changes (updates.non_interactive_local_changes=discard).") return True OFFICIAL_REPO_URLS = { "https://github.com/NousResearch/hermes-agent.git", "git@github.com:NousResearch/hermes-agent.git", "https://github.com/NousResearch/hermes-agent", "git@github.com:NousResearch/hermes-agent", } OFFICIAL_REPO_URL = "https://github.com/NousResearch/hermes-agent.git" SKIP_UPSTREAM_PROMPT_FILE = ".skip_upstream_prompt" def _get_origin_url(git_cmd: list[str], cwd: Path) -> Optional[str]: """Get the URL of the origin remote, or None if not set.""" try: result = subprocess.run( git_cmd + ["remote", "get-url", "origin"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if result.returncode == 0: return result.stdout.strip() except Exception: pass return None def _is_fork(origin_url: Optional[str]) -> bool: """Check if the origin remote points to a fork (not the official repo).""" if not origin_url: return False # Normalize URL for comparison (strip trailing .git if present) normalized = origin_url.rstrip("/") if normalized.endswith(".git"): normalized = normalized[:-4] for official in OFFICIAL_REPO_URLS: official_normalized = official.rstrip("/") if official_normalized.endswith(".git"): official_normalized = official_normalized[:-4] if normalized == official_normalized: return False return True def _has_upstream_remote(git_cmd: list[str], cwd: Path) -> bool: """Check if an 'upstream' remote already exists.""" try: result = subprocess.run( git_cmd + ["remote", "get-url", "upstream"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) return result.returncode == 0 except Exception: return False def _add_upstream_remote(git_cmd: list[str], cwd: Path) -> bool: """Add the official repo as the 'upstream' remote. Returns True on success.""" try: result = subprocess.run( git_cmd + ["remote", "add", "upstream", OFFICIAL_REPO_URL], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) return result.returncode == 0 except Exception: return False def _count_commits_between(git_cmd: list[str], cwd: Path, base: str, head: str) -> int: """Count commits on `head` that are not on `base`. Returns -1 on error.""" try: result = subprocess.run( git_cmd + ["rev-list", "--count", f"{base}..{head}"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if result.returncode == 0: return int(result.stdout.strip()) except Exception: pass return -1 def _should_skip_upstream_prompt() -> bool: """Check if user previously declined to add upstream.""" from hermes_constants import get_hermes_home return (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).exists() def _mark_skip_upstream_prompt(): """Create marker file to skip future upstream prompts.""" try: from hermes_constants import get_hermes_home (get_hermes_home() / SKIP_UPSTREAM_PROMPT_FILE).touch() except Exception: pass def _sync_fork_with_upstream(git_cmd: list[str], cwd: Path) -> bool: """Attempt to push updated main to origin (sync fork). Returns True if push succeeded, False otherwise. """ try: result = subprocess.run( git_cmd + ["push", "origin", "main", "--force-with-lease"], cwd=cwd, capture_output=True, text=True, encoding="utf-8", errors="replace", ) return result.returncode == 0 except Exception: return False def _sync_with_upstream_if_needed(git_cmd: list[str], cwd: Path) -> None: """Check if fork is behind upstream and sync if safe. This implements the fork upstream sync logic: - If upstream remote doesn't exist, ask user if they want to add it - Compare origin/main with upstream/main - If origin/main is strictly behind upstream/main, pull from upstream - Try to sync fork back to origin if possible """ has_upstream = _has_upstream_remote(git_cmd, cwd) if not has_upstream: # Check if user previously declined if _should_skip_upstream_prompt(): return # Ask user if they want to add upstream print() print("ℹ Your fork is not tracking the official Hermes repository.") print(" This means you may miss updates from NousResearch/hermes-agent.") print() try: response = ( input("Add official repo as 'upstream' remote? [Y/n]: ").strip().lower() ) except (EOFError, KeyboardInterrupt, UnicodeDecodeError): print() response = "n" if response in {"", "y", "yes"}: print("→ Adding upstream remote...") if _add_upstream_remote(git_cmd, cwd): print( " ✓ Added upstream: https://github.com/NousResearch/hermes-agent.git" ) has_upstream = True else: print(" ✗ Failed to add upstream remote. Skipping upstream sync.") return else: print( " Skipped. Run 'git remote add upstream https://github.com/NousResearch/hermes-agent.git' to add later." ) _mark_skip_upstream_prompt() return # Fetch upstream main only. This sync compares upstream/main with # origin/main, so there's no reason to pull every upstream ref — and a bare # fetch drags in thousands of auto-generated branches. print() print("→ Fetching upstream...") try: subprocess.run( git_cmd + ["fetch", "upstream", "main", "--quiet"], cwd=cwd, capture_output=True, check=True, ) except subprocess.CalledProcessError: print(" ✗ Failed to fetch upstream. Skipping upstream sync.") return # Compare origin/main with upstream/main origin_ahead = _count_commits_between(git_cmd, cwd, "upstream/main", "origin/main") upstream_ahead = _count_commits_between( git_cmd, cwd, "origin/main", "upstream/main" ) if origin_ahead < 0 or upstream_ahead < 0: print(" ✗ Could not compare branches. Skipping upstream sync.") return # If origin/main has commits not on upstream, don't trample if origin_ahead > 0: print() print(f"ℹ Your fork has {origin_ahead} commit(s) not on upstream.") print(" Skipping upstream sync to preserve your changes.") print(" If you want to merge upstream changes, run:") print(" git pull upstream main") return # If upstream is not ahead, fork is up to date if upstream_ahead == 0: print(" ✓ Fork is up to date with upstream") return # origin/main is strictly behind upstream/main (can fast-forward) print() print(f"→ Fork is {upstream_ahead} commit(s) behind upstream") print("→ Pulling from upstream...") try: subprocess.run( git_cmd + ["pull", "--ff-only", "upstream", "main"], cwd=cwd, check=True, ) except subprocess.CalledProcessError: print( " ✗ Failed to pull from upstream. You may need to resolve conflicts manually." ) return print(" ✓ Updated from upstream") # Try to sync fork back to origin print("→ Syncing fork...") if _sync_fork_with_upstream(git_cmd, cwd): print(" ✓ Fork synced with upstream") else: print( " ℹ Got updates from upstream but couldn't push to fork (no write access?)" ) print(" Your local repo is updated, but your fork on GitHub may be behind.") def _invalidate_update_cache(): """Delete the update-check cache for ALL profiles so no banner reports a stale "commits behind" count after a successful update. The git repo is shared across profiles — when one profile runs ``hermes update``, every profile is now current. """ homes = [] # Default profile home (Docker-aware — uses /opt/data in Docker) from hermes_constants import get_default_hermes_root default_home = get_default_hermes_root() homes.append(default_home) # Named profiles under /profiles/ profiles_root = default_home / "profiles" if profiles_root.is_dir(): for entry in profiles_root.iterdir(): if entry.is_dir(): homes.append(entry) for home in homes: try: cache_file = home / ".update_check" if cache_file.exists(): cache_file.unlink() except Exception: pass def _write_marker_file(path: Path, *, label: str) -> None: """Drop an update-recovery breadcrumb. Never raises.""" if _m()._pytest_owns_live_checkout(path.parent): logger.debug("Skipping %s marker under pytest (live checkout)", label) return try: path.write_text( f"started={_time.time()}\npid={os.getpid()}\n", encoding="utf-8" ) except OSError as exc: logger.debug("Could not write %s marker: %s", label, exc) def _write_update_incomplete_marker() -> None: """Drop the interrupted core-install breadcrumb. Never raises.""" _write_marker_file(_m()._update_marker_path(), label="update-incomplete") def _write_lazy_refresh_incomplete_marker() -> None: """Drop the interrupted lazy-refresh breadcrumb. Never raises.""" _write_marker_file(_m()._lazy_refresh_marker_path(), label="lazy-refresh-incomplete") def _format_concurrent_instances_message( matches: list[tuple[int, str]], scripts_dir: Path ) -> str: """Build a human-readable explanation + remediation hint for the user.""" shim = scripts_dir / "hermes.exe" lines = ["✗ Another hermes.exe is running:"] for pid, name in matches: lines.append(f" PID {pid} {name}") lines.append("") lines.append(f" Updating now would fail to overwrite {shim} because") lines.append(" Windows blocks REPLACE on a running executable.") lines.append("") lines.append(" Close Hermes Desktop, exit any open `hermes` REPLs, and") lines.append(" stop the gateway (`hermes gateway stop`) before retrying.") lines.append("") if matches: pid_args = " ".join(f"/PID {pid}" for pid, _ in matches) lines.append(" If you've already closed everything and these PIDs are") lines.append(" stale, terminate them directly, then retry the update:") lines.append(f" taskkill {pid_args} /F") lines.append("") lines.append(" Override with `hermes update --force` if you've already") lines.append(" confirmed those processes will not write to the venv.") return "\n".join(lines) def _upgrade_pip_before_lazy_refresh( install_cmd_prefix: list[str], *, env: dict[str, str] | None = None, ) -> None: """Upgrade pip before lazy-backend refreshes. Older pip (e.g. 24.0 on Python 3.11) can fail setuptools-backed source builds during lazy installs and leave a partially-written venv (#57828). Never raises. """ try: _m()._run_package_only_install( install_cmd_prefix + ["install", "--upgrade", "pip"], env=env, ) except subprocess.CalledProcessError as exc: logger.debug("pip upgrade before lazy refresh failed: %s", exc) def _capture_active_lazy_features() -> list[str]: """Snapshot active lazy backends before a managed runtime is replaced.""" try: from tools import lazy_deps return lazy_deps.active_features() except Exception as exc: logger.debug("Could not snapshot active lazy features: %s", exc) return [] def _capture_active_tool_dependencies() -> list[str]: """Snapshot Python dependencies installed explicitly through ``hermes tools``.""" try: from hermes_cli import tools_config return tools_config.active_restorable_python_tool_dependencies() except Exception as exc: logger.debug("Could not snapshot active Hermes Tools dependencies: %s", exc) return [] def _restore_active_tool_dependencies( dependencies: list[str], install_cmd_prefix: list[str], *, env: dict[str, str] | None = None, ) -> None: """Restore allowlisted ``hermes tools`` dependencies into a rebuilt venv. The dependency names came from a pre-rebuild import probe and are resolved through a static package allowlist. Never raises: a failed optional tool must not block the core update, but the user must be told what stayed unavailable. """ if not dependencies: return try: from hermes_cli import tools_config except Exception as exc: logger.debug("Hermes Tools dependency restore skipped (import failed): %s", exc) return target_python = _m()._resolve_install_target_python(install_cmd_prefix, env) missing: list[tuple[str, tuple[str, ...]]] = [] for name in dependencies: spec = tools_config.restorable_python_tool_dependency(name) if spec is None: continue module_name, install_args = spec if target_python is not None: try: probe = subprocess.run( [ str(target_python), "-c", "import importlib.util,sys; " "raise SystemExit(0 if importlib.util.find_spec(sys.argv[1]) else 1)", module_name, ], capture_output=True, env=env, check=False, ) if probe.returncode == 0: continue except (subprocess.SubprocessError, OSError): # An indeterminate probe is safer to repair than to treat as # proof that a pre-rebuild dependency survived. pass missing.append((name, install_args)) if not missing: return print() print(f"→ Restoring {len(missing)} Hermes Tools dependency set(s)...") restored: list[str] = [] failed: list[tuple[str, str]] = [] for name, install_args in missing: try: _m()._run_package_only_install( install_cmd_prefix + ["install", *install_args, "--quiet"], env=env, ) restored.append(name) except Exception as exc: # This is best-effort recovery for optional tooling. Unexpected # installer failures must be surfaced without aborting the core # runtime update. failed.append((name, str(exc))) if restored: print(f" ✓ {len(restored)} restored: {', '.join(restored)}") for name, reason in failed: if len(reason) > 200: reason = reason[:200] + "..." print(f" ⚠ {name} failed to restore: {reason}") def _refresh_active_lazy_features( install_cmd_prefix: list[str] | None = None, *, env: dict[str, str] | None = None, features: list[str] | None = None, ) -> bool: """Refresh lazy-installed backends after a code update. When pyproject.toml's ``[all]`` extra was slimmed down (May 2026), most optional backends moved to ``tools/lazy_deps.py`` and only install on first use. ``hermes update`` runs ``uv pip install -e .[all]`` which leaves those packages untouched — so if we bump a pin in :data:`LAZY_DEPS` (CVE response, transitive bug fix), users who already activated the backend keep the stale version forever. This function asks lazy_deps which features the user has previously activated and reinstalls them under the current pins. Features the user never enabled stay quiet — no churn for cold backends. Returns True when the venv is safe to use (refresh succeeded, or no active lazy backends, or post-failure import repair succeeded). Returns False when a failed lazy install left broken core imports that automatic repair could not fix (#57828). Never raises. A failure here must not block the rest of the update. """ try: from tools import lazy_deps except Exception as exc: logger.debug("Lazy refresh skipped (import failed): %s", exc) return True if features is None: try: active = lazy_deps.active_features() except Exception as exc: logger.debug("Lazy refresh skipped (active_features failed): %s", exc) return True else: active = features if not active: return True print() print(f"→ Refreshing {len(active)} active lazy backend(s)...") unexpected_failure = False try: if features is None: results = lazy_deps.refresh_active_features(prompt=False) else: results = lazy_deps.restore_features(active) except Exception as exc: # refresh_active_features is documented as never-raise, but defend # the update flow against future regressions. print(f" ⚠ Lazy refresh failed unexpectedly: {exc}") results = {} unexpected_failure = True refreshed = [f for f, s in results.items() if s in {"refreshed", "restored"}] current = [f for f, s in results.items() if s == "current"] failed = [(f, s) for f, s in results.items() if s.startswith("failed:")] skipped = [(f, s) for f, s in results.items() if s.startswith("skipped:")] if refreshed: print(f" ↑ {len(refreshed)} refreshed: {', '.join(refreshed)}") if current: print(f" ✓ {len(current)} already current") if skipped: # Most common reason: security.allow_lazy_installs=false. Show one # line so the user knows why; not an error. names = ", ".join(f for f, _ in skipped) reason = skipped[0][1].split(": ", 1)[-1] print(f" · {len(skipped)} skipped ({reason}): {names}") if not failed and not unexpected_failure: return True for feature, status in failed: reason = status.split(": ", 1)[-1] # Clip noisy pip stderr to keep update output legible. if len(reason) > 200: reason = reason[:200] + "..." print(f" ⚠ {feature} failed to refresh: {reason}") if install_cmd_prefix is None: print(" ⚠ Lazy refresh failed; rerun `hermes update` once resolved.") return False # Immediate import-based recovery — metadata-only verifiers miss the case # where DISTRIBUTION-INFO remains but import files were wiped (#57828). # Unavailable probes are indeterminate, not healthy — keep the lazy marker. status = _m()._repair_venv_via_import_probes(install_cmd_prefix, env=env) if status == "repaired": print( " Lazy backend(s) keep their previous version until refresh succeeds." ) return True if status == "healthy": print( " Lazy backend(s) keep their previous version; probed packages look intact." ) print(" Rerun `hermes update` once the upstream issue is resolved.") return True if status == "indeterminate": print( " ⚠ Leaving `.lazy-refresh-incomplete` until import probes can confirm health." ) return False def _refresh_active_memory_provider_dependencies() -> None: """Refresh pip dependencies for the configured external memory provider. Memory-provider bridge packages are declared in each provider's ``plugin.yaml`` (plus mode-dependent extras like Hindsight's ``hindsight-all``), NOT in Hermes' editable-install extras or ``LAZY_DEPS`` alone — so the core dependency reinstall above can strip or downgrade them (#53272 mem0ai, #70636 hindsight-embed). Re-run the provider's declared install for the ACTIVE provider only, after the core install and lazy refresh, so the last write to any shared package is the one the active provider needs. Never raises. A failure here must not block the rest of the update. """ try: from hermes_cli.config import load_config cfg = load_config() except Exception as exc: logger.debug("Memory provider refresh skipped (config load failed): %s", exc) return provider = "" if isinstance(cfg, dict): memory_cfg = cfg.get("memory") if isinstance(memory_cfg, dict): if memory_cfg.get("enabled") is False: return provider = str(memory_cfg.get("provider") or "").strip() # "default" / empty is the built-in file-backed store — no pip deps. if not provider or provider in {"default", "builtin", "none"}: return try: from hermes_cli.memory_setup import _install_dependencies except Exception as exc: logger.debug("Memory provider refresh skipped (import failed): %s", exc) return print() print(f"→ Refreshing active memory provider dependencies ({provider})...") try: _install_dependencies(provider, force=True) except Exception as exc: print(f" ⚠ {provider} dependencies failed to refresh: {exc}") def _is_android_python() -> bool: return _m().sys.platform == "android" def _install_psutil_android_compat( install_cmd_prefix: list[str], *, env: dict[str, str] | None = None, ) -> None: """Install psutil on Android by patching upstream platform detection. psutil's setup currently gates Linux sources behind ``sys.platform.startswith('linux')``. On Termux Python reports ``sys.platform == 'android'``, so setup aborts with "platform android is not supported" despite compiling fine when using the Linux source path. We patch only the extracted build tree used for this install attempt; nothing is persisted in the repository. Stopgap: remove this once https://github.com/giampaolo/psutil/pull/2762 merges and ships in a release. The standalone installer script uses the same shared helper and should be removed together. """ import tempfile import urllib.request from hermes_cli.psutil_android import PSUTIL_URL, prepare_patched_psutil_sdist with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) archive = tmp_path / "psutil.tar.gz" urllib.request.urlretrieve(PSUTIL_URL, archive) src_root = prepare_patched_psutil_sdist(archive, tmp_path) _m()._run_install_with_heartbeat( install_cmd_prefix + ["install", "--no-build-isolation", str(src_root)], env=env, ) def _ensure_uv_for_termux(pip_cmd: list[str]) -> str | None: """Best-effort uv bootstrap on Termux for faster update installs. The normal path (``ensure_uv()`` in managed_uv) installs the managed standalone uv into ``$HERMES_HOME/bin/uv``, but on Termux the official installer may not work (glibc vs bionic). Prefer a uv already on PATH (e.g. ``pkg install uv``); only if there is none do we fall back to a wheel-only ``pip install uv`` so we never source-build the Rust crate. """ from hermes_cli.managed_uv import resolve_uv existing = resolve_uv() if existing: return existing if not _m()._is_termux_env(): return None # A Termux-packaged uv lands on PATH but not in the managed bin dir, so # resolve_uv() misses it. Use it before pip, which has no Android wheel and # would otherwise build uv from source on a low-memory device. system_uv = shutil.which("uv") if system_uv: return system_uv try: print(" → Termux detected: trying to install uv for faster dependency updates...") result = subprocess.run( pip_cmd + ["install", "uv", "--only-binary", ":all:"], cwd=_m().PROJECT_ROOT, check=False, ) if result.returncode != 0: return None except Exception: pass # After pip install, check managed path first, then PATH return resolve_uv() or shutil.which("uv") def _npm_manifest_paths() -> tuple[Path, ...]: """Manifests whose changes must defeat the update-skip. The lockfile alone is NOT a sufficient key: on a local checkout a dev can edit package.json (root or a workspace) without running npm — the lockfile is then unchanged but `hermes update` is exactly the step expected to sync node_modules (via the `npm install` fallback in _run_npm_install_deterministic). The workspace list is pulled from the root package.json's `workspaces` globs (npm's own source of truth) rather than hardcoded, so adding a workspace can never silently escape the skip key. Every workspace manifest belongs in the key — desktop included, even though the install only names ui-tui and web — because the single lockfile spans the whole workspace graph, so any manifest edit can put the lockfile out of sync and change what the install must do. Falls back to hashing just root manifests if package.json is unreadable (never skips more than main would have installed). """ root_pkg = _m().PROJECT_ROOT / "package.json" paths = [_m().PROJECT_ROOT / "package-lock.json", root_pkg] try: workspaces = json.loads(root_pkg.read_text(encoding="utf-8")).get( "workspaces", [] ) if isinstance(workspaces, dict): # legacy {"packages": [...]} form workspaces = workspaces.get("packages", []) for pattern in workspaces: for match in sorted(_m().PROJECT_ROOT.glob(str(pattern))): manifest = match / "package.json" if manifest.is_file(): paths.append(manifest) except (OSError, json.JSONDecodeError, TypeError): pass return tuple(paths) def _npm_manifests_digest() -> str | None: """Combined sha256 over the lockfile + all workspace package.json files. Returns None when the lockfile is missing (never skip then). """ if not (_m().PROJECT_ROOT / "package-lock.json").exists(): return None h = hashlib.sha256() for p in _npm_manifest_paths(): h.update(str(p.relative_to(_m().PROJECT_ROOT)).encode()) try: h.update(p.read_bytes()) except OSError: h.update(b"") return h.hexdigest() def _npm_lockfile_changed(hermes_root: Path) -> bool: current = _npm_manifests_digest() if current is None: return True # Also check that node_modules exists; a matching hash with missing # node_modules means the cache was recorded by another checkout. if not (_m().PROJECT_ROOT / "node_modules").is_dir(): return True # A matching lockfile hash over a tree whose web build toolchain never # landed must NOT skip the reinstall — otherwise every later `hermes # update` keeps rebuilding against a half-installed tree and serving a # stale dist. web_dir = _m().PROJECT_ROOT / "web" if (web_dir / "package.json").is_file() and not _web_build_toolchain_ready( *_web_toolchain_roots(web_dir) ): return True try: # Key the cache by PROJECT_ROOT so parallel worktrees don't collide. cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12] cache_file = hermes_root / f".npm_lock_hash_{cache_key}" if not cache_file.exists(): return True return cache_file.read_text(encoding="utf-8").strip() != current except OSError: return True def _record_npm_lockfile_hash(hermes_root: Path) -> None: digest = _npm_manifests_digest() if digest is None: return try: cache_key = hashlib.sha256(str(_m().PROJECT_ROOT).encode()).hexdigest()[:12] cache_file = hermes_root / f".npm_lock_hash_{cache_key}" cache_file.write_text(digest, encoding="utf-8") except OSError: logger.debug("Could not write npm lockfile hash cache") def _repair_node_deps_on_current_checkout(print_completion) -> None: """Repair Node deps on the ``commit_count == 0`` path (#77211). A current checkout does not imply healthy Node deps: a previous npm install may have failed (EBADENGINE from a node/npm mismatch, network timeout, interrupted install) and its error message says to "re-run hermes update" — but the early return never reached the Node refresh, so that repair advice could never work. ``_update_node_dependencies`` self-gates on the lockfile hash, which is only recorded after a SUCCESSFUL npm install (and re-trips when node_modules is missing or the web toolchain never landed), so this is a cheap no-op on healthy installs and a real repair after a failed one. """ node_failures = _update_node_dependencies() if node_failures: print(f" ⚠ Node.js refresh failed for: {', '.join(node_failures)}") print(" Fix npm and re-run `hermes update`.") print_completion( "⚠ Checkout is current, but Node.js dependencies could not be repaired." ) return # Pair the refresh with the web build like every other # _update_node_dependencies call site; it staleness-checks internally, # so this is a no-op when nothing changed. _m()._build_web_ui(_m().PROJECT_ROOT / "web") print_completion("✓ Already up to date!") def _update_node_dependencies() -> list[str]: """Refresh Node deps for the ui-tui and web workspaces. Returns the list of labels whose npm install failed (empty on success), so the caller can treat a Node refresh failure as a partial update rather than silently reporting ``Update complete!`` (#30271). """ if not (_m().PROJECT_ROOT / "package.json").exists(): return [] npm = _m()._resolve_node_runtime_npm() if not npm: # If the only npm reachable inside this WSL shell is the Windows one, # flag it loudly: silently skipping leaves ui-tui deps stale while the # rest of the update proceeds, and running it would corrupt the tree. from hermes_constants import is_wsl path_npm = shutil.which("npm") if is_wsl() and path_npm and _m()._is_windows_npm_path(path_npm): print("→ Updating Node.js dependencies...") print(" ⚠ Skipped: only a Windows npm is reachable from this WSL shell.") print(" Install Node.js inside the WSL distro (nvm, or your distro's") print(" package manager), then re-run `hermes update`.") failed = [] if any( (_m().PROJECT_ROOT / workspace / "package.json").exists() for workspace in ("ui-tui", "web") ): failed.append("ui-tui, web workspaces") return failed return [] from hermes_constants import get_default_hermes_root # This cache describes PROJECT_ROOT/node_modules, which is shared by every # Hermes profile using this checkout. Keep one per-checkout cache under the # shared Hermes root rather than rerunning npm once per named profile. shared_hermes_root = get_default_hermes_root() # Best-effort: warm npx's cache for agent-browser (#43564). Runs before # the lockfile-unchanged early return below since that's the common # `hermes update` case. Synchronous and can block ~11s on a true cold # cache (~0.4s once warm) — print first so that doesn't look like a hang. print("→ Warming npx cache for agent-browser...") try: from tools.browser_tool import warm_agent_browser_npx_cache warm_agent_browser_npx_cache() except Exception: pass if not _m()._npm_lockfile_changed(shared_hermes_root): logger.info("npm lockfile unchanged, skipping npm install") return [] # Root package.json has no dependencies of its own (agent-browser and # @streamdown/math were moved out — see #43564): agent-browser resolves # at runtime via `npx agent-browser` (tools/browser_tool.py), and # @streamdown/math is a desktop-only import now declared in # apps/desktop/package.json. That means a plain workspace-scoped install # can never prune anything root-only, so we only need to name the # workspaces the CLI/TUI/web build actually requires. apps/desktop pulls # in Electron as a devDependency with a ~200MB postinstall download, so # it's deliberately never named here — desktop deps install on demand # (see _desktop_build_needed). print("→ Updating Node.js dependencies...") def _partial_update_failure(*labels: str) -> list[str]: print() print(" ⚠ Node.js dependency refresh did not complete cleanly; the") print(" installation may be in a mixed state (updated code, stale Node") print(" deps). Fix npm and re-run `hermes update`.") return list(labels) install_args = [ "--no-fund", "--no-audit", "--prefer-offline", "--progress=false", "--workspace", "ui-tui", "--workspace", "web", # Root package.json's own devDependencies (the shared ESLint flat # config every workspace's eslint.config.mjs imports) are otherwise # pruned by this scoped install, same as agent-browser/@streamdown # math used to be before they moved out of root entirely (#43564). # Unlike those, root's devDependencies have nowhere else to live — # this flag still excludes apps/desktop, which is never named above. "--include-workspace-root", ] from hermes_constants import with_hermes_node_path nixos_env = with_hermes_node_path(_m()._nixos_build_env()) # NOTE: capture_output=False here is deliberate (#18840) — optional # postinstall scripts print download progress, and capturing it makes a # long download look hung. The chatty npm-deprecation noise during # `hermes update` comes from the *desktop* build, not this step; that # one is captured to update.log. result = _m()._run_npm_install_deterministic( npm, _m().PROJECT_ROOT, extra_args=tuple(install_args), capture_output=False, env=nixos_env, ) if result.returncode == 0: _record_npm_lockfile_hash(shared_hermes_root) print(" ✓ ui-tui, web workspaces installed (desktop skipped)") failures: list[str] = [] else: print(" ⚠ npm install failed") stderr = (result.stderr or "").strip() if result.stderr else "" if stderr: print(f" {stderr.splitlines()[-1]}") failures = _partial_update_failure("ui-tui, web workspaces") return failures def _log_only_write(text: str) -> None: """Write ``text`` to ``~/.hermes/logs/update.log`` only, never the terminal. During ``hermes update`` ``sys.stdout`` is an ``_UpdateOutputStream`` that mirrors to both the terminal and ``update.log``. Loud, low-signal subprocess output (npm installs, the Electron/vite build, the cua-driver installer's "Next steps" wall) should be captured and tucked into the log so failures stay debuggable, without flooding the user's terminal. This reaches past the mirroring stream straight to the underlying log handle. """ if not text: return stream = _m().sys.stdout log_file = getattr(stream, "_log", None) if log_file is None: return try: log_file.write(text if text.endswith("\n") else text + "\n") log_file.flush() except Exception: pass def _run_logged_subprocess(cmd, *, cwd=None, env=None): """Run ``cmd`` capturing combined output into update.log (not the terminal). Returns the ``CompletedProcess`` (with ``stdout`` populated) so the caller can decide whether to surface the captured output on failure. """ result = subprocess.run( cmd, cwd=cwd, env=env, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace", ) _log_only_write(result.stdout or "") return result def _classify_fetch_failure(stderr: str) -> str: """Map git-fetch stderr to a one-line, user-facing diagnosis. Order matters: curl surfaces HTTP failures as ``fatal: unable to access '': The requested URL returned error: 429``, so the rate-limit/outage checks must run BEFORE the generic "unable to access" network check or a GitHub 429/5xx gets misreported as a local network problem. The caller always prints the first raw stderr line alongside this diagnosis — the friendly message adds guidance, it never replaces the wire error. """ def _has_http_code(*codes: str) -> bool: return any( f"HTTP {code}" in stderr or f"returned error: {code}" in stderr for code in codes ) if _has_http_code("429") or "rate limit" in stderr.lower(): return ( "✗ GitHub is rate limiting requests or having an outage (HTTP 429)" " — try again in 5 minutes." ) if _has_http_code("500", "502", "503", "504"): return ( "✗ GitHub appears to be having an outage — try again in a few" " minutes (https://www.githubstatus.com)." ) if "Could not resolve host" in stderr or "unable to access" in stderr: return "✗ Network error — cannot reach the remote repository." if "Authentication failed" in stderr or "could not read Username" in stderr: return "✗ Authentication failed — check your git credentials or SSH key." return "✗ Failed to fetch updates from origin." def _print_fetch_failure(stderr: str) -> None: """Print the classified diagnosis plus the first raw stderr line.""" stderr = (stderr or "").strip() print(_classify_fetch_failure(stderr)) if stderr: print(f" {stderr.splitlines()[0]}") def _cmd_update_check(branch: str = "main", *, branch_explicit: bool = False): """Implement ``hermes update --check``: fetch and report without installing. ``branch`` selects which branch the check compares against. Default is "main"; callers can pass another branch to ask "are there new commits on origin/?" without performing the update. ``branch_explicit`` is True iff the caller passed --branch on the CLI. Installs that can't honor non-default branches (e.g. Docker) surface a one-line notice instead of silently dropping the flag. """ from hermes_cli.config import ( detect_install_method, is_nix_install_method, recommended_update_command_for_method, ) method = detect_install_method(_m().PROJECT_ROOT) if method == "docker": # Docker can't ``git fetch`` from within the container. Surface the # same long-form ``docker pull`` guidance ``hermes update`` (apply # path) uses — telling the user to "reinstall via curl" or that # ".git is missing" would point them at the wrong remediation. from hermes_cli.config import format_docker_update_message print(format_docker_update_message()) sys.exit(1) if is_nix_install_method(method) or method == "apt": print(recommended_update_command_for_method(method)) sys.exit(1) git_dir = _m().PROJECT_ROOT / ".git" if not git_dir.exists(): print("✗ Not a git repository — cannot check for updates.") sys.exit(1) git_cmd = ["git"] if sys.platform == "win32": git_cmd = ["git", "-c", "windows.appendAtomically=false"] # A crashed/interrupted fetch can leave .git/shallow.lock (or another git # lock file) behind; every later fetch then fails with "File exists" and # the check reports a hard failure (or, in the banner path, silently # compares stale refs). Self-heal abandoned locks before fetching. from hermes_cli.gitlock import clear_stale_git_locks cleared = clear_stale_git_locks(_m().PROJECT_ROOT) for lock_path in cleared: print(f" (removed stale git lock: {lock_path})") # Fetch only the branch we compare against; prefer upstream as the canonical # reference. A bare `git fetch ` pulls every ref, and this repo has # thousands of auto-generated branches, so scope the fetch to . # Note: upstream/ may not exist for non-main branches (a fork's # bb/gui has no upstream counterpart), so when the caller picks a # non-default branch we skip the upstream probe and use origin directly. # Installer checkouts are shallow (`git clone --depth 1`). A plain # `git fetch` would unshallow the repo (dragging in the whole history — # the exact cost the shallow clone avoided) and the rev-list count below # would then report a huge bogus "behind" number. Detect shallow up front: # fetch with --depth 1 to preserve the boundary and report presence-only. is_shallow = ( subprocess.run( git_cmd + ["rev-parse", "--is-shallow-repository"], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ).stdout.strip() == "true" ) depth_args = ["--depth", "1"] if is_shallow else [] if branch == "main": # Probe locally (~6 ms) whether an 'upstream' remote exists at all # before spending a network fetch on it. Non-fork installs have no # 'upstream' remote, and the old flow burned a failed network attempt # (~0.3-1 s) on every --check before falling back to origin. has_upstream_remote = ( subprocess.run( git_cmd + ["remote", "get-url", "upstream"], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ).returncode == 0 ) fetch_result = None if has_upstream_remote: print("→ Fetching from upstream...") fetch_result = subprocess.run( git_cmd + ["fetch"] + depth_args + ["upstream", branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if fetch_result is not None and fetch_result.returncode == 0: upstream_exists = True compare_branch = f"upstream/{branch}" else: # No upstream remote, or the upstream fetch failed — use origin. print("→ Fetching from origin...") fetch_result = subprocess.run( git_cmd + ["fetch"] + depth_args + ["origin", branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ) upstream_exists = False compare_branch = f"origin/{branch}" else: # Non-default branch: compare against origin/ directly. print("→ Fetching from origin...") fetch_result = subprocess.run( git_cmd + ["fetch"] + depth_args + ["origin", branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ) upstream_exists = False compare_branch = f"origin/{branch}" if fetch_result.returncode != 0: _print_fetch_failure(fetch_result.stderr) sys.exit(1) # Verify the compare ref actually exists before asking rev-list about it. # Without this, `git rev-list HEAD..origin/ --count` exits 128 and # (with check=True) raises CalledProcessError, surfacing a Python # traceback. Friendlier to detect-and-report. verify_result = subprocess.run( git_cmd + ["rev-parse", "--verify", "--quiet", compare_branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ) if verify_result.returncode != 0: print(f"✗ Branch '{branch}' not found on {compare_branch.split('/', 1)[0]}.") sys.exit(1) if is_shallow: # No history to count across the shallow boundary. Compare tip SHAs # (mirrors the banner's _check_via_local_git), then try to recover the # exact count via the GitHub compare API — the remote graph is complete # even when the local one is truncated. head_sha = subprocess.run( git_cmd + ["rev-parse", "HEAD"], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ).stdout.strip() target_sha = subprocess.run( git_cmd + ["rev-parse", compare_branch], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", ).stdout.strip() if head_sha and target_sha and head_sha == target_sha: print("✓ Already up to date.") else: from hermes_cli.banner import _github_compare_behind from hermes_cli.config import recommended_update_command counted = _github_compare_behind(head_sha, target_sha) if counted == 0: # Local commits on top of the remote tip — not behind. print("✓ Already up to date.") return if counted is not None: commits_word = "commit" if counted == 1 else "commits" print(f"⚕ Update available: {counted} {commits_word} behind {compare_branch}.") else: print(f"⚕ Update available (behind {compare_branch}).") print(f" Run '{recommended_update_command()}' to install.") return rev_result = subprocess.run( git_cmd + ["rev-list", f"HEAD..{compare_branch}", "--count"], cwd=_m().PROJECT_ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace", check=True, ) behind = int(rev_result.stdout.strip()) if behind == 0: print("✓ Already up to date.") else: commits_word = "commit" if behind == 1 else "commits" print(f"⚕ Update available: {behind} {commits_word} behind {compare_branch}.") from hermes_cli.config import recommended_update_command print(f" Run '{recommended_update_command()}' to install.") def _ensure_fhs_path_guard() -> None: """Ensure /usr/local/bin is on PATH for RHEL-family root non-login shells. Mirrors the post-symlink probe added to ``scripts/install.sh`` so that existing FHS-layout root installs on RHEL/CentOS/Rocky/Alma 8+ get repaired on ``hermes update`` without requiring a reinstall. The installer's assumption that ``/usr/local/bin`` is on PATH for every standard shell breaks on those distros in non-login interactive shells (su, sudo -s, tmux panes, some web terminals): /etc/bashrc doesn't add /usr/local/bin and /root/.bash_profile doesn't either. Symptom: ``hermes`` prints ``command not found`` even though the symlink lives at /usr/local/bin/hermes. Silent no-op on: non-Linux, non-root, non-FHS installs, and any system where ``bash -i -c 'command -v hermes'`` already resolves. Idempotent. """ if _m().sys.platform != "linux": return try: if os.geteuid() != 0: # windows-footgun: ok — Linux FHS helper, guarded by sys.platform == "linux" above + AttributeError catch return except AttributeError: return # Only act when this is actually an FHS-layout install (command link at # /usr/local/bin/hermes, code at /usr/local/lib/hermes-agent). fhs_link = Path("/usr/local/bin/hermes") if not fhs_link.is_symlink() and not fhs_link.exists(): return # Probe a fresh non-login interactive bash the way the user will use it. # ``bash -i -c`` sources ~/.bashrc but NOT ~/.bash_profile or /etc/profile, # which is the exact scenario where RHEL root loses /usr/local/bin. home = os.environ.get("HOME") or "/root" try: probe = subprocess.run( [ "env", "-i", f"HOME={home}", f"TERM={os.environ.get('TERM', 'dumb')}", "bash", "-i", "-c", "command -v hermes", ], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, ) except (FileNotFoundError, subprocess.TimeoutExpired): return # no bash or probe hung — don't block update on this if probe.returncode == 0: return # already on PATH, nothing to do path_line = 'export PATH="/usr/local/bin:$PATH"' path_comment = ( "# Hermes Agent — ensure /usr/local/bin is on PATH " "(RHEL non-login shells)" ) wrote_any = False for candidate in (".bashrc", ".bash_profile"): cfg = Path(home) / candidate if not cfg.is_file(): continue try: existing = cfg.read_text(errors="replace", encoding="utf-8") except OSError: continue # Idempotency: skip if any uncommented PATH= line already references # /usr/local/bin. Mirrors the grep pattern used by install.sh. already_guarded = any( "/usr/local/bin" in line and "PATH" in line and not line.lstrip().startswith("#") for line in existing.splitlines() ) if already_guarded: continue try: with cfg.open("a", encoding="utf-8") as f: f.write("\n" + path_comment + "\n" + path_line + "\n") except OSError as e: print(f" ⚠ Could not update {cfg}: {e}") continue print(f" ✓ Added /usr/local/bin to PATH in {cfg}") wrote_any = True if wrote_any: print(" (reload your shell or run 'source ~/.bashrc' to pick it up)") def _ensure_acp_launcher() -> None: """Self-heal: install a ``hermes-acp`` launcher next to the ``hermes`` one. Mirrors the launcher block in ``scripts/install.sh`` so existing installs gain the ACP command on ``hermes update`` without a reinstall. ACP hosts (Zed, JetBrains, Buzz Desktop) spawn the agent by resolving the ``hermes-acp`` command name against the login-shell PATH; the console script of that name lives inside the install's venv, which is not on that PATH, so those hosts report Hermes as not installed even when it is. The shim simply delegates to the sibling ``hermes`` launcher with the ``acp`` subcommand, which makes it correct for every install layout (venv wrapper, FHS symlink, pipx/pip console script) without having to reconstruct interpreter/entrypoint paths. No-op on Windows (install.ps1 copies ``hermes.exe`` + ``hermes-acp.exe`` into ``$InstallDir\bin`` and puts THAT on the user PATH — never the whole ``venv\Scripts`` dir, which would shadow the user's ``python`` (#83797) — so ``hermes-acp.exe`` already resolves) and wherever a ``hermes-acp`` is already present next to the ``hermes`` command. Unwritable directories (e.g. ``/usr/local/bin`` as non-root) are skipped silently. Idempotent. """ if _m().sys.platform == "win32": return for bin_dir in (Path.home() / ".local" / "bin", Path("/usr/local/bin")): hermes_cmd = bin_dir / "hermes" acp_cmd = bin_dir / "hermes-acp" try: if not (hermes_cmd.is_file() or hermes_cmd.is_symlink()): continue # Already present — a console script (pip/pipx install), an # earlier shim, or a symlink. is_symlink() catches broken # symlinks that exists() would miss; never follow-and-overwrite # (the #21454 failure mode). if acp_cmd.exists() or acp_cmd.is_symlink(): continue shim = ( "#!/usr/bin/env bash\n" "# Hermes Agent — ACP launcher (written by `hermes update`).\n" "# ACP hosts (Zed, JetBrains, Buzz) resolve the agent by this\n" "# command name on the login-shell PATH.\n" f'exec "{hermes_cmd}" acp "$@"\n' ) acp_cmd.write_text(shim, encoding="utf-8") acp_cmd.chmod(acp_cmd.stat().st_mode | 0o755) except OSError: continue print(f" ✓ Installed hermes-acp launcher → {acp_cmd}") _PRE_UPDATE_SNAPSHOT_KEEP = 1 # Per-file size cap for the pre-update quick snapshot. Anything larger is # skipped with a warning: the snapshot exists to protect small, hard-to- # regenerate state (pairing JSONs, cron jobs, config, auth) — not to copy a # multi-GB state.db on every update (observed: a 24 GB state.db added ~60s # of wall time and silently ate 24 GB of disk per update). _PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE = 1 << 30 # 1 GiB def _resolve_pre_update_backup_mode(args) -> str: """Resolve the pre-update backup mode: ``"off"``, ``"quick"``, or ``"full"``. CLI flags win over config; ``--no-backup`` beats ``--backup`` when both are set. Config accepts the mode strings plus legacy booleans: ``true`` → ``full`` (the old zip behavior), ``false`` → ``off`` (an explicit opt-out now disables the quick snapshot too — previously it ran unconditionally, ignoring the user's setting). A missing key defaults to ``quick``. """ if getattr(args, "no_backup", False): return "off" if getattr(args, "backup", False): return "full" try: from hermes_cli.config import load_config cfg = load_config() except Exception as exc: logging.getLogger(__name__).debug( "Could not load config for pre-update backup: %s", exc ) cfg = {} updates_cfg = cfg.get("updates", {}) if isinstance(cfg, dict) else {} raw = updates_cfg.get("pre_update_backup", "quick") if raw is True: return "full" if raw is False: return "off" mode = str(raw).strip().lower() if mode in ("off", "false", "none", "disabled"): return "off" if mode in ("full", "zip", "true"): return "full" if mode == "quick": return "quick" logging.getLogger(__name__).warning( "Unknown updates.pre_update_backup value %r — using 'quick'", raw ) return "quick" def _run_pre_update_backup(args) -> Optional[str]: """Run the pre-update safety backup and return the quick-snapshot id. Single consolidated mechanism gated on ``updates.pre_update_backup``: - ``off`` — nothing runs. Explicit user opt-out is honored fully. - ``quick`` (default) — a state snapshot of critical small files (pairing JSONs, cron jobs, config, auth; see ``_QUICK_STATE_FILES``) under ``state-snapshots/``. Files over 1 GiB are skipped with a warning so a bloated state.db can never stall the update (issues #15733, #34600 are the reason this safety net exists). - ``full`` — the quick snapshot PLUS a full zip of HERMES_HOME under ``backups/`` (restorable via ``hermes import``; the #48200 wrong-path wipe is the reason this level exists). ``--backup`` forces ``full`` for one run; ``--no-backup`` forces ``off``. Never raises — a backup failure should not block the update itself. Returns the quick-snapshot id (used by the post-update cron-jobs restore safety net), or ``None`` when mode is ``off`` or the snapshot failed. """ mode = _resolve_pre_update_backup_mode(args) if mode == "off": if getattr(args, "no_backup", False): print("◆ Pre-update backup: skipped (--no-backup)") print() # Config-level off is silent — the user opted out; don't spam them # on every update. return None snapshot_id = None try: from hermes_cli.backup import ( _quick_snapshot_root, create_quick_snapshot, verify_sqlite_integrity, ) # NOTE: this function later does `from hermes_constants import # get_hermes_home`, which makes the name function-local — the # module-level import is shadowed and unbound here. Alias explicitly. from hermes_cli.config import get_hermes_home as _get_home snapshot_id = create_quick_snapshot( label="pre-update", keep=_PRE_UPDATE_SNAPSHOT_KEEP, max_file_size=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, ) # After the snapshot, verify the source state.db is still intact. # The snapshot was taken via _safe_copy_db (read-only SQLite backup # API), but a concurrent process (antivirus, force-killed gateway # releasing file handles, Windows filter driver) can corrupt the live # file at any point. A silent zeroing at this point would proceed with # the update and exit code 0 — exactly the #68474 symptom. if snapshot_id: _src_path = _get_home() / "state.db" if _src_path.exists(): _integrity = verify_sqlite_integrity( _src_path, check_header=True, run_pragma=True, max_bytes=_PRE_UPDATE_SNAPSHOT_MAX_FILE_SIZE, ) if not _integrity.get("valid"): _msg = _integrity.get("message", "unknown error") print( f" ⚠ state.db integrity check FAILED after snapshot: {_msg}" ) # Check if the snapshot itself is valid. _snap_root = _quick_snapshot_root(_get_home()) _snap_state = _snap_root / snapshot_id / "state.db" if _snap_state.exists(): _snap_ok = verify_sqlite_integrity( _snap_state, check_header=True, run_pragma=True ) if _snap_ok.get("valid"): print( " ✓ Snapshot copy is valid — continuing update." ) print( " If state.db is lost after update it will be auto-restored." ) else: print( " ✗ Snapshot copy ALSO failed integrity — " "the source was already corrupted before the backup." ) else: print( " ⚠ Snapshot does not contain state.db (was skipped or too large)." ) print() if snapshot_id: print(f"◆ Pre-update snapshot: {snapshot_id}") except Exception as exc: # Never let a snapshot failure block an update. logging.getLogger(__name__).debug("Pre-update snapshot failed: %s", exc) if mode != "full": if snapshot_id: print() return snapshot_id try: from hermes_cli.backup import create_pre_update_backup except Exception as exc: print( f"⚠ Pre-update backup: could not load backup module ({exc}); continuing update." ) print() return snapshot_id try: from hermes_cli.config import load_config _keep = (load_config() or {}).get("updates", {}).get("backup_keep", 5) except Exception: _keep = 5 print("◆ Creating pre-update backup...") t0 = _time.monotonic() try: out_path = create_pre_update_backup(keep=int(_keep)) except Exception as exc: # defensive — helper already swallows, but just in case print(f" ⚠ Backup failed: {exc}") print(" Continuing with update.") print() return snapshot_id elapsed = _time.monotonic() - t0 if out_path is None: print(" ⚠ Backup skipped (no files found or write failed); continuing update.") print() return snapshot_id try: size_bytes = out_path.stat().st_size except OSError: size_bytes = 0 # Human-readable size from hermes_cli.sizefmt import format_bytes size_str = format_bytes(size_bytes) # Render path using display_hermes_home so the user sees ~/.hermes/... try: from hermes_constants import get_hermes_home, display_hermes_home home = get_hermes_home() try: display_path = f"{display_hermes_home()}/{out_path.relative_to(home)}" except ValueError: display_path = str(out_path) except Exception: display_path = str(out_path) print(f" Saved: {display_path} ({size_str}, {elapsed:.1f}s)") print(f" Restore: hermes import {out_path}") print(" Disable: set updates.pre_update_backup: quick (or off) in config.yaml") print() return snapshot_id def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool: """Write a planned-stop marker into a specific profile home.""" try: from datetime import timezone from gateway.status import _get_process_start_time from utils import atomic_json_write record = { "target_pid": pid, "target_start_time": _get_process_start_time(pid), "stopper_pid": os.getpid(), "written_at": datetime.now(timezone.utc).isoformat(), } atomic_json_write( Path(profile_path) / ".gateway-planned-stop.json", record, indent=None, separators=(",", ":"), ) return True except (OSError, PermissionError): return False def _wait_for_windows_update_gateway_exit( pids: list[int], *, timeout: float ) -> set[int]: """Wait for the given gateway PIDs to exit, returning survivors.""" if not pids: return set() from gateway.status import _pid_exists remaining = set(pids) deadline = _time.monotonic() + max(timeout, 0.0) while remaining and _time.monotonic() < deadline: for pid in list(remaining): try: if not _pid_exists(pid): remaining.discard(pid) except Exception: remaining.discard(pid) if remaining: _time.sleep(0.25) survivors: set[int] = set() for pid in remaining: try: if _pid_exists(pid): survivors.add(pid) except Exception: pass return survivors def _venv_core_imports_healthy() -> tuple[bool, str]: """Probe the project venv for the core imports the backend needs to boot. Runs a tiny import check inside the venv interpreter (NOT this process — ``hermes update`` may be driven by a different Python). Catches the half-updated-venv state: git checkout current but a dependency sync that failed or was killed partway (e.g. Windows access-denied on a loaded .pyd), leaving imports like ``fastapi``'s new transitive deps missing. Without this probe, ``hermes update`` on a current checkout prints "Already up to date!" and returns without ever re-syncing dependencies — the user's install stays broken no matter how many times they update (ryanc's incident, July 2026). Returns ``(healthy, detail)``. Never raises; unknown states report healthy so a probe failure can't force needless reinstalls. """ venv_dir = _m().PROJECT_ROOT / "venv" venv_python = venv_python_path(venv_dir, windows=_m()._is_windows()) if not venv_python.exists(): # No venv interpreter at all. In a dev checkout that's normal (the # dev may run hermes from any interpreter), so report healthy to # avoid forcing reinstalls. But on a MANAGED install (the Windows # installer / desktop bootstrap stamps `.hermes-bootstrap-complete`, # and an interrupted update leaves `.update-incomplete`), the venv # IS the install — its absence means a repair got interrupted after # the old venv was moved aside, and "Already up to date!" would # gaslight the user while nothing can run. managed_markers = ( _m().PROJECT_ROOT / ".hermes-bootstrap-complete", _m()._update_marker_path(), ) if any(m.exists() for m in managed_markers): return False, f"venv python missing ({venv_python})" return True, "" # Core web/serve imports plus their newest transitive deps. Import (not # just metadata) — a package can have intact dist-info but a missing # module after an interrupted uninstall/install cycle. check = ( "import importlib\n" "mods = ['fastapi', 'uvicorn', 'pydantic', 'openai', 'yaml']\n" "missing = []\n" "for m in mods:\n" " try: importlib.import_module(m)\n" " except Exception as e: missing.append(f'{m}: {e}')\n" "print('\\n'.join(missing))\n" ) try: result = subprocess.run( [str(venv_python), "-c", check], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=60, cwd=_m().PROJECT_ROOT, ) except Exception as exc: logger.debug("venv health probe failed to run: %s", exc) return True, "" missing = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] if result.returncode != 0 and not missing: # Interpreter itself is broken (e.g. deleted stdlib) — that IS unhealthy. detail = (result.stderr or "").strip().splitlines() return False, detail[0] if detail else "venv python failed to run" if missing: return False, "; ".join(missing[:4]) return True, "" def _detect_venv_python_processes( *, exclude_pids: set[int] | None = None ) -> list[tuple[int, str, str]]: """Find live processes running from the project venv's interpreter. The hermes.exe shim guard misses the biggest lock-holder class on Windows: the Desktop app's backend (``python.exe -m hermes_cli.main serve``) and anything else running straight off ``venv\\Scripts\\python (w).exe``. Those processes keep native ``.pyd`` extensions mapped, so a dependency sync mid-update dies with access-denied and strands the venv half-updated (ryanc's brotlicffi/_sodium.pyd incidents, July 2026). Killing them from here is pointless — the Desktop app supervises its backend and respawns it within seconds — so the caller should refuse and tell the user to close the app instead. Returns ``(pid, name, cmdline)`` tuples; empty off-Windows / without psutil / when nothing matches. The calling process and its ancestors are always excluded (a CLI ``hermes update`` itself runs from the venv python). Never raises. """ if not _m()._is_windows(): return [] try: import psutil except Exception: return [] venv_dir = _m().PROJECT_ROOT / "venv" try: venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep except OSError: venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep try: root_prefix = str(_m().PROJECT_ROOT.resolve()).lower().rstrip(os.sep) + os.sep except OSError: root_prefix = str(_m().PROJECT_ROOT).lower().rstrip(os.sep) + os.sep skip: set[int] = set(exclude_pids or set()) skip.add(os.getpid()) try: for anc in psutil.Process().parents(): skip.add(int(anc.pid)) except Exception: pass matches: list[tuple[int, str, str]] = [] try: proc_iter = psutil.process_iter(["pid", "exe", "name", "cmdline", "cwd"]) except Exception: return [] for proc in proc_iter: try: info = proc.info except Exception: continue pid = info.get("pid") exe = info.get("exe") if not exe or pid is None or int(pid) in skip: continue try: exe_norm = str(Path(exe).resolve()).lower() except (OSError, ValueError): exe_norm = str(exe).lower() cmdline_raw = " ".join(info.get("cmdline") or []) cmdline_low = cmdline_raw.lower() cwd_low = str(info.get("cwd") or "").lower().rstrip(os.sep) + os.sep # Primary match: the executable itself lives under this venv # (venv\Scripts\python(w).exe — the desktop backend / gateway case). is_holder = exe_norm.startswith(venv_prefix) # Fallback: uv/base-interpreter trampolines run a python whose exe is # OUTSIDE the venv but which still imports from it and holds its .pyd # files. Catch those by what they're running: a cmdline that references # this venv's path, or a `-m hermes_cli.main ...` invocation tied to # this install (install root in the cmdline or as the working dir). if not is_holder and venv_prefix in cmdline_low: is_holder = True if not is_holder and "hermes_cli.main" in cmdline_low: if root_prefix in cmdline_low or cwd_low.startswith(root_prefix): is_holder = True if not is_holder: continue name = info.get("name") or Path(exe).name # Return the FULL cmdline: callers match against it (the Desktop # preflight's pausable-gateway exemption parses for `gateway run`). # Truncating here cut long managed-runtime interpreter paths before # the `-m hermes_cli.main gateway run` argv, so autostarted gateways # were misreported as blockers and the update dead-ended. Truncate # only at display time. matches.append((int(pid), str(name), cmdline_raw)) return matches # Native-extension modules that pin files inside the venv once imported. If # the updater process itself has any of these loaded, the dependency sync # below cannot rewrite the backing ``.pyd``/``.dll`` — Windows blocks REPLACE # on a mapped image — and the update dies with ``os error 5`` between # uninstall and reinstall, stranding the venv half-updated (#83569). # ``cryptography`` is the canonical case: ``hermes_cli.main`` used to import # it at startup while resolving external secret sources; ``PyYAML``'s # ``_yaml`` C extension is loaded by every CLI process (config parsing). # Keep this guard as defence-in-depth against future eager imports (new # secret sources, plugins absorbed into core, refactors of the startup # order) — but the guard must be HONEST (#86735/#86780/#86781: a preflight # that fired on every run, before the fetch, re-bricked the exact flow it # was meant to protect). Two honesty gates: # # 1. It only fires when the dependency sync would actually REWRITE the # loaded distribution (``_dependency_sync_would_rewrite``): if the # installed version already satisfies the on-disk pyproject pins, uv/pip # will not touch the mapped ``.pyd``, so there is no lock to trip. # 2. It runs AFTER the code swap (git pull / ZIP commit), immediately # before the venv rewrite — so the on-disk pyproject is the NEW one # (gate 1 compares against the right target) and a deferral no longer # strands the user on the old checkout: the next launch's marker # recovery completes the dependency install against the already-updated # pyproject. # # Keys are module prefixes in ``sys.modules``; values are # ``(display name, PyPI distribution name)``. _SELF_LOCKING_NATIVE_MODULES: dict[str, tuple[str, str]] = { "cryptography.hazmat.bindings._rust": ("cryptography (_rust.pyd)", "cryptography"), "yaml._yaml": ("PyYAML (_yaml.pyd)", "pyyaml"), } def _dependency_sync_would_rewrite(dist_name: str) -> bool | None: """Whether ``uv pip install -e .[all]`` would replace *dist_name*'s files. Compares the installed distribution version against every applicable requirement for it in the on-disk ``pyproject.toml`` (base dependencies plus all optional extras). Returns: - ``False`` — installed version satisfies every pin: the resolver will leave the wheel alone, so a mapped extension is NOT at risk. - ``True`` — some pin is not satisfied (or the distribution is missing): the sync will rewrite it. - ``None`` — could not determine (parse failure, unparseable pins). Never raises. Callers treat ``None`` as fail-OPEN (no deferral): a module in the registry can be loaded by every process (PyYAML), so deferring on uncertainty would recreate the #86735 always-firing loop. """ try: from importlib import metadata as _ilmd installed = _ilmd.version(dist_name) except Exception: return True # not installed → the sync will definitely install it try: import tomllib from packaging.requirements import Requirement from packaging.utils import canonicalize_name from packaging.version import Version pyproject = _m().PROJECT_ROOT / "pyproject.toml" data = tomllib.loads(pyproject.read_text(encoding="utf-8")) project = data.get("project") or {} req_strings: list[str] = list(project.get("dependencies") or []) for extra_reqs in (project.get("optional-dependencies") or {}).values(): req_strings.extend(extra_reqs or []) target = canonicalize_name(dist_name) installed_v = Version(installed) saw_pin = False for req_str in req_strings: try: req = Requirement(req_str) except Exception: continue if canonicalize_name(req.name) != target: continue if req.marker is not None and not req.marker.evaluate(): continue saw_pin = True if installed_v not in req.specifier: return True if saw_pin: return False # Not pinned anywhere in pyproject: the resolver may still move it # as a transitive — we cannot cheaply predict that, so stay honest # about the uncertainty. return None except Exception: return None def _detect_self_loaded_native_modules() -> list[str]: """Native venv extensions loaded into THIS process that the sync would rewrite. Returns display names (empty off Windows — POSIX lets a running process keep using an unlinked inode, so self-locking is a Windows-only hazard). A loaded module whose installed version already satisfies the on-disk pyproject pins is NOT reported: the dependency sync will not touch its files, so there is no swap at risk (#86735 — the always-firing variant of this preflight bricked every Windows update). Never raises. """ if not _m()._is_windows(): return [] found = [] for prefix, (display, dist) in _SELF_LOCKING_NATIVE_MODULES.items(): if prefix not in sys.modules: continue # Defer ONLY on a CONFIRMED pending rewrite. An "unknown" result # (unreadable/unparseable pyproject, no pin found) must fail OPEN: # PyYAML is loaded in every CLI process, so treating unknown as # at-risk would re-create the exact always-firing loop this guard's # first version caused (#86735). The downside of a missed deferral # is the pre-existing failure mode — a mid-sync os error 5 that the # marker recovery already handles — which is strictly less harmful # than an update that can never run. if _m()._dependency_sync_would_rewrite(dist) is not True: continue found.append(display) return sorted(set(found)) def _abort_dependency_sync_if_self_locked(gateway_resume=None) -> None: """Defer the venv rewrite when THIS process holds something it must replace. Runs at the last moment before the venv rewrite — after the code swap — so the on-disk pyproject reflects the update target and a deferral leaves the user on NEW code with only the dependency install pending. No-op when nothing at-risk is held. Two hazards, both "this process holds a file the sync must replace", and they end differently because their recoveries differ: - A mapped native extension (``.pyd``). Exit 2 and let the next launch's marker recovery finish the install: that launch runs the install before importing anything heavy, so it maps nothing and the swap succeeds. - The ``hermes.exe`` console shim we were launched from (#88838, #89599). The marker cannot help here — every future ``hermes`` launch is also the shim, so deferring to the next launch defers forever. Hand the install to a child under the venv interpreter and exit, releasing the shim. """ locked = _m()._detect_self_loaded_native_modules() if locked: _m()._defer_update_for_self_lock(locked) if gateway_resume is not None: _m()._resume_windows_gateways_after_update(gateway_resume) sys.exit(2) if _m()._reexec_dependency_sync_off_windows_shim(): if gateway_resume is not None: _m()._resume_windows_gateways_after_update(gateway_resume) sys.exit(0) def _defer_update_for_self_lock(loaded: list[str]) -> None: """Bail out before the dependency sync when the updater holds a lock. The install cannot win this race from inside the locked process — even killing threads would not unmap the image — so defer it: drop the update-incomplete marker (next launch's fresh process completes the install before importing anything heavy), explain, and exit 2 like the other preflight refusals. """ print("✗ This updater process has already loaded native venv modules that") print(" the dependency sync must replace:") for name in loaded: print(f" {name}") print() print(" On Windows a mapped extension cannot be replaced by the process") print(" holding it. The code update has been applied; only the dependency") print(" sync has been deferred: the next `hermes` launch will complete it") print(" in a fresh process before anything imports these modules.") _m()._write_update_incomplete_marker() def _format_venv_python_holders_message(matches: list[tuple[int, str, str]]) -> str: """Explain which venv processes block the update and how to clear them.""" lines = [ "✗ Other Hermes processes are running from this install's venv:", ] for pid, name, cmdline in matches[:6]: hint = "" low = cmdline.lower() if "serve" in low or "dashboard" in low: hint = " ← Hermes Desktop backend (close the desktop app)" elif "gateway" in low: hint = " ← gateway" lines.append(f" PID {pid} {name} {cmdline[:120]}{hint}") if len(matches) > 6: lines.append(f" ... and {len(matches) - 6} more") lines.append("") lines.append( " On Windows these keep native extension files (.pyd) locked, so the" ) lines.append( " dependency update would fail partway and leave a broken install." ) lines.append( " Close the Hermes desktop app / other Hermes terminals, then re-run:" ) lines.append(" hermes update") lines.append(" (or use `hermes update --force-venv` to proceed anyway at your own risk)") return "\n".join(lines) def _venv_launcher_ancestors(pids: list[int]) -> list[int]: """Return venv-interpreter ancestors of *pids* that hold the install open. On Windows a gateway started through the venv shim is a **two-process chain**: ``venv\\Scripts\\python.exe`` (the launcher, which keeps native ``.pyd`` files from the venv mapped) spawns the actual interpreter from uv's managed CPython directory (``AppData\\Roaming\\uv\\python\\...``). The gateway writes its PID file from the *child*, so ``find_gateway_pids()`` — and therefore this module's pause set — only ever sees the uv-side worker. ``_detect_venv_python_processes()`` matches on the venv path prefix, so the guard downstream of the pause sees the *launcher* instead. The two sets are disjoint, which meant a paused gateway still tripped the venv-holder guard and aborted the update every time (the Desktop "venv-blocked: N process(es) hold the install" dead-end, where the reported holder is a gateway the updater believes it already stopped). Walking one hop up from each mapped gateway PID and keeping ancestors that live under the project venv closes the gap. Only the venv-side parent is returned — unrelated ancestors (the Scheduled Task's ``cmd.exe``, an operator's shell) are ignored so we never widen the blast radius beyond the gateway's own launcher. Never raises. """ if not _m()._is_windows() or not pids: return [] try: import psutil except Exception: return [] venv_dir = _m().PROJECT_ROOT / "venv" try: venv_prefix = str(venv_dir.resolve()).lower().rstrip(os.sep) + os.sep except OSError: venv_prefix = str(venv_dir).lower().rstrip(os.sep) + os.sep # Never return ourselves or our own ancestry: a CLI ``hermes update`` # runs from the venv python and would otherwise nominate itself. skip: set[int] = {os.getpid()} try: for anc in psutil.Process().parents(): skip.add(int(anc.pid)) except Exception: pass found: list[int] = [] for pid in pids: try: parent = psutil.Process(int(pid)).parent() except Exception: continue if parent is None: continue ppid = int(parent.pid) if ppid in skip or ppid in found or ppid in set(pids): continue try: exe = (parent.exe() or "").lower() except Exception: continue if exe.startswith(venv_prefix): found.append(ppid) return found def _leftover_pausable_gateway_pids( matches: list[tuple[int, str, str]], ) -> list[int] | None: """PIDs from *matches* when every remaining venv holder is a pausable gateway. ``_pause_windows_gateways_for_update()`` stops every gateway its discovery finds, but the venv-holder guard downstream sees the process table as it is *now*: a gateway respawned by its supervisor (Scheduled Task, login watchdog) inside the pause→guard window, or one started through a spawn path the discovery does not map, still holds venv ``.pyd`` files and would dead-end the update — an abort pointed at exactly the kind of process the pause machinery exists to stop. Holders are classified with the same matcher the Desktop preflight uses to exempt them (``_is_pausable_gateway``), so the preflight's exemption and this guard's tolerance cannot drift apart — matcher drift between two views of the same process table is what produced the launcher/worker dead-end fixed above. The scan captures only a 120-char cmdline prefix, so the live argv is re-read where psutil allows; an unreadable argv falls back to the captured prefix. Returns ``None`` when any holder is not a pausable gateway — an operator REPL, a stray script, or the Desktop backend has no pause machinery downstream, and the guard must keep refusing exactly as before. """ from hermes_cli._scan_venv_blockers import _is_pausable_gateway try: import psutil # type: ignore except Exception: psutil = None pids: list[int] = [] for pid, _name, cmdline in matches: argv = cmdline if psutil is not None: try: argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline except Exception: pass if not _is_pausable_gateway(argv): return None pids.append(int(pid)) return pids def _orphaned_desktop_backend_pids( matches: list[tuple[int, str, str]], ) -> list[int] | None: """PIDs from *matches* when every remaining holder is an ORPHANED backend. The venv-holder guard refuses on the Desktop app's ``serve`` backend by design: while the Desktop is open, killing its backend is futile (the app supervises and respawns it within seconds), so the user must close the app. But in the GUI-updater handoff path the Desktop has *already exited* — by contract it tree-kills its backends and waits for the venv shim before spawning hermes-setup, and the update-in-progress marker parks any relaunched Desktop from spawning a fresh backend (#50238). A ``serve`` backend still holding the venv at that point is a straggler whose supervisor is gone: SIGTERM raced its spawn, or it belongs to a crashed window. Nothing will respawn it, and refusing on it dead-ends the update with "Hermes is still running" while the user stares at zero open windows (ryanc's 2026-08-09 01:59/02:17 failures). A holder qualifies only when BOTH hold: - its cmdline is a Hermes backend (``hermes_cli.main`` + ``serve`` / ``dashboard``), and - its supervising parent is demonstrably gone: the parent PID no longer exists, or the PID was reused (parent created *after* the child). Tree-aware: the scanner can return an orphaned backend AND one of its managed-runtime descendants (the ``.hermes-runtime`` interpreter child) in the same holder set. That descendant has a live parent — the orphaned backend itself — and isn't a ``serve`` cmdline, so per-process rules would refuse a set that is entirely safe to reap. Holders that sit inside an accepted orphan root's tree are therefore folded into that root (only roots are returned; ``taskkill /T`` reaps the descendants). Any other live-parent backend (the Desktop is still open), non-backend holder outside an orphan tree, or unprovable case disqualifies the whole set — the guard must keep refusing exactly as before. Returns ``None`` in that case, or when psutil is unavailable (can't prove orphanhood → refuse). Never raises. """ try: import psutil # type: ignore except Exception: return None def _is_backend(argv_low: str) -> bool: return "hermes_cli.main" in argv_low and ( " serve" in argv_low or " dashboard" in argv_low ) # Pass 1: find orphaned backend ROOTS among the holders. roots: list[int] = [] remaining: list[tuple[int, str]] = [] # (pid, argv_low) still to justify for pid, _name, cmdline in matches: argv = cmdline try: argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline except psutil.NoSuchProcess: # Holder exited between scan and classification — nothing to # reap, nothing blocking. Skip it. continue except Exception: pass low = argv.lower() if not _is_backend(low): remaining.append((int(pid), low)) continue try: proc = psutil.Process(int(pid)) ppid = proc.ppid() parent = psutil.Process(ppid) if ppid else None if parent is not None and parent.is_running(): # PID-reuse check: a "parent" created after its child is a # recycled PID, not the real (dead) supervisor. if parent.create_time() <= proc.create_time(): # Live parent — NOT a root. But it may still be a # descendant of an orphan root: the venv python.exe is # a trampoline that re-execs the uv-managed interpreter # with the SAME backend argv, so the worker half of the # two-process chain lands here. Defer to pass 2 instead # of refusing outright. remaining.append((int(pid), low)) continue except psutil.NoSuchProcess: pass # parent gone → orphan except Exception: return None roots.append(int(pid)) # Pass 2: every non-backend holder must be a descendant of an accepted # orphan root — then it dies with the root's tree reap. Anything else # (operator REPL, stray script) keeps the refusal. root_set = set(roots) for pid, _low in remaining: if not root_set: return None try: ancestors = {int(a.pid) for a in psutil.Process(pid).parents()} except psutil.NoSuchProcess: continue # exited already except Exception: return None if not (ancestors & root_set): return None return roots def _ledger_reapable_backend_pids( matches: list[tuple[int, str, str]], ) -> list[int]: """PIDs positively identified by the spawn ledger as orphaned backends. The strongest rung: instead of inferring lineage from PPIDs or cmdline shape, look each venv holder up in the machine spawn ledger (``hermes_cli.process_identity``). A holder qualifies when ALL of: - its ``(pid, create_time)`` matches a live ledger entry (PID reuse cannot forge this pair); - the entry's purpose is a reapable backend kind (serve/dashboard/ gateway — never interactive processes); - the entry's recorded SPAWNER is provably dead (``spawner_is_dead``). Unlike the heuristic rungs, this is safe in ANY update context — no hand-off contract needed — because the ownership claim is explicit: the process itself declared who supervises it, and that supervisor is gone. Holders not in the ledger are simply not returned (they fall through to the later rungs); they never disqualify the identified ones. Never raises. """ try: from hermes_cli.process_identity import ( REAPABLE_PURPOSES, ledger_entries, spawner_is_dead, ) entries = ledger_entries() except Exception: return [] by_pid = {e.get("pid"): e for e in entries if isinstance(e.get("pid"), int)} roots: list[int] = [] for pid, _name, _cmdline in matches: entry = by_pid.get(int(pid)) if not entry: continue if entry.get("purpose") not in REAPABLE_PURPOSES: continue if spawner_is_dead(entry) is True: roots.append(int(pid)) return roots def _handoff_reapable_backend_pids( matches: list[tuple[int, str, str]], ) -> list[int] | None: """PIDs of Hermes ``serve``/``dashboard`` backends safe to reap during a GUI-updater hand-off, INCLUDING ones with a still-live parent. Complements ``_orphaned_desktop_backend_pids``, which only reaps backends whose supervisor is provably dead. That check returns ``None`` (keep refusing) the moment ANY holder still has a live parent — which is exactly the case that produced the field incident this fixes: a Windows Desktop update hand-off (``update --yes --gateway --force``) left a *swarm* of per-profile ``serve`` backends (mr-tester, probe-inherit, turqoise, …) holding ``cryptography\\_rust.pyd``. Several still had a lingering parent (the tearing-down Electron process, or the two-hop venv launcher→worker chain mid-exit), so the orphan check disqualified the WHOLE set and the update dead-ended — the user saw a 12-minute hang, then force-closed, and the half-done state stranded bot sessions. The hand-off is the safe signal: when the update-incomplete marker is present (the GUI updater claimed it) AND this is a ``--gateway`` hand-off run AND no live Desktop shim (``hermes.exe``) is open, NOTHING legitimate is supervising or respawning a ``serve`` backend from this venv — by the hand-off contract the Desktop tree-kills its backends and parks any relaunch behind the marker (#50238). Any ``serve`` backend still holding the venv here is therefore a leak, live parent or not, and reaping its tree is correct rather than a race. Guarded conservatively: - Only Hermes backends (``hermes_cli.main`` + ``serve``/``dashboard``) from THIS install's venv qualify; a non-backend holder (operator REPL, stray script) disqualifies the whole set → ``None`` (keep refusing), so we never widen the blast radius during a hand-off. - Only runs when the CALLER has confirmed the hand-off context (``args.gateway`` AND a claimed update-incomplete marker AND no live ``hermes.exe`` shim) — outside that gate this function is never called and the stricter orphan-only path stands. - psutil unavailable → ``None`` (can't re-read argv to classify → refuse). Returns the backend root PIDs to tree-reap, or ``None`` to leave the decision to the caller's existing rungs. Never raises. """ try: import psutil # type: ignore except Exception: return None def _is_backend(argv_low: str) -> bool: return "hermes_cli.main" in argv_low and ( " serve" in argv_low or " dashboard" in argv_low ) roots: list[int] = [] for pid, _name, cmdline in matches: argv = cmdline try: argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline except psutil.NoSuchProcess: # Exited between scan and classification — nothing to reap. continue except Exception: pass if not _is_backend(argv.lower()): # A non-backend holder during a hand-off is unexpected; refuse the # whole set rather than reap something we cannot justify. return None roots.append(int(pid)) return roots or None def _stop_process_trees(pids: list[int]) -> None: """Force-stop each PID with its full child tree (Windows). ``taskkill /T /F`` mirrors the Desktop's ``forceKillProcessTree`` and install.ps1's venv sweep: stopping only the parent can leave a managed ``.hermes-runtime`` interpreter child alive and holding the install open (#70026). Best effort; never raises. """ for pid in pids: try: subprocess.run( ["taskkill", "/PID", str(int(pid)), "/T", "/F"], check=False, capture_output=True, ) except Exception as exc: logger.debug("Could not stop process tree %s: %s", pid, exc) def _pause_windows_gateways_for_update() -> dict | None: """Stop running Windows gateways before mutating the checkout or venv. Windows scheduled/startup gateways run through pythonw.exe, so the generic hermes.exe concurrent-instance guard does not see them. They still import from the checkout and can keep files locked while ``git`` or ``uv`` updates the install. Stop only PIDs that the gateway discovery code identifies. """ if not _m()._is_windows(): return None try: from gateway.status import terminate_pid from hermes_cli.gateway import ( _capture_gateway_argv, _get_restart_drain_timeout, find_gateway_pids, find_profile_gateway_processes, ) except Exception as exc: logger.debug("Could not prepare Windows gateway pause for update: %s", exc) return None try: running_pids = list(dict.fromkeys(find_gateway_pids(all_profiles=True))) except Exception as exc: logger.debug("Could not discover Windows gateway PIDs before update: %s", exc) return None if not running_pids: # No gateway is running right now, but the user may have installed an # autostart entry (Scheduled Task or Startup-folder login item) — that # is an explicit "I want a gateway" signal. A gateway that died between # updates (e.g. the spawning terminal/TUI closed, taking its child with # it) would otherwise never come back: the autostart entry only fires on # the next login, and the update flow's resume path only relaunched # gateways that were running when the update began. Cold-start one after # the update so an installed gateway is actually up post-update. Users # who run gateway-less (no autostart entry) get nothing forced on them. try: from hermes_cli import gateway_windows if gateway_windows.is_installed(): return { "resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": [], "cold_start_if_installed": True, } except Exception as exc: logger.debug( "Could not check Windows gateway autostart state before update: %s", exc, ) return None profile_processes = {} try: profile_processes = { proc.pid: proc for proc in find_profile_gateway_processes() } except Exception as exc: logger.debug("Could not map Windows gateway PIDs to profiles: %s", exc) profiles: dict[str, int] = {} mapped_pids = [] for pid in running_pids: proc = profile_processes.get(pid) if proc is None: continue profiles[str(proc.profile)] = int(pid) mapped_pids.append(int(pid)) _write_update_planned_stop_marker(Path(proc.path), int(pid)) # Resolve each mapped worker's venv-side launcher BEFORE draining: the # drain stops tracking a PID exactly when it dies, so a gracefully # drained worker is gone by the time the wait returns — and a dead pid's # parent cannot be recovered (psutil raises NoSuchProcess). The snapshot # is stopped after the drain alongside the survivors. # # Why launchers matter: the drain targets the PID that wrote the PID # file (the uv-side worker). On Windows that worker's parent is usually # the venv-side ``python.exe`` launcher, which keeps venv ``.pyd`` files # mapped and is what ``_detect_venv_python_processes()`` reports # downstream. Left alive, it trips the venv-holder guard and aborts the # update even though the gateway itself is stopped. launcher_pids = _m()._venv_launcher_ancestors(mapped_pids) print("→ Stopping Windows gateway process(es) before updating Hermes...") try: drain_timeout = max(float(_get_restart_drain_timeout()), 1.0) except Exception: drain_timeout = 10.0 survivors = _m()._wait_for_windows_update_gateway_exit( mapped_pids, timeout=drain_timeout, ) unmapped_pids = [pid for pid in running_pids if pid not in profile_processes] # Snapshot each unmapped gateway's command line *before* we force-kill it, # so ``_resume_windows_gateways_after_update`` can respawn it by replaying # its own argv. Unmapped gateways are ones with no profile→PID-file mapping # — e.g. a Windows Scheduled Task running ``pythonw.exe -m hermes_cli.main # gateway run``. Without this snapshot they were force-killed and never # restarted (the "Restart manually after update" dead-end from #50090). unmapped: list[dict] = [] for pid in unmapped_pids: argv = None try: argv = _capture_gateway_argv(int(pid)) except Exception as exc: logger.debug("Could not capture argv for unmapped gateway %s: %s", pid, exc) unmapped.append({"pid": int(pid), "argv": argv}) # Stop drain survivors, unmapped gateways, and the pre-drain launcher # snapshot. ``terminate_pid(force=True)`` is a tree kill, so a launcher # that outlived its worker takes any stragglers with it; a launcher that # already exited with its drained worker raises ProcessLookupError below # and is skipped. force_killed = [] for pid in sorted(set(survivors).union(unmapped_pids).union(launcher_pids)): try: terminate_pid(int(pid), force=True) force_killed.append(int(pid)) except (ProcessLookupError, PermissionError, OSError): pass if profiles: print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}") if force_killed: print(f" → Force-stopped {len(force_killed)} gateway process(es)") if unmapped_pids: respawnable = sum(1 for u in unmapped if u.get("argv")) print( f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping" ) if respawnable < len(unmapped_pids): # Some had no recoverable command line (psutil missing, access # denied, already gone): those still need a manual restart. print(" Restart manually after update: hermes gateway run") return { "resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped, } def _cold_start_windows_gateway_after_update() -> None: """Start a fresh detached gateway after update when one is installed but down. Invoked from ``_resume_windows_gateways_after_update`` for the ``cold_start_if_installed`` case: no gateway was running when the update began, but an autostart entry (Scheduled Task / Startup-folder login item) is installed, signalling the user wants a gateway. Unlike the relaunch paths — which watch an old PID and respawn once it exits — this is a direct fresh spawn via the same hidden-console + breakaway path that ``hermes gateway start`` uses (``gateway_windows._spawn_detached``). Best-effort and idempotent: re-checks that nothing is running first so a concurrent start (e.g. the autostart entry firing) can't produce a duplicate gateway. A successful ``Popen`` only proves the process was created, not that it survived (e.g. a Windows job object denying breakaway kills it before it logs anything — #84185). So the success line is gated on the same post-spawn liveness poll every other ``_spawn_detached`` caller uses (``gateway_windows._report_gateway_start``), instead of being printed unconditionally from the returned PID. """ if not _m()._is_windows(): return try: from hermes_cli import gateway_windows from hermes_cli.gateway import find_gateway_pids except Exception as exc: logger.debug("Could not load Windows gateway cold-start helpers: %s", exc) return # Re-check liveness right before spawning — between pause and resume the # autostart entry may have already brought a gateway up, or a leftover # process may have re-registered. Don't double-start. try: if list(find_gateway_pids(all_profiles=True)): return except Exception as exc: logger.debug("Could not re-check gateway liveness before cold-start: %s", exc) return try: pid = gateway_windows._spawn_detached() except Exception as exc: logger.debug("Could not cold-start Windows gateway after update: %s", exc) return if pid: print() gateway_windows._report_gateway_start(f"cold-start after update (PID {pid})") def _for_each_systemd_gateway_unit( list_units_stdout: str, *, process_unit, on_unit_timeout, ) -> None: """Process each ``hermes-gateway*.service``/``hermes-serve*.service`` unit from ``systemctl list-units``. ``subprocess.TimeoutExpired`` raised by ``process_unit`` is isolated to that unit via ``on_unit_timeout`` so one wedged systemctl call cannot abort the rest of the fleet (#68523). """ for line in (list_units_stdout or "").strip().splitlines(): parts = line.split() if not parts: continue unit = parts[0] if not unit.endswith(".service"): continue # list-units is already pattern-filtered, but keep the name gate so a # stray non-gateway/serve line cannot enter the restart path. # ``unit.startswith("hermes-serve")`` alone would also accept the # unrelated ``hermes-server.service`` — require the exact base unit # or the hyphenated profile family instead (review on #83595). if not ( unit == "hermes-gateway.service" or unit.startswith("hermes-gateway-") or unit == "hermes-serve.service" or unit.startswith("hermes-serve-") ): continue svc_name = unit.removesuffix(".service") try: process_unit(svc_name) except subprocess.TimeoutExpired as exc: on_unit_timeout(svc_name, exc) def _service_unit_supports_graceful_sigusr1_restart(svc_name: str) -> bool: """Whether *svc_name* wires SIGUSR1 to a graceful drain-then-restart. Only ``hermes-gateway*`` units run ``gateway/run.py``, which installs the SIGUSR1 handler. ``hermes-serve*`` units (#83438) don't, so sending them SIGUSR1 would just invoke the default terminate action and burn the full drain budget waiting for an exit that was never graceful — go straight to the blunt ``systemctl restart`` path for those instead. Uses the same strict exact/hyphenated shape as the unit-name gate in ``_for_each_systemd_gateway_unit`` so a hypothetical near-prefix unit (``hermes-gateway-helper`` is fine — profile units are ``hermes-gateway-`` — but ``hermes-gatewayd``-style names are not) can't be sent a SIGUSR1 it doesn't handle. """ return svc_name == "hermes-gateway" or svc_name.startswith("hermes-gateway-") def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None: """Print an explicit incomplete-update warning for unrestarted units.""" if not failed_units: return # Preserve discovery order while de-duplicating. seen = set() ordered = [] for name in failed_units: if name in seen: continue seen.add(name) ordered.append(name) print() print("⚠ Update incomplete — some units were not restarted:") for name in ordered: print(f" - {name}") print(" Skipped units may still be running pre-update code (mixed") print(" sys.modules). Restart them manually, then verify:") print(" hermes gateway status") if any(not name.startswith("ai.hermes.") for name in ordered): print(" systemctl --user restart # user-scope") print(" sudo systemctl restart # system-scope") if any(name.startswith("ai.hermes.") for name in ordered): print(" launchctl kickstart -k gui/$UID/