
    sj7                       U d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlZddlZddlZddlmZmZ 	 ddlZn # e$ r dZ	 ddlZn# e$ r dZY nw xY wY nw xY wddlmZ ddlmZmZmZmZ ej                             d e! ee"          j#        j#                             ddl$m%Z% ddl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z- dd	l.m/Z/ dd
l0m1Z2 ddl3m4Z4 ddl5m6Z6m7Z7  e	j8        e9          Z:ddZ;d Z<de!fdZ=de>de!fdZ?de>de!dz  de!fdZ@ G d deA          ZBde>deCe!         fdZDdeCe!         de>deCe!         fdZEde>de>deCe!         dz  fdZFde>de>de!de>dz  fdZG eHh d           ZIi d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@ZJd<dAiZKddBlLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZW ddClXmYZYmZZZm[Z[ dDZ\dEe!de]fdFZ^da_eej`        ja                 ebdG<   daceed         ebdH<    ee            ZfeeebdI<   i Zge>e!e>eheiee!         ef         f         f         ebdJ<    ejj                    Zki Zle>ebdK<   i Zme>ebdL<    eh            ZndaoedebdM<   g ZpeCebdN<   dOZqdPZr ee            ZseeebdQ<    G dR dSe          Zt G dT dU          ZuddWZvdXe!de]fdYZwdXe!ddfdZZxdeyfd[Zzi Z{e>ebd\<   d]e!deey         fd^Z|de>deey         fd_Z}de>fd`Z~dXe!dae!dbeydceyddf
ddZddeeeC         deCfdfZddgdhe!dieee         deCfdjZddXe!dkeeh         de]fdlZddXe!dkeeh         de]fdmZdaeej`        ja                 ebdn<    G do dp          Z e            ZdqZdrZdeyfdsZdeyfdtZdueed         dej`        ja        fdvZdej`        ja        fdwZd	dxZ ej        e           defdyZde!fdzZd{e>ddfd|Zdd}ee         de]fd~ZdZedz  ebd<   defdZdeieef         fdZdede]fdZdEe!de]fdZd}ede]fdZd	dZde!de!fdZde>dee>         fdZdde>dee>         de]fdZde>de!de!dee!         de]f
dZ	 	 d
ddde>de!de!de!dee!         dee!         de]ddfdZde>de!dee!         fdZ	 	 	 dde>de!de!de!de!dee!         de]dee!         ddfdZdddde>de!de!de!de]dee!         dee!         dee!         de]fdZde>de!fdZde!de!fdZde!de]fdZde!de!fdZde!fdZde!de!fdZde!de!fdZde!dee!         fdZd ZdeedeefdZdeCe>         fdZde>de]fdZde>fdZde>de!dee>         fdZde!fdZ eHdh          Zde!dee!         fdZde>dee>         fdZde>dee>         fdZ eHh d          Z eHh d          Z	 dde!deCde>dz  de>deCf
dZde]fdZdedededXe!de]f
dZd
de>de!dee!         fdZdZeZdrZedz  ZdedfdZdZdedfdZdede>e!e!f         fdĄZde!deie!e>e!e!f         f         fdƄZdej        ddfdȄZdej        ddfdɄZde!de>e!e!f         de!deCe!         fd̄Z	 	 d
de!dee!         deet         deie]e!f         fdτZ	 	 d
de>de!dee!         deet         deie]e!f         f
dЄZde!de]fd҄Z	 	 d
de>deei         dee!         de!fdՄZdddd֜de!de>de]de]dee!         de!fdۄZde>ddfd܄ZdZdZdZdZd}ede]fdZde>de]fdZde>de>dee!         fdZde>dee!         fdZde>dee!         fdZde>de>dee!         fdZdeyfdZdddXe!de!deey         de]fdZ G d d          Zddddde>deeC         dee!         deet         deie]e!e!ee!         f         f
dZdddXe!deey         ddfdZde>de]fdZddddddde>de]dee!         deet         de]f
dZdddddddde>de]dee!         deet         deeh         de]fdZd	dZ G d de          Zde>fdZdZdaeey         ebd<   	 	 	 	 dddde]de]fdZe9dk    r ed            dS dS (  a  
Cron job scheduler - executes due jobs.

Provides tick() which checks for due jobs and runs them. The gateway
calls this every 60 seconds from a background thread.

Uses a file-based lock (~/.hermes/cron/.tick.lock) so only one tick
runs at a time if multiple processes overlap.
    N)datetimetimezonePath)AnyListOptionalProtocol)get_hermes_home)windows_hide_flags)_expand_env_varscron_model_drift_axescron_model_drift_guard_enabledload_config!resolve_cron_model_drift_defaults)get_fallback_chainnow)request_hard_interrupt)"enter_non_dispatcher_owned_context!exit_non_dispatcher_owned_contextfutureconcurrent.futures.Futurereturnc                     	 |                                  }||                                 dS dS # t          $ r Y dS w xY w)u+  Done-callback: close a SessionDB whose constructor finished after run_job's timeout.

    When ``run_job``'s SessionDB init times out, the worker thread is abandoned
    (``shutdown(wait=False)``) so the job can proceed without a session store.
    If the constructor later completes inside that abandoned worker, the
    Future's result — an open SessionDB holding .db / WAL / SHM file handles —
    would be orphaned and never closed, leaking descriptors until EMFILE
    (#72782).  This callback retrieves and closes that eventual late result.
    N)resultclose	Exception)r   dbs     2/home/agent/.hermes/hermes-agent/cron/scheduler.py_close_late_session_db_resultr!   C   sS    ]]__>HHJJJJJ >   s   *0 
>>c                    | r|sdS |pd                                 }|sdS 	 |                     ||           |S # t          $ rC t          | dd          }|  ||          }|r||k    r |                     ||           |cY S w xY w)a  Robustly title a finished cron session before it is closed.

    Centralizes the title write so the cron finally block can guarantee a
    non-blank, unique title is persisted before end_session()/close() tear
    the connection down (issues #50535, #50536, #50537):

    - #50535: never leaves the session blank. base_title already carries a
      cron-id fallback for nameless jobs; this also guards a failed write.
    - #50537: a duplicate title makes set_session_title raise ValueError (the
      unique-title index). Recover by appending a #N suffix via
      get_next_title_in_lineage() when supported, instead of swallowing the
      error and ending up untitled. If lineage dedup is unavailable, raise.
    - #50536: this runs synchronously in the cron finally block ahead of the
      session close, so no in-flight title write can race the close.

    Returns the title actually persisted, or None if nothing could be set.
    N get_next_title_in_lineage)stripset_session_title
ValueErrorgetattr)
session_db
session_id
base_titletitlenext_title_fndedupeds         r    _set_cron_session_titler/   U   s    $  Z t2$$&&E t$$Z777 
 
 
  
,GNN -&& 	'U**$$Z999
s   : A
BBc                  t    	 t                      pi } t          |           }n# t          $ r Y dS w xY w|rdS 	 dS )a  Wording for the fallback-chain clause of a provider-failure message.

    "Fallback chain was exhausted or unavailable." used to fire
    unconditionally on every provider failure, which implies a fallback was
    attempted and failed. Most installs have fallback_providers: [] (no
    chain configured at all), so that wording was actively misleading: it
    sent the operator looking for why a fallback "failed" when none was
    ever attempted. Distinguish the two cases explicitly.

    Fails open to the original ambiguous-but-safe wording if config can't be
    read (e.g. mid-shutdown, permissions) -- never let a lookup error crash
    failure-message generation itself.
    z,Fallback chain was exhausted or unavailable.u   No fallback chain configured — add one with `hermes fallback add`, or set a cron fleet default via `cron.model` + `cron.model_provider` in config.yaml.)r   r   r   )cfgchains     r    _fallback_chain_phraser3   |   se    >mm!r"3'' > > >===> >==	 s   " 
00jobc                     |                      d          pi                      d          }|dvrdS 	 t                      pi }t          t          |t                    r|                     d          pi ni                      dd                    }n# t
          $ r d}Y nw xY w|dk    rdS t          |                      d	          pd          d
z   }||k     rdS |                      d          p|                      d          pd}d| d| dS )uS  Return a review nudge when a recurring job keeps failing, else "".

    Inspired by Poke (poke.com), which "encourages users to review recurring
    automations that haven't been acted upon": once a recurring job has failed
    several runs in a row, the per-run failure ping stops being information and
    starts being noise — the useful message is "this automation needs your
    attention (fix, pause, or remove it)".

    The streak counter (``failure_streak``) is persisted by
    ``cron.jobs.mark_job_run`` and reset on any successful run. Because the
    failure message is delivered BEFORE ``mark_job_run`` records this run, the
    prospective streak for the current failure is stored+1.

    Threshold config: ``cron.failure_nudge_threshold`` (default 3, ``0``
    disables the nudge). One-shot jobs never nudge — they don't recur.
    schedulekind>   cronintervalr#   r8   failure_nudge_threshold   r   failure_streak   nameidzthis jobz
This job has failed u^    runs in a row — worth a review. Fix its prompt/config, or pause it with `hermes cron pause z3` (resume/remove also available) to stop the noise.)getr   int
isinstancedictr   )r4   schedule_kindr1   	thresholdstreakjob_refs         r    _failure_streak_nudgerH      sI   " WWZ((.B33F;;M000rmm!r(23(=(=Ecggfoo#2JJ)1 
 
		
    			A~~r)**/a0014F	rggfoo<<*G	< 	< 	<FM	< 	< 	<s   AB B! B!errorc                     |                      d          p|                      d          pd}|pd                                }|                                }d|v r,d|v rd}n|                      d          pd}d	| d
}d| d| S |                      d           }|                    d          rd| dS |rDt	          j        d|          sd|v sd|v r'd}d|v rd}nd|v rd}d| d| dt                       dS t	          j        d|          rd| dS d|v rd|v sd|v rd| dS |r!d|v sd|v sd |v rd| d!t                       dS |r0t	          j        d"|          st	          j        d#|          rd| d$S t	          j        d%d&|d'd(                   }	t	          j        d)d*|	                                          }	t          |	          d+k    r|	d'd,         	                                d-z   }	d| d.|	 S )/a  Return a compact one-line failure message for chat delivery.

    Full details stay in the cron output directory and the logs. Chat should
    show the operator what broke without dumping provider JSON, retry noise, or
    stack traces into the delivery channel.
    r>   r?   cron jobzunknown errorzDskipped to prevent unintended spend: global inference config driftedzfinite one-shot job is consumedzqThis finite one-shot is consumed; create a new one-shot job at a future time with an explicit provider and model.z<job_id>zAOn the host running Hermes, pin it explicitly: `hermes cron edit z( --provider <provider> --model <model>`.   ⚠️ Cron 'z8' skipped before inference to prevent unintended spend. no_agentzscript timed outzT' failed: script timed out. No model was invoked. Full details saved in cron output.z\b429\bz
rate limitzusage limitzweekly usage limitquotazquota limitz' failed: provider z. z# Full details saved in cron output.zidle for \d+s\s*\(limit \d+s\)u   ' failed: the job itself stalled — no tool/API activity for the configured inactivity window. Not a provider or fallback-chain issue; check what the job was doing when it went quiet. Full details saved in cron output.terminal_cwdlock	timed outu  ' failed: could not acquire the scheduler's working-directory lock — another cron job (a workdir writer or long-running readers) held it too long. Not a provider or fallback-chain issue; stagger the holder's schedule or remove its workdir. Full details saved in cron output.readtimeouttimeoutz' failed: provider timeout. zauthenticat|authorizz\b(401|403)\bzK' failed: provider authentication error. Full details saved in cron output.z8^(RuntimeError|Exception|ValueError|HTTPStatusError):\s*r#   Ni  z\s+       z...z
' failed: )
r@   r%   lower
startswithresearchr3   sublenrstrip)
r4   rI   job_nametextrW   remediationjob_idprovider_reachablereasoncleaneds
             r    $_summarize_cron_failure_for_deliveryre      sC    wwv=#''$--=:H$_++--DJJLLEMQVVV,55E K
 WWT]]0jF$%+$ $ $ /H / /!,/ /	
2 !WWZ000 *++ 
GH G G G	
  

	*d##
'3u'<'<QV@V@V5(()FF"F1H 1 1 1 1%''1 1 1	
, 
y2E:: 
8H 8 8 8	
 FeOO{e7K7K>H > > >	
  
+"6"6)u:L:L1H 1 1%''1 1 1	
  

	)511
57Y?OQU5V5V
1H 1 1 1	
 fC
D$K G fVS'**0022G
7||c$3$-&&((508888w888    c                       e Zd ZdZdS )CronPromptInjectionBlockeda  Raised by _build_job_prompt when the fully-assembled prompt trips the
    injection scanner. Caught in run_job so the operator sees a clean
    "job blocked" delivery instead of the scheduler crashing.

    Assembled-prompt scanning (including loaded skill content) plugs the
    gap from #3968: create-time scanning only covers the user-supplied
    prompt field; skill content loaded at runtime was never scanned, so a
    malicious skill could carry an injection payload that reached the
    non-interactive (auto-approve) cron agent.
    N)__name__
__module____qualname____doc__ rf   r    rh   rh   Y  s        	 	 	 	rf   rh   r1   c                 p   | pi                      d          pi }|                     d          rddg}ng d}| pi                      d          pi }ddlm}  ||                     d	                    }|D ]>}t          |                                          }|r||vr|                    |           ?|S )
u  Toolsets a cron-spawned agent must never receive.

    Two toolsets are always disabled in cron context regardless of config:
      - ``messaging`` — interactive, needs a live gateway session
      - ``clarify`` — interactive, blocks waiting for user input

    ``cronjob`` is policy-denied by default (loop prevention, not a security
    boundary) and config-gated: setting ``cron.allow_agent_scheduling: true``
    in config.yaml drops it from the base denylist so cron-spawned agents may
    manage the user's cron table. The gate only removes the built-in policy
    denial — it never overrides the user denylist below.

    User-level ``agent.disabled_toolsets`` from config.yaml is layered on top
    so per-job ``enabled_toolsets`` cannot bypass policy that applies to
    ordinary agent runs (#25752 — LLM-supplied enabled_toolsets was widening
    past config.yaml's denylist).
    r8   allow_agent_scheduling	messagingclarify)cronjobrp   rq   agentr   )parse_config_string_listdisabled_toolsets)r@   agent.skill_utilsrt   strr%   append)r1   cron_cfgdisabled	agent_cfgrt   user_disabledr>   s          r    _resolve_cron_disabled_toolsetsr}   f  s    $ 	rv&&,"H||,-- 7+666((.BI::::::,,Y]];N-O-OPPM " "4yy   	"D((OOD!!!Orf   per_jobc                     d | D             }d| v r|S ddl m}  ||          }t          |          |z  r|S t          |          D ]}||vr|                    |           |S )a  Layer enabled MCP servers onto a per-job ``enabled_toolsets`` allowlist.

    A per-job list scopes the *native* toolsets, but on its own it silently
    drops every MCP server: ``discover_mcp_tools()`` registers the tools into
    the global registry, yet ``get_tool_definitions(enabled_toolsets=...)``
    only keeps toolsets named in the list. The agent then rejects every
    ``mcp_*`` call with "Unknown tool". This restores parity with
    ``_get_platform_tools`` MCP semantics:

      * ``no_mcp`` sentinel present  -> no MCP servers (sentinel stripped)
      * one or more MCP server names already listed -> treat as an allowlist,
        add nothing further (the user named exactly the servers they want)
      * otherwise -> union in every globally-enabled MCP server
    c                     g | ]
}|d k    |S )no_mcprm   ).0ts     r    
<listcomp>z4_merge_mcp_into_per_job_toolsets.<locals>.<listcomp>  s    222AAMMaMMMrf   r   r   )enabled_mcp_server_names)hermes_cli.tools_configr   setsortedrx   )r~   r1   r   r   enabled_mcpr>   s         r     _merge_mcp_into_per_job_toolsetsr     s     32222F7 A@@@@@**3//K
6{{[  {##    vMM$Mrf   c                    |                      d          }|rt          t          |          |pi           S 	 ddlm} t           ||pi d                    S # t          $ r&}t                              d|           Y d}~dS d}~ww xY w)u  Resolve the toolset list for a cron job.

    Precedence:
    1. Per-job ``enabled_toolsets`` (set via ``cronjob`` tool on create/update).
       Keeps the agent's job-scoped toolset override intact — #6130. Enabled
       MCP servers are layered on per ``_merge_mcp_into_per_job_toolsets`` so a
       native-toolset allowlist does not silently strip MCP tools.
    2. Per-platform ``hermes tools`` config for the ``cron`` platform.
       Mirrors gateway behavior (``_get_platform_tools(cfg, platform_key)``)
       so users can gate cron toolsets globally without recreating every job.
    3. ``None`` on any lookup failure — AIAgent loads the full default set
       (legacy behavior before this change, preserved as the safety net).

    _DEFAULT_OFF_TOOLSETS ({moa, homeassistant, rl}) are removed by
    ``_get_platform_tools`` for unconfigured platforms, so fresh installs
    get cron WITHOUT ``moa`` by default (issue reported by Norbert —
    surprise $4.63 run).
    enabled_toolsetsr   )_get_platform_toolsr8   zHCron toolset resolution failed, falling back to full default toolset: %sN)	r@   r   listr   r   r   r   loggerwarning)r4   r1   r~   r   excs        r    _resolve_cron_enabled_toolsetsr     s    & gg())G J/WsybIII??????))#)V<<===   V	
 	
 	
 ttttts    A 
B	#BB	modelc           	         ddl m}m} |                     d          }| ||          }|2t                              d|                     dd          |           |S t                              d|                     dd          ||                     dd                      |t          |t                    r|ni t          |                    S )	uy  Resolve the effective reasoning config for a cron run.

    Precedence: per-job ``reasoning_effort`` pin (validated at the store
    choke point, ``cron/jobs.py::_normalize_reasoning_effort``) wins outright
    over config resolution — both the global ``agent.reasoning_effort`` and
    per-model ``agent.reasoning_overrides``. The pin is model-independent by
    design: it also governs an auth-fallback model swap, and capability
    clamping for the model that actually runs stays owned by the provider
    transports at send time (exactly like config-set effort).

    A value that no longer parses (hand-edited jobs.json) logs a warning and
    falls back to config resolution — a bad pin must degrade the run's
    thinking level, never kill the tick.

    Absent/None pin returns ``resolve_reasoning_config(cfg, model)``
    byte-identical, preserving pre-feature behavior.
    r   )parse_reasoning_effortresolve_reasoning_configreasoning_effortNz-Job '%s': using per-job reasoning_effort '%s'r?   ?u   Job '%s': invalid stored reasoning_effort %r — ignoring the pin and falling back to config resolution. Fix with `cronjob action=update job_id=%s reasoning_effort=<level>` (valid: none, minimal, low, medium, high, xhigh, max, ultra).)
hermes_constantsr   r   r@   r   infor   rB   rC   rw   )r4   r1   r   r   r   pinnedparseds          r    _resolve_job_reasoning_configr     s    $ RQQQQQQQWW'((F''//KK?c""  
 M> GGD#GGD#	
 	
 	
 $#:c4+@+@$HCCb#e**UUUrf   >   smsemailqqbotslackwecomfeishumatrixsignalweixindiscordwebhookyuanbaodingtalktelegramwhatsapp
mattermostbluebubbleshomeassistantwecom_callbackr   MATRIX_HOME_ROOMr   TELEGRAM_HOME_CHANNELr   DISCORD_HOME_CHANNELr   SLACK_HOME_CHANNELr   SIGNAL_HOME_CHANNELr   MATTERMOST_HOME_CHANNELr   SMS_HOME_CHANNELr   EMAIL_HOME_ADDRESSr   DINGTALK_HOME_CHANNELr   FEISHU_HOME_CHANNELr   WECOM_HOME_CHANNELr   WEIXIN_HOME_CHANNELr   BLUEBUBBLES_HOME_CHANNELr   QQBOT_HOME_CHANNELr   WHATSAPP_HOME_CHANNELwhatsapp_cloudWHATSAPP_CLOUD_HOME_CHANNELQQ_HOME_CHANNEL)advance_next_runsclaim_dispatchclaim_job_for_firefire_claim_fenceclear_run_claimget_due_jobsheartbeat_fire_claimheartbeat_run_claimmark_job_runsave_job_outputuse_cron_store)create_executionfinish_executionmark_execution_runningz[SILENT]r_   c                 $    ddl m}  ||           S )a(  Return True when a cron final response should suppress delivery.

    Recognizes the bracketed ``[SILENT]`` sentinel (whole-response, first line,
    or last line) plus the bracketless ``SILENT`` / ``NO_REPLY`` / ``NO REPLY``
    variants the model emits when it drops the brackets (#51438, #46917).
    Whitespace-trimmed and case-insensitive.  A token buried mid-sentence is
    treated as real content and delivered.

    Delegates to the shared autonomous-lane matcher in
    :mod:`gateway.response_filters` (also used by the webhook adapter).
    r   )is_autonomous_silence_response)gateway.response_filtersr   )r_   r   s     r    _is_cron_silence_responser   8  s(     HGGGGG))$///rf   _parallel_pool_parallel_pool_max_workers_running_job_ids_running_fire_owners_running_since_running_futures_forced_release_count_forced_releases   g      >@_interrupted_job_idsc                   &    e Zd ZdZdefdZddZdS )_CancelEventLikezpStructural type for cancellation sources (``threading.Event`` and
    ``_CombinedCancelEvent`` both satisfy it).r   c                     d S Nrm   selfs    r    is_setz_CancelEventLike.is_set        rf   Nc                     d S r   rm   r   s    r    r   z_CancelEventLike.set  r   rf   r   N)ri   rj   rk   rl   boolr   r   rm   rf   r    r   r     s9        2 2 "!!!!rf   r   c                   B    e Zd ZdZded         ddfdZdefdZd	dZdS )
_CombinedCancelEventap  Duck-typed ``threading.Event`` that ORs several cancellation sources.

    ``run_one_job`` already derives a ``lost_ownership`` event from the
    fire-claim heartbeat; transports (dashboard webhook drain, API server
    shutdown) contribute their own per-task event. The worker only ever
    calls ``is_set()`` / ``set()``, so a tiny wrapper beats a pump thread.
    eventsr   r   Nc                 (    d |D             | _         d S )Nc                     g | ]}||S r   rm   r   events     r    r   z1_CombinedCancelEvent.__init__.<locals>.<listcomp>  s    GGG%U5F5F5F5Frf   )_events)r   r   s     r    __init__z_CombinedCancelEvent.__init__  s    GG6GGGrf   c                 >    t          d | j        D                       S )Nc              3   >   K   | ]}|                                 V  d S r   )r   r   s     r    	<genexpr>z._CombinedCancelEvent.is_set.<locals>.<genexpr>  s*      <<e5<<>><<<<<<rf   )anyr   r   s    r    r   z_CombinedCancelEvent.is_set  s!    <<t|<<<<<<rf   c                 B    | j         D ]}|                                 d S r   )r   r   )r   r   s     r    r   z_CombinedCancelEvent.set  s,    \ 	 	EIIKKKK	 	rf   r   )	ri   rj   rk   rl   r	   r   r   r   r   rm   rf   r    r   r     s{         H); < H H H H H= = = = =     rf   r   frozenset[str]c                      t           5  t          t          t                                          z            cddd           S # 1 swxY w Y   dS )u  Thread-safe snapshot of cron job IDs currently executing.

    A job ID is a member from the moment ``_submit_with_guard`` dispatches
    it onto the parallel/sequential pool until ``_process_job`` returns —
    i.e. for the job's *entire* run, tool calls included, not just the
    ticker's dispatch instant.

    The gateway shutdown path (``gateway/run.py::GatewayRunner.
    _drain_active_agents``) reads this to treat in-flight cron work as
    active the same way it already treats in-flight chat sessions via
    ``_running_agents`` — cron jobs run through their own thread pool here,
    entirely outside that dict, so without this the drain is structurally
    blind to them (#60432).
    N)_running_lock	frozensetr   r   keysrm   rf   r    get_running_job_idsr     s     
 I I),@,E,E,G,GGHHI I I I I I I I I I I I I I I I I Is   .AA
Ara   c                     t           5  | t          v r	 ddd           dS t                              |            t          j                    t          | <   t
          t          | <   	 ddd           dS # 1 swxY w Y   dS )u2  Atomically add ``job_id`` to the in-flight running set.

    Returns False (without registering) when the job is already mid-run —
    the caller must skip the fire. This is the single dedupe owner shared by
    the ticker's ``_submit_with_guard`` and manual runs
    (``tools/cronjob_tools``): the fire claim alone cannot prevent a
    double-fire because its TTL (300s) is routinely outlived by real jobs,
    after which a manual ``cronjob(action='run')`` would claim successfully
    and run the same job concurrently (idea from #53395 by @izumi0uu).

    Registration also makes the run visible to ``get_running_job_ids`` (the
    gateway shutdown drain, #60432) and ``mark_running_jobs_interrupted``.
    Callers MUST pair a successful registration with
    ``release_running_job`` in a ``finally`` block.
    NFT)r   r   addtimer   _FUTURE_PENDINGr   ra   s    r    try_register_running_jobr    s      
  %%%        	V$$$ "&v#2                  s   A2AA22A69A6c                     t           5  t                              |            t                              | d           t
                              | d           ddd           dS # 1 swxY w Y   dS )z>Remove ``job_id`` from the in-flight running set (idempotent).N)r   r   discardr   popr   r  s    r    release_running_jobr    s    	 + +  (((64(((VT***+ + + + + + + + + + + + + + + + + +s   AA&&A*-A*c                     	 t                      pi } t          | t                    r|                     di           ni                     d          }|t	          |          }|dk    r|S n# t
          $ r Y nw xY wt          j        dd                                          }|rQ	 t	          |          }|dk    r|S n8# t          t          f$ r$ t                              d|t                     Y nw xY wt          S )ab  Floor for the stale in-flight allowance, in minutes.

    Effective allowance per job is ``max(2 * interval, this)``, so a
    slow-but-healthy long-interval job is never clipped by the sweep.
    Reads ``cron.inflight_max_minutes`` from config.yaml; the
    ``HERMES_CRON_INFLIGHT_MAX_MINUTES`` env var is kept as an internal
    escape hatch only.
    r8   inflight_max_minutesNr    HERMES_CRON_INFLIGHT_MAX_MINUTESr#   z=Invalid HERMES_CRON_INFLIGHT_MAX_MINUTES=%r; using default %s)r   rB   rC   r@   floatr   osgetenvr%   r'   	TypeErrorr   r   _INFLIGHT_MIN_ALLOWANCE_MINUTES)_ucfg_cfg_valvalraws       r    _inflight_min_allowance_minutesr    s)   
#%/t%<%<DEIIfb!!!"
#$
%
% 	 //CQww
   
)6
;
;
A
A
C
CC
 
		**CQww
 I& 	 	 	NNO/    	 +*s$   A(A, ,
A98A9&B> >2C32C3_cron_interval_cacheexprc                 x   | t           v rt           |          S d}	 ddlm}  |            ruddlm} ddlm}  |j                    } || |          }|                    |          }|                    |          }||z
                                  dz  }	|	dk    r|	nd}n# t          $ r Y nw xY w|t           | <   |S )u|  Approximate the natural interval of a cron expression, in minutes.

    The persisted job store keeps ``schedule`` as an already-parsed dict
    (``{"kind": "cron", "expr": "0 9 * * 1"}``), so the stale allowance for
    a cron job cannot be derived from a schedule *string* — it must come
    from the expression itself.  We measure the gap between the next two
    fire times with croniter; that is the job's cadence, and the sweep's
    allowance becomes ``max(2 * cadence, floor)`` exactly like interval
    jobs.  Falls back to ``None`` (→ floor allowance) if croniter is
    missing or the expression cannot be evaluated.
    Nr   )_ensure_croniter)croniter)r         N@)	r  	cron.jobsr  r  r   r   get_nexttotal_secondsr   )
r  r   r  	_croniterr   baseitfirstsecondgaps
             r    _cron_interval_minutesr$    s    ####D))F...... 		.777777))))))8<>>D4&&BKK))E[[**FE>0022T9C!GGSSF   !'Ms   BB   
B-,B-c                    	 |                      d          }t          |t                    r'|                                rddlm}  ||          pi }t          |t                    rz|                     d          }|dk    r(|                     d          }|rt          |          ndS |dk    r1t          t          |                     d	          pd
                    S n# t          $ r Y nw xY wdS )un  Best-effort interval length for a job, in minutes (None if unknown).

    Reads the PERSISTED schedule shape first: the job store keeps
    ``schedule`` as an already-parsed dict (``{"kind": "interval",
    "minutes": N}`` or ``{"kind": "cron", "expr": "..."}``), NOT the string
    form that ``parse_schedule`` consumes.  The string path is kept only as
    a defensive fallback for programmatic callers that still build string
    schedules (and for tests that exercise that shape).

    ``kind == "once"`` (one-shot) has no recurring interval — returns None,
    so the sweep uses the documented floor allowance.
    r6   r   )parse_scheduler7   r9   minutesNr8   r  r#   )
r@   rB   rw   r%   r  r&  rC   r  r$  r   )r4   r6   r&  r7   r'  s        r    _job_interval_minutesr(  #  s   77:&&h$$ 	6)9)9 	6000000%~h//52Hh%% 	O<<''Dz!!",,y11)0:uW~~~d:v~~-c(,,v2F2F2L".M.MNNN   4s   B(C# +6C# #
C0/C0c                      t          j                      t          5  t          t                     fdt                                          D             t          t          t                    dcddd           S # 1 swxY w Y   dS )zProbe-visible snapshot of the in-flight guard.

    ``forced_releases`` is a monotonic counter of stale claims this process
    has force-released; any non-zero value means a cron job wedged and was
    recovered without a gateway restart.
    c                 >    i | ]\  }}|t          |z
  d           S )r=   )round)r   jidstartedr   s      r    
<dictcomp>z,get_inflight_guard_stats.<locals>.<dictcomp>M  s=     % % % C U3=!,,% % %rf   )runningrunning_ages_secondsforced_releasesrecent_forced_releasesN)	r   r   r   r   r   itemsr   r   r   r   s   @r    get_inflight_guard_statsr4  B  s     )++C	 	
 	
.//% % % %$2$8$8$:$:% % %  5&*+;&<&<
 
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
s   AA<<B B r>   age_secondsallowance_secondsc                    | |t          |d          t          |d          t                                                      d}t          5  t                              |           t          dt           = ddd           n# 1 swxY w Y   	 t                      dz  dz  }|j        	                    dd           t          |dd	
          5 }|                    t          j        |          dz              ddd           dS # 1 swxY w Y   dS # t          $ r&}t                              d|           Y d}~dS d}~ww xY w)z@Persist a countable signal for one forced release (best-effort).r=   )ra   r>   r5  r6  atNr8   zinflight_forced_releases.jsonlTparentsexist_okautf-8encoding
z*Could not append forced-release record: %s)r+  _hermes_now	isoformatr   r   rx   _FORCED_RELEASE_HISTORY_get_hermes_homeparentmkdiropenwritejsondumpsr   r   debug)ra   r>   r5  r6  entrypathfhes           r    _record_forced_releaserP  V  s    [!,,"#4a88mm%%'' E 
 8 8&&&666678 8 8 8 8 8 8 8 8 8 8 8 8 8 8F!!F*-MM$666$g... 	/"HHTZ&&-...	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ 	/ F F FA1EEEEEEEEEFsO   
+BBBAD +D:D DD DD 
ED??Edue_jobsc           
      r   d | pg D             }t                      dz  }t          j                    }g }ddlm} t          5  d t
          D             }ddd           n# 1 swxY w Y   d |                                D             }i }|r%	 ddlm}	  |	|          }n# t          $ r i }Y nw xY wd	t          d
t          dt          fd}
d |                                D             }t          5  t          t
                    D ]T}t                              |          }||t          |<   *||z
  }|                    |          }|}|rt          |d|z  dz            }t                               |          }|t"          u rn||                                s|                    |          }|&|                    d          |v r |
||          rd}n
||k    rd}nt
                              |           t                              |d           t                               |d           t*          dz  a|                    |||||f           V	 ddd           n# 1 swxY w Y   |D ]6\  }}}}}|                    |          pi }|                    d          p|}|t"          u rd}n|d}nd}t.                              d||||||           t3          ||||           |dk    r|                    d          pi }t5          |t                    r2|                    d          t.                              d||           	 t7          |dd|dz  dd|dz  dd            # t          $ r'}t.                              d!||           Y d}~0d}~ww xY wd" |D             S )#u  Force-release in-flight claims that can no longer be making progress.

    A claim is stale when it is older than ``max(2 * interval, floor)`` AND
    either has no live future at all (the wedge class: the claim was taken
    but the release path was never installed — e.g. a hang in the submit
    path before ``pool.submit`` returned) or has a future that already
    finished without discarding the id.

    Every release logs a WARNING with the countable ``event=forced_release``
    signal, bumps a probe-visible counter (``get_inflight_guard_stats()``),
    mirrors a JSONL row under the cron dir, and writes ``last_error`` on the
    job so the wedge surfaces on the job row instead of being invisible
    until a downstream liveness key goes dead hours later.  A forced release
    never consumes a finite-repeat job's budget (see below).

    Returns the list of released job ids.
    c                 d    i | ]-}t          |t                    |                    d           |.S r?   rB   rC   r@   r   js     r    r.  z(sweep_stale_inflight.<locals>.<dictcomp>  s3    MMMAt9L9LMQUU4[[!MMMrf   r  r   )_TERMINAL_STATESc                 D    i | ]}|t                               |          S rm   )r   r@   r   ra   s     r    r.  z(sweep_stale_inflight.<locals>.<dictcomp>  s6     
 
 
5;F$((00
 
 
rf   Nc                 V    g | ]&\  }}||t           u s|                                $|'S r   )r  done)r   ra   futs      r    r   z(sweep_stale_inflight.<locals>.<listcomp>  sB       FC;#00CHHJJ0 	000rf   )latest_executionsrowclaim_startedr   c                     |                      d          }|sdS 	 ddlm}  |t          j        |                    }|                                |k    S # t          t          t          f$ r Y dS w xY w)u  True when the ledger row was claimed at/after this in-memory claim.

        The latest terminal row proves THIS claim's run ended only if it was
        created by this claim's dispatch (create_execution runs moments AFTER
        try_register_running_job).  A terminal row older than the in-memory
        claim is the PREVIOUS run's outcome — for a recurring job that is the
        common case in the window between try_register and create_execution,
        and releasing on it would double-dispatch a healthy fresh claim.
        Unparseable timestamps fail closed (row treated as previous-run; the
        age-based path below still bounds the claim).
        
claimed_atFr   )_ensure_aware)	r@   r  rc  r   fromisoformat	timestampr'   r  OSError)r_  r`  rb  _ensure_aware_tsrow_tss        r    _row_belongs_to_claimz3sweep_stale_inflight.<locals>._row_belongs_to_claim  s     WW\**
 	5	CCCCCC%%h&<Z&H&HIIF##%%66Iw/ 	 	 	55	s   :A A10A1c                 4    i | ]\  }}|t          |          S rm   )r(  )r   r,  rW  s      r    r.  z(sweep_stale_inflight.<locals>.<dictcomp>  s'    LLLFC#,Q//LLLrf   g       @statuszledger-terminalager=   r>   pendingmissingfinishedu   cron.inflight.forced_release event=forced_release reason=%s job='%s' id=%s age=%.0fs allowance=%.0fs future=%s — stale in-flight claim released; the job was skipping every fire with 'already running'repeattimesu   cron.inflight.forced_release.job_untouched job='%s' id=%s — finite-repeat job released without mark_job_run (repeat budget preserved); row left in place so it re-fires normallyFz+Stale in-flight claim force-released after <   z.1fzm (allowance z=m); previous run never released the scheduler in-flight guardz.Could not record forced release for job %s: %sc                     g | ]
}|d          S )r   rm   )r   ss     r    r   z(sweep_stale_inflight.<locals>.<listcomp>0  s       QAaD   rf   )r  r   cron.executionsrX  r   r   r3  r^  r   rC   r  r   r   r   r@   maxr   r  r\  r  r  r   rx   r   r   rP  rB   r   )rQ  by_idfloor_secondsr   stale_terminal_states_claim_futures_ledger_candidates_latest_latest_execsri  
_intervalsra   r-  rl  interval_minutes	allowancer]  latestrc   _reasonr4   r>   future_staterp  rO  s                             r    sweep_stale_inflightr  k  sr   ( NMhn"MMME355<M
)++CE EDDDDD	 
 

 
?O
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 )//11  
 G 	JJJJJJ#m$677GG 	 	 	GGG	4  $    0 MLekkmmLLLJ	 5@ 5@+,, 4	@ 4	@F$((00G *-v&-C)~~f55%I J	31A+AD+HII	"&&v..Co%%
 " [[((F"JJx((,<<<))&':: = +	!!$$V,,,vt,,,  ...!Q&!LL&#y#v>????i4	@5@ 5@ 5@ 5@ 5@ 5@ 5@ 5@ 5@ 5@ 5@ 5@ 5@ 5@ 5@n 16 9X 9X,YWii%2wwv(&/!!$LL[$LL%LO 
	
 
	
 
	
 	vtS)<<< ''' ""(bfd## 	

7(;(;(GNNH    		X1cBhN 1 1'"nR1 1 1     	X 	X 	XNNKVUVWWWWWWWW	X ! %    sO   A  A$'A$B   B/.B/0E+I((I,/I, M99
N*N%%N*)only_ownersrc   r  c                   t           5  d t                                          D             }fd|D             }d |D             }'|                    d t          |z
  D                        t
                              d |D                        ddd           n# 1 swxY w Y   g }|D ]\  }}}}|s1t                              d|           |	                    |           :	 t          |          5  t          |d| |	          r|	                    |           ddd           n# 1 swxY w Y   # t          $ r&}	t                              d
||	           Y d}	~	d}	~	ww xY w|S )u  Best-effort: mark every currently in-flight cron job interrupted.

    Called by the gateway shutdown path immediately after it force-kills
    tool subprocesses (``process_registry.kill_all()``). A job whose tool
    subprocess was just killed out from under it must never be allowed to
    report success — even though its agent thread is still alive in this
    same process and may go on to produce a plausible-looking final
    response from the now-truncated tool output.

    Records the job IDs in ``_interrupted_job_ids`` BEFORE writing
    ``last_status`` so ``run_one_job``'s own eventual completion for the
    same job (racing in its own thread) sees the flag and skips its normal
    write instead of clobbering this one — see the check near the end of
    ``run_one_job``. This does not attempt to correlate the killed
    subprocess PID to a specific job ID (the process registry tracks PIDs,
    not cron job IDs); any job still dispatched at the moment of a forced
    kill is treated as interrupted, matching the coarser precedent already
    set by ``GatewayRunner._interrupt_running_agents``, which interrupts
    every entry in ``_running_agents`` on a drain timeout without
    per-agent correlation either.

    ``only_owners``: optional set of ``(job_id, fire_owner)`` pairs. When
    given (dashboard webhook drain), ONLY those exact executions are
    marked — unrelated runs sharing the process (e.g. the desktop ticker's
    own jobs) are left untouched. Interruption flags are recorded per
    execution token, so a later run of the same job ID never consumes a
    stale flag that targeted its dead predecessor.

    Returns the list of job IDs marked, for the caller to log.
    c                 Z    g | ](\  }}|                                 D ]\  }\  }}||||f)S rm   )r3  )r   ra   
executionstokenownerprofile_homes         r    r   z1mark_running_jobs_interrupted.<locals>.<listcomp>W  sa     
 
 
"
0:0@0@0B0B
 
 -,| FE<0
 
 
 
rf   Nc                 8    g | ]}|d          |d         fv |S )r=      rm   )r   firer  s     r    r   z1mark_running_jobs_interrupted.<locals>.<listcomp>]  s;       GT!W%44 444rf   c                     h | ]	\  }}}}|
S rm   rm   )r   _tra   _o_ps        r    	<setcomp>z0mark_running_jobs_interrupted.<locals>.<setcomp>a  s!    HHH%7RR&HHHrf   c              3   :   K   | ]}d |d t                      fV  d S r   rD  rZ  s     r    r   z0mark_running_jobs_interrupted.<locals>.<genexpr>c  sF           vt%5%7%78           rf   c              3   ,   K   | ]\  }}}}||n|V  d S r   rm   )r   r  ra   _owner_profile_homes        r    r   z0mark_running_jobs_interrupted.<locals>.<genexpr>g  sG       $
 $
4vv} &EEF$
 $
 $
 $
 $
 $
rf   zdJob '%s' interrupted before its durable fire owner was registered; leaving persisted state untouchedFexpected_fire_ownerz%Failed to mark job %s interrupted: %s)r   r   r3  extendr   r   updater   r   rx   r   r   r   )
rc   r  active_firesregistered_idsmarked_tokenra   
fire_ownerr  rO  s
    `        r    mark_running_jobs_interruptedr  3  s   F 
 
 

 
&:&@&@&B&B
 
 

 "   !-  L IH<HHH    .?         	## $
 $
8D$
 $
 $
 	
 	
 	
#
 
 
 
 
 
 
 
 
 
 
 
 
 
 
* F4@ O O0
L 	NN4   MM&!!!
	O-- * *(2	   * MM&)))* * * * * * * * * * * * * * *  	O 	O 	ONNBFANNNNNNNN	OMsN   BB""B&)B&,D<;)D0$D<0D4	4D<7D4	8D<<
E,E''E,r  c                     t           5  ||t          v r	 ddd           dS | t          v cddd           S # 1 swxY w Y   dS )u  Non-destructive peek at whether the shutdown path has marked THIS
    execution interrupted (see ``mark_running_jobs_interrupted``).

    Called by ``run_one_job`` BEFORE it decides what to deliver — a job
    whose tool subprocess was killed mid-flight may still produce a
    plausible-looking ``final_response`` from the truncated output, and
    that must not go out to the user as if it were a normal result.
    Unlike ``_consume_interrupted_flag`` below, this does not clear the
    flag: the later, authoritative check (right before ``last_status`` is
    written) still needs to see it. ``token`` scopes the check to one
    exact execution: owner-registered runs are matched by token, so a
    fresh run reusing the same job ID is not poisoned by a flag that
    targeted its dead predecessor. The bare job ID is only ever stored
    for legacy dispatch paths with no registered fire owner.
    NT)r   r   )ra   r  s     r    _is_interruptedr    s      
 . .*>!>!>. . . . . . . . --. . . . . . . . . . . . . . . . . .s   77;;c                     t           5  d}|%|t          v rt                              |           d}| t          v rt                              |            d}|cddd           S # 1 swxY w Y   dS )a  Return True and clear the flag if the shutdown path already marked
    THIS execution interrupted (see ``mark_running_jobs_interrupted``).

    Called by ``run_one_job`` right before it would otherwise write its own
    ``last_status``. Consuming (discarding) rather than just checking keeps
    the flag from leaking across a later, unrelated run of the same job ID
    (recurring jobs reuse their ID every fire).FNT)r   r   r  )ra   r  hits      r    _consume_interrupted_flagr    s     
  *>!>!> ((///C))) ((000C                 s   AA%%A),A)_sequential_poolc                   Z    e Zd ZdZd
dZddedz  defdZd
dZddedz  defdZ	d
d	Z
dS )_ReadWriteLocku6  Writer-preferring readers-writer lock.

    Guards the process-global ``os.environ["TERMINAL_CWD"]`` override that a
    workdir cron job applies for the whole of its agent run.  Workdir jobs are
    writers: they mutate the shared env and need exclusive access.  Workdir-less
    jobs are readers: they only observe ``TERMINAL_CWD`` (indirectly, via the
    terminal / file / code-exec tools), so any number of them may run
    concurrently with each other, but none may run alongside a writer — that is
    exactly what stops a workdir-less job from picking up another job's workdir
    override and running its commands in the wrong directory.

    Writer preference bounds the wait for a workdir job (dispatched on the
    single-thread sequential pool) so a stream of workdir-less readers cannot
    starve it.
    r   Nc                     t          j        t          j                              | _        d| _        d| _        d| _        d S )Nr   F)	threading	ConditionLock_cond_readers_writer_active_writers_waitingr   s    r    r   z_ReadWriteLock.__init__  s9    ()9)9::
# !rf   rS   c                    |t          j                    |z   nd}| j        5  | j        s| j        dk    r|_|t          j                    z
  }|dk    r'| j                                         	 ddd           dS | j                            |           n| j                                         | j        | j        dk    | xj        dz  c_        ddd           n# 1 swxY w Y   dS )u   Acquire a read lock.

        Returns ``True`` if the lock was acquired, ``False`` on timeout.
        A timed-out caller proceeds without the lock (degraded mode) —
        see the call-site in ``run_job`` for the logging / trade-off.
        Nr   FrS   r=   T)r   	monotonicr  r  r  
notify_allwaitr  r   rS   deadline	remainings       r    acquire_readz_ReadWriteLock.acquire_read  sZ    +2*=DNw&&4 	 Z 
	 
	% &)>)B)B' (4>+;+; ;I A~~
--///$
	 
	 
	 
	 
	 
	 
	 
	 JOOIO6666JOO%%% % &)>)B)B MMQMM
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 
	 ts   AC:ACC!$C!c                     | j         5  | xj        dz  c_        | j        dk    r| j                                          d d d            d S # 1 swxY w Y   d S )Nr=   r   )r  r  r  r   s    r    release_readz_ReadWriteLock.release_read  s    Z 	( 	(MMQMM}!!
%%'''	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(s   5A

AAc                 P   |t          j                    |z   nd}| j        5  | xj        dz  c_        	 | j        s| j        dk    r|o|t          j                    z
  }|dk    r7| j                                         	 | xj        dz  c_        ddd           dS | j                            |           n| j                                         | j        | j        dk    | xj        dz  c_        n# | xj        dz  c_        w xY wd| _        ddd           n# 1 swxY w Y   dS )zAcquire a write lock.

        Returns ``True`` if the lock was acquired, ``False`` on timeout.
        A timed-out caller proceeds without the lock (degraded mode).
        Nr=   r   Fr  T)r   r  r  r  r  r  r  r  r  s       r    acquire_writez_ReadWriteLock.acquire_write  s    +2*=DNw&&4 	 Z 	' 	'!!Q&!!+) *T]Q->->+$,t~/?/?$?	$>> J11333#(
 %%*%%	' 	' 	' 	' 	' 	' 	' 	' 
	::::
))) ) *T]Q->-> %%*%%%%%*%%%%%%"&D	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' 	' ts6   DA	C3>DAC3"D3D
DD"Dc                     | j         5  d| _        | j                                          d d d            d S # 1 swxY w Y   d S NF)r  r  r  r   s    r    release_writez_ReadWriteLock.release_write  s    Z 	$ 	$"'DJ!!###	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$ 	$s   !6::r   r   )ri   rj   rk   rl   r   r  r   r  r  r  r  rm   rf   r    r  r    s          " " " " EDL D    .( ( ( ( UT\ T    4$ $ $ $ $ $rf   r  g      ^@r  c                      t          j        dd                                          } | sdS 	 t          |           S # t          t
          f$ r t                              d|            Y dS w xY w)uz  Parse HERMES_CRON_TIMEOUT (seconds). 0 = unlimited; bad input = 600.

    Shared by run_job's inactivity monitor (which maps 0 to "no limit") and
    the cwd-lock bound below (which keeps the wait bounded regardless) so
    the two sites cannot drift apart — the lock bound must stay at or above
    the inactivity limit or waiters would fail while a healthy holder runs.
    HERMES_CRON_TIMEOUTr#        @z2Invalid HERMES_CRON_TIMEOUT=%r; using default 600s)r  r  r%   r  r'   r  r   r   )r  s    r    _cron_inactivity_secondsr  #  s|     ))2
.
.
4
4
6
6C uSzz	"   KSQQQuus   < ,A,+A,c                  h    t                      } | dk    rd} t          | t                    t          z   S )z@Bound for the TERMINAL_CWD lock wait: inactivity limit + margin.r   r  )r  rv  _CWD_LOCK_TIMEOUT_FLOOR_SECONDS _CWD_LOCK_TIMEOUT_MARGIN_SECONDS)
inactivitys    r    _cwd_lock_timeout_secondsr  5  s6    )++JQ
J788
*	+rf   max_workersc                     t           t          | k    rFt           t                               dd           t          j                            | d          a | at           S )z0Return (or create) the persistent parallel pool.NFr  cancel_futureszcron-parallelr  thread_name_prefix)r   r   shutdown
concurrentfuturesThreadPoolExecutorr  s    r    _get_parallel_poolr  @  se     !;{!J!J%##u#EEE#+>>#. ? 
 
 &1"rf   c                  `    t           !t          j                            dd          a t           S )a)  Return (or create) the persistent single-thread sequential pool.

    A single worker guarantees env-mutating jobs never overlap, even
    across ticks: a job queued by a newer tick waits for the previous tick's
    sequential jobs to finish rather than corrupting their os.environ
    state.
    Nr=   zcron-seqr  )r  r  r  r  rm   rf   r    _get_sequential_poolr  N  s7     %-@@) A 
 
 rf   c                      t            t                               dd           da dat           t                              dd           dadS dS )z/Shut down the persistent pools on process exit.NTFr  )r   r  r   r  rm   rf   r    _shutdown_parallel_poolr  _  sa     !T%@@@%)"#!!tE!BBB $#rf   c                  *    t                      dz  dz  S )Nr8   zusage_audit.jsonlr  rm   rf   r    _usage_audit_pathr  n  s    &)<<<rf   c                      t          j        t          j                  } |                     d          | j        dz  ddz   S )z@RFC3339 UTC timestamp with millisecond precision and 'Z' suffix.z%Y-%m-%dT%H:%M:%S.  03dZ)r   r   r   utcstrftimemicrosecondr   s    r    _utcnow_iso_msr  r  s>    
,x|
$
$C<<,--3?d3J0Q0Q0Q0QQQrf   recordc                 t   	 t                      }|j                            dd           t          j        | d          }t          |dd          5 }|                    |dz              d	d	d	           d	S # 1 swxY w Y   d	S # t          $ r&}t          	                    d
|           Y d	}~d	S d	}~ww xY w)u   Append a single JSONL line to ~/.hermes/cron/usage_audit.jsonl.

    NEVER raises — a logger bug must not break cron jobs. Wraps the entire
    write (path resolve, mkdir, json.dumps, file append) in a single try.
    Tr9  F)ensure_asciir<  r=  r>  r@  Nzusage_audit write failed: %s)
r  rE  rF  rI  rJ  rG  rH  r   r   r   )r  rM  linefrO  s        r    _write_usage_auditr  y  s   : ""$666z&u555$g... 	!!GGD4K   	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! : : :5q999999999:s<   AB A:-B :A>>B A>B 
B7B22B7r   c                 z    t          j                    rdS | #dt          |                                           v S dS )u  True when the Python interpreter is finalizing.

    A cron tick can fire while the gateway is tearing down — SIGTERM from
    ``hermes update`` / ``hermes gateway stop`` / systemd restart, or an
    OOM-kill. Once finalization starts, ``concurrent.futures`` refuses new
    work with ``RuntimeError: cannot schedule new futures after interpreter
    shutdown`` and asyncio's default executor is gone, so *any* attempt to
    schedule delivery (live-adapter, ``asyncio.run``, or a fresh pool) is
    doomed and only pollutes ``errors.log`` with a traceback. Callers use
    this to skip gracefully with a warning instead of crashing (#58720,
    #55924).

    ``exc`` lets a caller also treat an already-raised scheduling error as a
    shutdown signal: the ``concurrent.futures`` module-global flag can be set
    a hair before ``sys.is_finalizing()`` flips, so matching the error text is
    a safe fallback for that race.
    TNzcannot schedule new futuresF)sysis_finalizingrw   rW   r   s    r    _interpreter_shutting_downr    sA    $  t
 -C0@0@@@5rf   _hermes_homec                  ,    t           pt                      S )u  Resolve Hermes home dynamically while preserving test monkeypatch hooks.

    Cron is per-profile by design (#4707): the in-process ticker runs inside a
    profile-scoped gateway, so resolving the active HERMES_HOME at call time
    means a profile's jobs are stored AND executed under that profile's home
    (its .env, config.yaml, scripts, skills). Do not freeze this at import or
    anchor it at the shared default root — either re-breaks profile isolation.
    )r  r   rm   rf   r    rD  rD    s     ,?,,,rf   c                  6    t                      } | dz  }||dz  fS )zHResolve cron lock paths at call time so profile/env changes are honored.r8   z
.tick.lockr  )hermes_homelock_dirs     r    _get_lock_pathsr    s'    "$$KV#HX,,,rf   errc                     | j         dS t          *| j         t           j        t           j        t           j        fv S t
          | j         t           j        t           j        fv S dS )a!  Return True when *err* from the lock syscall means the lock is held.

    - POSIX: ``flock(LOCK_EX|LOCK_NB)`` reports EWOULDBLOCK/EAGAIN when
      another process holds the lock (EACCES on some NFS implementations).
    - Windows: ``msvcrt.locking(LK_NBLCK)`` reports EACCES/EDEADLK.
    NF)errnofcntlEWOULDBLOCKEAGAINEACCESmsvcrtEDEADLK)r  s    r    _is_lock_contention_errnor    sS     yuyU.elKKKyU\5=9995rf   c                 :    |                                  }d|v pd|v S )zHText-level half of :func:`_is_fd_exhaustion` (shared with the CLI hint).ztoo many open filesemfile)rW   )r_   lowereds     r    _is_fd_exhaustion_textr    s$    jjllG G+Bx7/BBrf   c                     t          | t                    r!| j        t          j        t          j        fv rdS t          t          |                     S )a  Return True when *exc* indicates file-descriptor exhaustion.

    Recognizes EMFILE/ENFILE by errno, and the "Too many open files" wording
    for wrapped exceptions (``load_jobs`` wraps the raw OSError in a
    RuntimeError with that message, #87644).
    T)rB   rf  r  EMFILEENFILEr  rw   r  s    r    _is_fd_exhaustionr	    sD     #w CI%,1M$M$Mt!#c((+++rf   c                      	 ddl } |                                  n# t          $ r Y nw xY w	 ddlm}  |d           dS # t          $ r Y dS w xY w)u  Best-effort attempt to free leaked file descriptors.

    The cron FD-leak family (#60859, #79742, #80792) leaks descriptors from
    abandoned workers/sessions.  Two safe, idempotent levers:

    1. ``gc.collect()`` — closes file-like objects held only in reference
       cycles (the classic unclosed-file leak shape), which CPython would
       otherwise never finalize.
    2. ``apply_nofile_soft_limit()`` — raise RLIMIT_NOFILE's soft limit
       toward the configured target when the hard limit allows, giving the
       process headroom to keep serving even before every leak is freed.

    Never raises: a reclamation attempt must not make the ticker worse.
    r   N)apply_nofile_soft_limit)gccollectr   hermes_cli.resource_limitsr  )r  r  s     r    _reclaim_fds_best_effortr    s    			


   FFFFFF%%%%%   s    
((? 
AAlogical_platform_namec                    	 t          | dd          pi }|                    t          |pd                                                    }t	          |t
                    r+|                    d          |                    d          }n|                    d          }|9t          |                                                                          dk    rdS n# t          $ r Y nw xY wdS )u  Resolve the continuable-cron delivery surface for a platform config.

    Returns ``"in_channel"`` or ``"thread"`` (default). Two config shapes:

    - Native adapter: the flat key ``platforms.<p>.extra.cron_continuable_surface``
      (shipped shape, unchanged).
    - Relay-fronted: ``platforms.relay.extra.<logical>.cron_continuable_surface``
      — the same per-logical-platform sub-block the relay's documented Slack
      knobs use (``reply_in_thread``, ``dm_top_level_threads_as_sessions``;
      see RelayAdapter._relay_slack_extra). The sub-block wins over a flat
      key when both exist, matching _relay_slack_extra precedence, and is
      scoped to its logical platform so a ``slack:`` block cannot leak onto
      another fronted platform.

    Precedence nuance vs _relay_slack_extra: that helper is all-or-nothing
    (a sub-dict REPLACES the flat extra entirely), while this one falls back
    to the flat key when the sub-block exists but omits the knob. The
    difference is deliberate — the flat key is the legacy staging shape and
    must keep working — but note a flat ``cron_continuable_surface`` then
    applies to EVERY platform this relay fronts; only the per-platform D6
    capability gate contains it. Scope the knob under the sub-block on
    multi-platform relays.

    Field gap (2026-08-18): the scheduler read only the flat key, so on the
    relay lane — where pconfig is platforms.relay — operators had NO working
    location for the knob and briefs always threaded.
    extraNr#   cron_continuable_surface
in_channelthread)r(   r@   rw   rW   rB   rC   r%   r   )pconfigr  r  r[   r  s        r    _resolve_cron_surface_moder    s    8
$//52ii17R88>>@@AAc4   	8SWW-G%H%H%T''455CC))677C?s3xx~~//5577<GG<   8s   CC 
C)(C)c                     |                      d          }t          |t                    sdS |                     d          }|                     d          }|r|r|S dS )up  Extract origin info from a job, preserving any extra routing metadata.

    Treats non-dict origins (free-form provenance strings, ints, lists from
    migration scripts or hand-edited jobs.json) as missing instead of
    crashing with ``AttributeError`` on ``origin.get(...)``. Without this
    guard, a job tagged with e.g. ``"combined-digest-replaces-x-and-y"``
    crashed every fire attempt with
    ``'str' object has no attribute 'get'`` — ``mark_job_run`` recorded the
    failure, but the next tick re-loaded the same poisoned origin and
    crashed identically until the field was patched manually (#18722).
    originNplatformchat_idr@   rB   rC   )r4   r  r  r  s       r    _resolve_originr  -  sg     WWXFfd## tzz*%%Hjj##G G 4rf   c                    |                      d          }t          |t                    r|S 	 |t                      pi }t          |                     di           pi                      dd                    S # t          $ r Y dS w xY w)u  Whether a cron delivery should also be mirrored into the target chat's
    gateway session transcript.

    Default OFF — preserves the historical isolation guarantee (cron deliveries
    live only in the cron job's own session, never the target chat's history)
    byte-for-byte for everyone who does not opt in.

    CARVE-OUT: the ``in_channel`` continuable surface seeds its target
    session independently of this knob (see ``_deliver_result`` /
    ``_seed_cron_channel_session``). in_channel is itself opt-in
    (``cron_continuable_surface: in_channel`` + the adapter capability bit),
    and the seed IS the feature — a continuable flat brief without its seed
    is a brief the next reply can't see. This knob keeps governing the
    SEPARATE default/thread-surface transcript mirror only.

    Precedence (first decisive value wins):
      1. Per-job ``attach_to_session`` (bool) — set via the ``cronjob`` tool,
         lets one briefing job opt in without flipping global behaviour.
      2. Global ``cron.mirror_delivery`` (bool) in config.yaml.
      3. False.

    When enabled, the cron's final output is appended to the target session as
    an assistant turn via the existing ``gateway.mirror.mirror_to_session`` —
    the same primitive ``send_message`` uses — so the next user reply in that
    chat sees the brief in context (no "what is Task #2?" amnesia). This is
    alternation- and cache-safe: the append lands at a turn boundary between
    user turns, never mid-loop, and never mutates the cached system prompt.
    attach_to_sessionNr8   mirror_deliveryF)r@   rB   r   r   r   )r4   r1   r~   s      r    _cron_mirror_delivery_enabledr!  C  s    : gg)**G'4   ;--%2CSWWVR((.B334EuMMNNN   uus   A
A9 9
BBr  platform_namer  	thread_idc                    | sdS t          |                     dd                                                    t          |                                          k    rdS t          |                     dd                    t          |          k    rdS |                     d          }|$t          |          t          |pd          k    rdS dS )u  True when a delivery target is the job's own origin conversation.

    Mirroring is scoped to the origin session by design (see
    ``_maybe_mirror_cron_delivery``). A job created from a live gateway chat
    stamps that chat as ``origin`` (``cronjob_tools._origin_from_env``), and
    that session is guaranteed to exist — it is the very conversation the user
    was in when they scheduled the job. Fan-out targets (``deliver=all``,
    explicit ``platform:chat_id`` to some *other* chat, or a home-channel
    fallback for an origin-less API/script job) are deliberately NOT mirrored:
    they are broadcasts, not a continuation of a conversation, and may point at
    a chat the user never opened an agent session in.

    This makes the historical "cold-start" worry a non-case: when the mirror
    semantically applies (target == origin) the session always exists; when no
    session exists, the target was never the origin conversation, so we simply
    do not mirror.
    Fr  r#   r  r#  NT)rw   r@   rW   )r  r"  r  r#  origin_threads        r    _target_matches_originr&  k  s    &  u
6::j"%%&&,,..#m2D2D2J2J2L2LLLu
6::i$$%%W55u JJ{++M S%7%73yB;O;O%O%Ou4rf   Fenabledmirror_textuser_idr(  c          	      L   |sdS |pd                                 }|sdS 	 ddlm}  ||t          |          d|                     d          p|                     dd           d	| d||d
          }	|	r3t
                              d|                     dd          ||           dS t
                              d|                     dd          ||           dS # t          $ r=}
t
                              d|                     dd          |||
           Y d}
~
dS d}
~
ww xY w)u  Best-effort mirror of a cron delivery into the origin chat's session.

    No-op unless ``enabled`` (resolved once by the caller, and already scoped to
    the origin target — see ``_target_matches_origin``). Reuses the shipped
    ``mirror_to_session`` so cron rides exactly the same path that interactive
    ``send_message`` mirroring already uses, including passing ``user_id`` so a
    per-user-isolated group chat resolves to the exact member who scheduled the
    job (parity with ``send_message``). All failures are swallowed — a delivery
    that succeeded must never be reported as failed because the transcript
    mirror hit a problem.

    Because the caller only enables this for the target that equals the job's
    origin conversation, the session is expected to exist (the job was born in
    that session). A missing session therefore indicates an origin-less /
    fan-out delivery that should not have been mirrored anyway, and is treated
    as a silent no-op — never a synthetic session is created.
    Nr#   r   mirror_to_session[Cron delivery: r>   r?   r8   ]
user)source_labelr#  r*  rolez9Job '%s': mirrored delivery into %s:%s session transcriptr   uX   Job '%s': delivery mirror skipped for %s:%s (no matching gateway session — cold start)z.Job '%s': delivery mirror failed for %s:%s: %s)	r%   gateway.mirrorr-  rw   r@   r   r   rK  r   )r4   r"  r  r)  r#  r*  r(  r_   r-  okrO  s              r    _maybe_mirror_cron_deliveryr5    s   6  2$$&&D #
444444 LLRswwvG#''$2G2GRRDRR
 
 
  
	KKKc""M7    
 LL?c""M7    
  
 
 
<GGD#w	
 	
 	
 	
 	
 	
 	
 	
 	

s   BC )1C 
D#&2DD#c           
         t          |dd          }t          |          r|dS |                     d          p|                     dd          }d| }	 ddlm}  |t          |          |          } |||          }	|	dS |	                    d	
          }
|
rt          |
          ndS # t          $ rK}t          	                    d|                     dd          t          |dd          |           Y d}~dS d}~ww xY w)u  Open a dedicated thread for a continuable cron job (thread-preferred).

    Returns the new ``thread_id`` on success, or ``None`` when the platform has
    no thread primitive (WhatsApp/Signal/SMS) or creation failed — the ``None``
    return is the caller's signal to fall back to the origin-DM mirror, the same
    open-thread-or-fallback shape as ``GatewayRunner._process_handoff``. Reuses
    the shipped ``adapter.create_handoff_thread``; no new adapter surface.
    create_handoff_threadNr>   r?   r8   u   Hermes — r   safe_schedule_threadsafe   r  uV   Job '%s': create_handoff_thread failed on %s — falling back to DM-session mirror: %sr   )
r(   callabler@   agent.async_utilsr9  rw   r   r   r   rK  )r4   adapterr  loopcreate_thread	task_namethread_namer9  coror   new_thread_idrO  s               r    _open_continuable_cron_threadrD    s=    G%<dCCMM"" dlt83774#8#8I+	++K>>>>>>}S\\;77))$55>4b11%2<s=!!!<   $GGD# = =q	
 	
 	

 ttttts   -B. (B. .
D8A C>>D	chat_nameis_dmscope_idc	                 &   |pd                                 }	|	sdS 	 ddlm}
 ddlm} d}t          |dd          }|	  |
|                                          }n# t          t          f$ r d}Y nw xY w|||
j	        k    rt          |          }nt          |          } |||||rdndd	d
t          |          |rt          |          nd          }|                    |          }t          |dd          }ddlm}  ||t          |          d|                     d          p|                     dd           d|	 dt          |          d	d|          }|r4t                              d|                     dd          |||           dS t                              d|                     dd          |||           dS # t$          $ r>}t                              d|                     dd          ||||           Y d}~dS d}~ww xY w)u  Seed the freshly-opened cron thread's session with the brief.

    Without this the brief is *visible* in the new thread but absent from any
    transcript, so the user's first reply in-thread would hit a session with no
    record of it ("what is Task #2?"). We create the thread-keyed session (the
    same key the user's reply will resolve to — ``build_session_key`` keys
    threads as participant-shared, so no ``user_id`` is needed) and append the
    brief as an assistant turn via the shipped ``mirror_to_session``.

    ``scope_id`` is the workspace/server scope (Slack team id).
    ``build_session_key`` embeds it in every Slack key, so a scoped reply's
    key carries it — the seed must reproduce it or the seeded row is
    unreachable (the scope-less flat-seed sibling of the is_dm keying bug).
    Best-effort None for platforms without scope.

    ``is_dm`` selects the seeded ``chat_type``: a thread under a DM must seed
    ``chat_type="dm"`` because the user's in-thread DM reply arrives with
    chat_type="dm" and ``build_session_key`` routes DM threads through the DM
    arm (``...:dm:<chat>:<thread>``) — a "thread"-typed seed lands in
    ``...:thread:<chat>:<thread>``, a row no DM reply ever resolves to
    (continuation amnesia, Alice live 2026-08-20, job 8e21a957b77b). Channel
    threads keep ``chat_type="thread"`` (their replies really do arrive as
    threads). Same sibling-lane class as the flat seed's ``is_dm``
    (dcca9d8cfe).

    Mirrors ``GatewayRunner._process_handoff``'s seed step, but standalone:
    cron reaches the live ``SessionStore`` through the adapter's
    ``_session_store`` handle rather than the gateway object. Best-effort — a
    delivery that already succeeded is never failed by a seeding problem.
    r#   Nr   PlatformSessionSource_session_storedmr  zsystem:cronCron)r  r  rE  	chat_typer*  	user_namer#  rG  r*   r,  r.  r>   r?   r8   r/  r0  )r1  r#  r*  r2  r*   zDJob '%s': opened continuable thread %s on %s:%s and seeded the briefr   ud   Job '%s': thread seed did NOT land on %s:%s thread=%s — an in-thread reply will not see this briefz=Job '%s': seeding cron thread session failed for %s:%s:%s: %s)r%   gateway.configrJ  gateway.sessionrL  r(   rW   r'   KeyErrorDISCORDrw   get_or_create_sessionr3  r-  r@   r   r   r   r   )r4   r=  r"  r  r#  r)  rE  rF  rG  r_   rJ  rL  seeded_session_idsession_storeplatform_enumseed_chat_iddest_source_entryr-  r4  rO  s                        r    _seed_cron_thread_sessionr]    s   R 2$$&&D L
++++++111111+/)94@@$% ()<)<)>)> ? ?) % % % $%( !H$444#&y>>LL#&w<<L+m*(' ',9dd)$!)nn.6@S]]]D  " '<<[II$+FL$$G$G!444444 LLRswwvG#''$2G2GRRDRR)nn!(	
 	
 	
  
	KKVc""I}g    
 NN:c""M7I    
  
 
 
 	KGGD#w	1	
 	
 	
 	
 	
 	
 	
 	
 	

sA   !G A G A2/G 1A22D G 2G 
H3HH)rE  rG  c                   |pd                                 }	|	sdS 	 ddlm}
 ddlm} |rdnd}t          |dd	          }d	}|	  |
|                                          }n# t          t          f$ r d	}Y nw xY w|g ||t          |          |||rt          |          nd	d	|rt          |          nd	
          }|
                    |          }t          |dd	          }ddlm}  ||t          |          d|                     d          p|                     dd           d|	 dd	|rt          |          nd	|d          }|r2t                              d|                     dd          |||           t!          |          S # t"          $ r=}t                              d|                     dd          |||           Y d	}~dS d	}~ww xY w)u  Seed the FLAT (thread_id=None) session for an ``in_channel`` cron delivery.

    The ``in_channel`` surface (D1/D2) delivers the brief flat into the channel
    with no thread, so the continuation surface is the whole-channel /
    whole-DM session keyed ``thread_id=None`` — the same bucket
    ``reply_in_thread: false`` routes an inbound plain reply to.

    Unlike the thread path, the shipped delivery-mirror alone is NOT sufficient
    here: ``mirror_to_session`` only APPENDS to a session that already EXISTS
    (``_find_session_id`` → no-op when none matches), and a flat channel
    ``(…, None)`` row is only created when a human posts a top-level message the
    bot processes — a ``chat_postMessage`` cron delivery never goes through the
    inbound handler, so the row is usually absent and the mirror silently drops
    the brief (verified live: the brief never landed, the reply had no context).
    So we CREATE the flat session row first, exactly like
    ``_seed_cron_thread_session`` does for threads, then mirror into it.

    The session KEY must match what the user's later inbound reply resolves to
    (``build_session_key``):
    - **Channel** (``chat_type="group"``): key is
      ``…:group:<chat_id>:<user_id>`` — user-isolated — so the seed MUST carry
      the **origin's real ``user_id``** (the member who scheduled the job), NOT
      a synthetic ``system:cron`` id, or the reply keys to a different session.
    - **1:1 DM** (``chat_type="dm"``): the key is ``…:dm:<chat_id>`` and does
      NOT embed ``user_id``, so any ``user_id`` resolves to the same session.
    ``chat_type`` mirrors the inbound handler's own choice
    (``"dm" if is_dm else "group"``, ``adapter.py``), so the seeded key is
    byte-identical to the reply's key.

    Returns True if a seed row was created and the brief mirrored, else False
    (caller falls back to the plain mirror). Best-effort — a delivery that
    already succeeded is never failed by a seeding problem.
    r#   Fr   rI  rK  rN  grouprM  N)r  r  rE  rP  r*  r#  rG  r*   r,  r.  r>   r?   r8   r/  r0  )r1  r#  r*  r*   r2  z@Job '%s': seeded flat in_channel session on %s:%s (chat_type=%s)r   z9Job '%s': seeding in_channel session failed for %s:%s: %s)r%   rR  rJ  rS  rL  r(   rW   r'   rT  rw   rV  r3  r-  r@   r   r   r   r   r   )r4   r=  r"  r  r)  rF  r*  rE  rG  r_   rJ  rL  rP  rX  rW  rY  r[  r\  r-  r4  rO  s                        r    _seed_cron_channel_sessionr`  r  sp   Z 2$$&&D u<++++++111111!.DDw	)94@@+/$% ()<)<)>)> ? ?) % % % $%(+m*LL'',3=CLLL" /7@S]]]D  $ '<<[II$+FL$$G$G!444444LLRswwvG#''$2G2GRRDRR$+5CLLL(	
 	
 	
  	KKRc""M7I   Bxx    	GGGD#w	
 	
 	
 uuuuus<   'F A" !F "A85F 7A88DF 
G2GGc           	         |                      d          }t          |t                    sdS g }dD ]}|                     |          }|t          |                              dd                              dd                                          }|r#|                    d| d	|dd
                    |rdd                    |          z   ndS )a  Return safe provenance details for security warnings about a cron job.

    The scheduler normally has no live HTTP request object when it detects a
    bad stored ``context_from`` reference. Including the job's saved origin
    makes future probe logs actionable without exposing secrets: platform/chat
    metadata for gateway-created jobs, and optional source-IP fields for API
    surfaces that persist them in origin metadata.
    r  r#   )r  r  r#  	source_ipremoteforwarded_forNrT   r@  origin_=   )r@   rB   rC   rw   replacer%   rx   join)r4   r  fieldskeyvaluer_   s         r    _cron_job_origin_log_suffixrn    s     WWXFfd## rF[ : :

3=5zz!!$,,44T3??EEGG 	:MM8C88$tt*88999%+33&!!!!3rf   c                     	 ddl m}  |             ddlm} |                    |                                           }|r|j        r|j        S n# t          $ r Y nw xY wdS )zReturn the cron home-channel env var registered by a plugin platform.

    Falls through the platform registry so plugins that set
    ``cron_deliver_env_var`` on their ``PlatformEntry`` get cron delivery
    support without editing this module.
    r   discover_pluginsplatform_registryr#   )hermes_cli.pluginsrq  gateway.platform_registryrs  r@   rW   cron_deliver_env_varr   )r"  rq  rs  rL  s       r    _plugin_cron_env_varrw    s    777777??????!%%m&9&9&;&;<< 	.U/ 	.--   2s   AA 
AAc                 x    |                                  }|t          v rdS t          t          |                    S )zWhether ``platform_name`` is a valid cron delivery target.

    Hardcoded built-ins in ``_KNOWN_DELIVERY_PLATFORMS`` are checked first;
    plugin platforms registered via ``PlatformEntry`` are accepted if they
    provide a ``cron_deliver_env_var``.
    T)rW   _KNOWN_DELIVERY_PLATFORMSr   rw  )r"  r>   s     r    _is_known_delivery_platformrz    s=       D(((t$T**+++rf   c                     |                                  }t                              |          }|r|S t          |          S )zReturn the env var name for a platform's cron home channel.

    Built-in platforms are in ``_HOME_TARGET_ENV_VARS``; plugin platforms are
    resolved from the platform registry.
    )rW   _HOME_TARGET_ENV_VARSr@   rw  )r"  r>   env_vars      r    _resolve_home_env_varr~    sC       D#''--G %%%rf   c                     	 ddl m}m}  |            } ||                                           }|                    |          S # t
          $ r! t                              d| d           Y dS w xY w)u  Return the persisted ``HomeChannel`` for a platform from gateway config.

    ``/sethome`` declares ``config.yaml`` canonical (it is the only store that
    survives for relay-fronted logical platforms, whose adapters are not
    natively enabled) and mirrors the value into the legacy
    ``<PLATFORM>_HOME_CHANNEL`` env var only as a best-effort compatibility
    shim.  Cron historically read ONLY the env mirror, so a home channel that
    existed solely in config.yaml — e.g. Discord fronted by the relay
    connector, where no ``DISCORD_HOME_CHANNEL`` was ever exported — was
    invisible and jobs silently fell back to local-only.  Reading the
    canonical store here fixes that for every relay-fronted platform at once.
    r   load_gateway_configrJ  z1config home_channel lookup failed for platform %rTexc_infoN)rR  r  rJ  rW   get_home_channelr   r   rK  )r"  r  rJ  configr  s        r    _get_config_home_channelr  &  s    @@@@@@@@$$&&8M//1122&&x000   ?D 	 	
 	
 	
 tts   AA 'A10A1c                     t          |           }|sdS t          j        |d          }|s1t                              |          }|rt          j        |d          }|S )zDReturn the home chat id from the legacy env mirror only (no config).r#   )r~  r  r  _LEGACY_HOME_TARGET_ENV_VARSr@   )r"  r}  rm  legacys       r    _env_home_target_chat_idr  A  sf    #M22G rIgr""E *-11':: 	*Ifb))ELrf   c                     t          |           }|r|S t          |           }||j        rt          |j                  S dS )u)  Return the configured home target chat/room ID for a delivery platform.

    Resolution order: platform env var (legacy mirror, kept first so an
    operator override keeps winning) → legacy env var name → the canonical
    ``home_channel`` block persisted in config.yaml by ``/sethome``.
    Nr#   )r  r  r  rw   )r"  rm  homes      r    _get_home_target_chat_idr  N  sM     %]33E #M22DDL4<   2rf   c                    t          |           }|                                 dk    r+t          j        dd                                          }|r|S |r*t          j        | dd                                          nd}|sH|rFt
                              |          }|r*t          j        | dd                                          }|r|S t          |           s,t          |           }||j	        rt          |j	                  S dS )u?  Return the optional thread/topic ID for a platform home target.

    Telegram-only override: ``TELEGRAM_CRON_THREAD_ID`` takes precedence over
    ``TELEGRAM_HOME_CHANNEL_THREAD_ID`` for cron delivery. When topic mode is
    enabled, deliveries that land in the root DM (thread_id unset) end up in
    the system-only lobby where the user cannot reply — the gateway returns
    the lobby reminder and drops ``reply_to_message_id`` (#24409). Pointing
    cron at a dedicated topic via this env var lets replies work as expected
    without changing the lobby invariant.
    r   TELEGRAM_CRON_THREAD_IDr#   
_THREAD_IDN)r~  rW   r  r  r%   r  r@   r  r  r#  rw   )r"  r}  cron_threadrm  r  r  s         r    _get_home_target_thread_idr  ^  s%    $M22G
**i 92>>DDFF 	=DLBI,,,b1177999"E AW A-11':: 	AI333R88>>@@E 
 $M22 ''66t~&&&4rf   c               #      K   t           D ]} | V  	 ddlm}  |             ddlm} |                                D ] }|j        r|j        t           vr	|j        V  !dS # t          $ r Y dS w xY w)zIterate built-in + plugin platform names that expose a home channel.

    Used by the ``deliver=origin`` fallback when the job has no origin.
    r   rp  rr  N)	r|  rt  rq  ru  rs  plugin_entriesrv  r>   r   )r>   rq  rs  rL  s       r    _iter_home_target_platformsr    s      
 &  



777777??????&5577 	! 	!E) !ej@U.U.Uj   	! 	!    s   AA 
A-,A-	connectedc                     d| vrt                      S 	 ddlm}  |            S # t          $ r- t                              dd           t                      cY S w xY w)a  Logical platforms deliverable through a connected relay connector.

    ``get_connected_platforms()`` only sees NATIVELY configured platforms.
    On a relay-fronted deployment (relay in ``config.platforms``, the real
    platform credential living in the connector) the fronted platforms are
    absent from that set although fire-time routing delivers to them via
    ``resolve_delivery_transport`` + ``RelayAdapter.fronts_platform``. This
    keeps validation symmetric with routing by consulting the same
    env-derived deploy stamp (``GATEWAY_RELAY_PLATFORMS``) the live
    adapter's identity set is seeded from. No relay connected -> empty set,
    so native topologies keep the strict credential check unchanged.
    relayr   )relay_fronted_platformsz$relay fronted-platform lookup failedTr  )r   gateway.relayr  r   r   rK  )r  r  s     r    !_relay_fronted_delivery_platformsr    s}     iuu999999&&(((   ;dKKKuus   $ 4AAc            
      $   g } 	 ddl m}  |            }d |                                D             }|t          |          z  }n:# t          $ r- t
                              dd           t                      }Y nw xY wt                      D ]}||vrt          |          st          |          }|                     ||                    dd                                          t          t          |                    |pd	d
           | S )u  Return the platforms a cron job can auto-deliver to.

    Single source of truth for any UI (dashboard dropdown, etc.) that lets a
    user pick a cron delivery target. A platform is included when it is a valid
    cron delivery platform AND its gateway is configured (enabled + credentials
    present). Each entry reports whether the platform's home target (the
    room/channel cron posts to) is set — a platform can be configured for
    interactive use but still lack the home target an unattended cron job needs.

    Returns a list of dicts: ``{"id", "name", "home_target_set", "home_env_var"}``
    ordered by the gateway's canonical platform order. Callers should always
    prepend the implicit ``local`` option themselves — it needs no config.
    r   r  c                     h | ]	}|j         
S rm   rm  r   ps     r    r  z(cron_delivery_targets.<locals>.<setcomp>  s    OOOQWOOOrf   z1cron_delivery_targets: gateway config unavailableTr  _rT   N)r?   r>   home_target_sethome_env_var)rR  r  get_connected_platformsr  r   r   rK  r   r  rz  r~  rx   ri  r,   r   r  )targetsr  gateway_configr  r>   r}  s         r    cron_delivery_targetsr    sN    G666666,,..OOn&L&L&N&NOOO	6yAAA		   HSWXXXEE			 ,-- 
 
y  *400 	'--S#..4466#'(@(F(F#G#G '4	 	
 	
 	
 	
 Ns   A A 4A<;A<c                 J   t          |                     d          pd                                          dk    rdS |                     d          sdS t          d          }t	          |          o2t          |                     d                    t          |          k    S )u*  True when a Slack origin's thread is a stale creation-turn artifact.

    Relay-fronted Slack in thread-per-message mode stamps each top-level
    message's own id as the session thread (a session KEY, not a durable
    location). Jobs persisted before origin capture learned to drop that
    stamp carry it as ``origin.thread_id`` forever. Heuristic that repairs
    them at fire time without touching genuine threads: when the origin
    chat IS the configured Slack home chat (the ``/sethome`` conversation),
    a pinned origin thread is the creation-message artifact — the user's
    delivery expectation for their home conversation is top-level (or the
    home target's own configured thread). Non-home chats keep their
    threads: a job deliberately created inside a working thread stays there.
    r  r#   r   Fr#  r  )rw   r@   rW   r  r   )r  	home_chats     r    _origin_thread_is_staler    s     6::j!!'R((..00G;;u::k"" u(11I	??Ks6::i#8#899S^^KKrf   c                 t    t          |           rt          d          }|r|ndS |                     d          S )zAThe thread a deliver=origin job should use, stale stamps dropped.r   Nr#  )r  r  r@   )r  home_threads     r    _origin_delivery_threadr    s@    v&& 4099)3{{t3::k"""rf   deliver_valuec                 2   t          |           }|dk    rdS |dk    r|r,|d         t          |d                   t          |          dS t                      D ]l}t	          |          }|rYt
                              d|                     d|                     d	d
                    |           ||t          |          dc S mdS d|v r'|	                    dd          \  }}|
                                }ddlm}m}  |              |||d          \  }}	}
|
rt
                              d||
           dS |	|dk    r|rt          |                    d          pd          
                                |k    rlt          |                    d                    t          |          k    r9|                    d          r$t          |          s|                    d          }	|||	dS |}|ri|                    d          |k    rPt	          |          }|r||t          |          dS |t          |d                   |                    d          dS t!          |          sdS t	          |          }|sdS ||t          |          dS )z9Resolve one concrete auto-delivery target for a cron job.localNr  r  r  )r  r  r#  zJJob '%s' has deliver=origin but no origin; falling back to %s home channelr>   r?   r   :r=   r   )prepare_send_message_platformsresolve_send_targetT)pass_unresolved_referencesz%Invalid cron delivery target '%s': %sr   r#   r#  )r  rw   r  r  r  r   r   r@   r  splitrW   tools.send_message_toolr  r  r   r  rz  )r4   r  r  r"  r  restplatform_keyr  r  r#  resolution_errors              r    _resolve_single_delivery_targetr    sn    S!!Ft   	":.vi0114V<<   9:: 	 	M.}==G 
`GGFCGGD#$6$677!   !.&!;M!J!J    
 t
m+11#q99t$**,,	
 	
 	
 	
 	
 	
 	
 	

 	'&((( 0C/B$40
 0
 0
,,  	NN7   
 4 '' (FJJz**0b117799\IIFJJy))**c'll::

;'' ;+F33 ; 

;//I &"
 
 	
 "M 
&**Z((M99*=99 	)"7FF   &6),--K00
 
 	
 '}55 t&}55G t "/>>  rf   c                     | | dk    rdS t          | t          t          f          r%d | D             }|rd                    |          ndS t	          |           S )u  Normalize a stored/submitted ``deliver`` value to its canonical string form.

    The contract is that ``deliver`` is a string (``"local"``, ``"origin"``,
    ``"telegram"``, ``"telegram:-1001:17"``, or comma-separated combinations).
    Historically some callers — MCP clients passing an array, direct edits of
    ``jobs.json``, or stale code paths — have stored a list/tuple like
    ``["telegram"]``.  ``str(["telegram"])`` would serialize to the literal
    string ``"['telegram']"``, which is not a known platform and fails
    resolution silently.  Flatten lists/tuples into a comma-separated string
    so both forms work.  Returns ``"local"`` for anything falsy.
    Nr#   r  c                     g | ]D}t          |                                          #t          |                                          ES rm   rw   r%   r  s     r    r   z,_normalize_deliver_value.<locals>.<listcomp>h	  s9    CCCACFFLLNNCQCCCrf   ,)rB   r   tuplerj  rw   )deliverpartss     r    _normalize_deliver_valuer  Y	  sd     'R--w'D%=)) 5CCCCC"'4sxxW4w<<rf   allpartc                     |                                  }|t          vr| gS g }t                      D ]&}t          |          r|                    |           '|S )a@  Expand a routing-intent token to concrete platform names.

    ``all`` expands to every platform in ``_iter_home_target_platforms()``
    that has a configured home chat_id right now.  Unknown / non-token
    values pass through unchanged as a single-element list, so the caller
    can treat every token uniformly.
    )rW   _ROUTING_TOKENSr  r  rx   )r  r  expandedr"  s       r    _expand_routing_tokensr  t	  se     JJLLEO##vH466 + +#M22 	+OOM***Orf   c                    t          |                     dd                    }|dk    rg S d |                    d          D             }g }|D ]$}|                    t	          |                     %t                      }g }|D ]}t          | |          }|rq|d                                         t          |d                   |                    d          f}	|	|vr*|	                    |	           |
                    |           |S )a  Resolve all concrete auto-delivery targets for a cron job.

    Accepts the legacy comma-separated ``deliver`` string plus the
    ``all`` routing-intent token, which expands to every platform with
    a configured home channel.  Tokens may be combined with explicit
    targets: ``origin,all`` and ``all,telegram:-100:17`` both work.
    Duplicate (platform, chat_id, thread_id) tuples are collapsed by the
    existing dedup pass.
    r  r  c                 ^    g | ]*}|                                 |                                 +S rm   r%   r  s     r    r   z-_resolve_delivery_targets.<locals>.<listcomp>	  s-    DDDq!''))DDDDrf   r  r  r  r#  )r  r@   r  r  r  r   r  rW   rw   r   rx   )
r4   r  	raw_partsr  r  seenr  r  targetrl  s
             r    _resolve_delivery_targetsr  	  s    'swwy''B'BCCG'	DDGMM#$6$6DDDI E 2 2+C00111155DG ' '0d;; 	'*%++--s6)3D/E/EvzzR]G^G^_C$v&&&Nrf   c                 8    t          |           }|r|d         ndS )zAResolve the concrete auto-delivery target for a cron job, if any.r   N)r  )r4   r  s     r    _resolve_delivery_targetr  	  s"    ',,G *71::d*rf   >   .3gp.avi.mkv.mov.mp4.webm>   .gif.jpg.png.jpeg.webpmedia_filesmetadatac           	         ddl m} ddlm}m}	 g }
d |pg D             }|                    |          }d |D             }|D ]Y\  }}	 ddlm}  ||          |vr|
                    d|            2# t          $ r |
                    d|            Y Vw xY w|D ]`\  }}	  ||          j	        
                                }||nt          | d	d          } |	|||
          r|                     |||          }n\|t          v r|                     |||          }n:|t          v r|                     |||          }n|                     |||          }ddlm}  |||          }|Od| d}t(                              d|                    dd          |           |
                    |           |
c S 	 |                    t1                                }n## t2          $ r |                                  w xY w|rmt          |dd          s\d| dt          |dd           }t(                              d|                    dd          |           |
                    |           # t          $ ry}d| dt7          |          pt9          |          j         }t(                              d|                    dd          |           |
                    |           Y d}~Zd}~ww xY w|
S )u  Send extracted MEDIA files as native platform attachments via a live adapter.

    Routes each file to the appropriate adapter method (send_voice, send_image_file,
    send_video, send_document) based on file extension — mirroring the routing logic
    in ``BasePlatformAdapter._process_message_background``.

    Returns a list of per-file error strings (empty when every attachment
    delivered). Callers surface these into the job's delivery errors so a
    dropped attachment is visible in ``last_error``/run status instead of
    only in the gateway log (the silent-drop half of the manual-run
    attachment bug: text delivered, file vanished, job marked ok).
    r   r   )BasePlatformAdaptershould_send_media_as_audioc                 6    g | ]\  }}t          |          |fS rm   rw   r   r  vs      r    r   z+_send_media_via_adapter.<locals>.<listcomp>	  s&    ===A#a&&!===rf   c                     h | ]\  }}|S rm   rm   )r   r  r  s      r    r  z*_send_media_via_adapter.<locals>.<setcomp>	  s    &&&$!QA&&&rf   )validate_media_delivery_pathz)attachment dropped by media path policy: Nr  )is_voice)r  
audio_pathr  )r  
video_pathr  )r  
image_pathr  )r  	file_pathr  r8  zcannot send media z: gateway loop unavailableJob '%s': %sr?   r   r  successTzmedia send failed for : rI   unknownzfailed to send media )pathlibr   gateway.platforms.baser  r  filter_media_delivery_pathsr  rx   r   suffixrW   r(   
send_voice_VIDEO_EXTS
send_video_IMAGE_EXTSsend_image_filesend_documentr<  r9  r   r   r@   r   _get_media_send_timeoutTimeoutErrorcancelrw   typeri   )r=  r  r  r  r>  r4   r  r   r  r  errors	requestedkeptraw_path_vr  
media_path	_is_voiceextroute_platformrB  r9  r   msgr   rO  s                             r    _send_media_via_adapterr
  	  s   * VVVVVVVVF==+*;===I%AA+NNK '&+&&&D! 	R 	R"	RKKKKKK++H55TAAJJJ    	R 	R 	RMMPhPPQQQQQ	R "- - -
I,	$z"")//11C)1)=XX77T^`dCeCeN)).#	RRR g))'j[c)dd##))'j[c)dd##..w:`h.ii,,W
]e,ffBBBBBB--dD99F~Q:QQQ~swwtS/A/A3GGGc"""	  /F/H/HII     #gfi>> #=Z = =vw	::= =  ~swwtS/A/A3GGGc""" 	 	 	
 S
RRc!ff6PQ@PRR  NN>3774+=+=sCCCMM#	 MsE   +A44"BB%DI:"G&%I:& HA2I::
K=A.K88K=c                 h    | dS t          | d          sdS t          t          | d                    S )u  Return True only if ``send_result`` unambiguously confirms delivery.

    A live adapter that returns ``None`` (e.g. a swallowed exception, a busy
    platform, or a code path that returns early without producing a
    ``SendResult``) must NOT be treated as success — doing so causes the
    scheduler to log ``"delivered to <chat> via live adapter"`` while the
    gateway never actually sees the message (#47056).

    Likewise, an object missing a ``success`` attribute (e.g. a bare ``dict``
    or a partial mock) is a contract violation: it does not actually tell us
    whether the send succeeded.  Require an explicit, truthy ``success``
    attribute to count as confirmed.
    NFr  )hasattrr   r(   )send_results    r    _confirm_adapter_deliveryr  
  s>     u;	** uY//000rf   runtime_adapterr>  c                 &   t          t          |           dd          }t          |          sdS 	 ddlm}  | || t          |                    |          }|dS |                    d          }n/# t          $ r" t          	                    d||d	
           Y dS w xY wt          |t                    o9t          |                    d          pd                                          dk    }|rt                              d||           |S )u  Decide whether an (already-ambiguous) Telegram topic target is a genuine
    Bot API *channel* Direct-Messages topic (route via
    ``direct_messages_topic_id``) rather than a forum-style topic in a private
    chat (route via ``message_thread_id``).

    Callers gate this on the ambiguous shape first
    (``telegram:<positive_chat_id>:<numeric_thread_id>``) — that shape is
    identical for both cases, so shape alone cannot decide (this was the #52060
    regression).  The real signal is the chat *type*: a genuine channel DM topic
    lives on a ``channel`` chat.  Probe the live adapter's ``get_chat_info`` once
    and only return True when the chat is a channel.

    Fails SAFE to ``message_thread_id`` (returns False) for adapters without a
    probe, or any probe error/timeout — that is the pre-#22773 behaviour and the
    correct default for the common forum-topic case.
    get_chat_infoNFr   r8  
   r  u\   Job '%s': get_chat_info probe failed for chat=%s — defaulting to message_thread_id routingTr  r  r#   channeluG   Job '%s': chat=%s is a channel — routing via direct_messages_topic_id)r(   r  r;  r<  r9  rw   r   r   r   rK  rB   rC   r@   rW   r   )	r  r  r>  ra   r  r9  r   r   
is_channels	            r    _is_channel_dm_topicr  $
  sO   4 D11?DIIMM"" u>>>>>>))M/3w<<88$
 
 >5 }}R}((   6Gd 	 	
 	
 	

 uu D$''\C0@0@0FB,G,G,M,M,O,OS\,\J 
UG	
 	
 	
 s   +A5 A5 5(B! B!contentc                 z$  ] t          |           }|st          |                     dd                    }|dk    rdS |dk    rEt                              d|                     d|                     dd                               dS d	| }t                              d
| d         |           |S ddlm} ddlm	}m
}	 d}
d}	 t                      }|                    di                               dd          }
n# t          $ r Y nw xY w|
rB|                     d| d                   }|                     dd          }d| d| d| d| d	}n|}ddlm} ddlm}  ||           |                    |          \  }}d |D             }|                    |          }t'          |          t'          |          z
  }|dk    r| dgng }	 t)          | |          }n# t          $ r d}Y nw xY w|                    |          \  }}|pd                                }	  |            }n@# t          $ r3}d| }t                              d
| d         |           |cY d}~S d}~ww xY wg }|D ]}|d         }|d         }|                    d          }t/          |           pi } |                     d          }!|!r;|s9t                              d | d         |!|                     dd          |           n&|r$t                              d!| d         |||           t3          | |||          }"|o|"}#|"r|                     d"          nd}$	  |	|                                          }%nU# t6          t8          f$ rA d#| d$}t                              d
| d         |           |                    |           Y Pw xY wdd%lm}&  |&|%||          }'|'|'j         }(|'j!        })n|j"                            |%          }(d})|'|'j#        r|(dd&lm$}*  |*d'          }(nH|(r|(j%        s?d(| d)}t                              d
| d         |           |                    |           |)duo|duo tM          |d*d+                       }+d},g }-tO          |(|          }.|.d,k    }/|/r|)tM          |)d-d          }0tQ          |0          r,	 tS           |0|                    }1n0# t          $ r d}1Y n"w xY wtS          tM          |)d.d                    }1|1s2t                              d/|                     dd          |           d}/|/r|"r|+rd}tU          |                     d0          pd                                          }2|2d1k    p$|2 o!tU          |          +                    d2          }3d}4d}5d}6|#r |/s|)||stY          | |)||          }7|7r|7}|7}6|+rdd3lm-}8m.}9m/}:m0}; |%|	j1        k    o3|duo/ |;tU          |                    o |:tU          |                    }<|<ote          |)||| d                   }=|=r,d}>tU          |          | d         d4}?d5tU          |          i}@n,|tU          |          nd}>d6| d         i}?|>r|>|?d<   |rd|ind}@|"rx|                     d7          rc|?3                    d7tU          | d7                              ti          |@pi           }@|@3                    d7tU          | d7                              	 |                                }Ad}Bd}Cd}D|Ardd8l5m6}E  |8||          }F |9|%tU          |          |>d9          }G |E|F7                    |G|A|?          |          }H|Hd}B|-                    d:           nd}Id}J	 |H8                    d;<          }In# tr          $ r |H:                                }K|KrEd=| d>| d?}t                              d@| d         |           |-                    |           d}Bd}Jn'd}Cd}Jt                              dA| d         ||           Y n*t          $ r}L|-                    dB|L             d}L~Lww xY w|Jrntw          |Ith                    rNtS          |I                    dCd                    }M|I                    dD          }N|I                    dE          }Dn1ty          |I          }MtM          |IdDd          }NtM          |IdEd          }D|Mstw          |Ith                    r|I                    dFdG          }OdH}Pn,|I&tM          |IdFd          }Ot{          |I          j>        }PndI}OdJ}Pd=| d>| dK|P dL|O dM	}|'*|'j#        r#t                              d
| d         |           n"t                              d@| d         |           |-                    |           d}Bns|Nrq|ro|N                    dN          rZ|N                    dO          p|}QdP|Q dQ| d>| dR}t                              d
| d         |           |                    |           |Br|Cs|rti          |@pi           }R|'U|'j#        rN|%j?        |RdS<   |@                    |%          }S|S-|SjA        |k    r"|SjB        r
|SjB        |Rd"<   |SjC        r
|SjC        |Rd7<   t          |)|||Rpd|| |%T          }T|TD ]"}U|U dU| d>| dM}V|                    |V           #nS|CrQ|rOt'          |           dV| d>| dW}t                              d
| d         |           |                    |           |Brt                              dX| d         ||           d},|6rB|5s@t          | |)|||6||                     dY          |3|                     d7          Z	  	         d}5|/r|"r|5st          | |)||||3|$|                     dY          |                     d7          [	  	        }4|4s#t                              d\| d         ||           |DrKt          | |)||tU          |D          ||                     dY          |3|                     d7          Z	  	         nc|/ra|"s_t                              d]| d         |||                     d          |                     d          |                     d                     t          | |||||$|#o|5 o|4 ^           n# t          $ r}d_| d>| d`| ]t          ]fda|-D                       s|-                    ]           |'*|'j#        r#t                              d
| d         ]           n"t                              d@| d         ]           Y d}~nd}~ww xY w|,sv|'<|'j#        r5|-s|-                    db| d>| dc           |I                    |-           `t                      rWdd| d>| de}t                              d
| d         |           |-                    |           |I                    |-            ||%|(||||f          }W	 t          jL        |W          }Xn# t          $ r}Y|WN                                 t          |Y          r[dd| d>| de}t                              d
| d         |           |-                    |           |I                    |-           Y d}Y~Yv	 t          jP        Q                    dgh          }Z	 |ZR                    t          jL         ||%|(||||f                    }H|H8                    di<          }X|ZS                    dj           n# |ZS                    dj           w xY wn# t          $ r}t          |          r_dd| d>| de}t                              d
| d         |           |-                    |           |I                    |-           Y d}~Y d}Y~Ydd| d>| d`| }t                              d
| d         |dk           |-I                    |g           |I                    |-           Y d}~Y d}Y~Yd}~ww xY wY d}Y~Ynud}Y~Ywt          $ re}dd| d>| d`| }t                              d
| d         |dk           |-I                    |g           |I                    |-           Y d}~fd}~ww xY w|Xrv|X                    dF          radl|XdF          dU| d>| dM}t                              d
| d         |           |-I                    |g           |I                    |-           tw          |Xth                    r|X                    dm          ndpg }[|[D ]E}\dn|\ dU| d>| dM}t                              d
| d         |           |                    |           Ft                              do| d         ||           t          | |||||$|#o|5 ^           |r|I                    |           |rdpT                    |          S dS )qu  
    Deliver job output to the configured target(s) (origin chat, specific platform, etc.).

    When ``adapters`` and ``loop`` are provided (gateway is running), tries to
    use the live adapter first — this supports E2EE rooms (e.g. Matrix) where
    the standalone HTTP path cannot encrypt.  Falls back to standalone send if
    the adapter path fails or is unavailable.

    Returns None on success, or an error string on failure.
    r  r  Nr  uk   Job '%s': deliver=origin but no origin or home channels — skipping delivery (output saved in last_output)r>   r?   r   z(no delivery target resolved for deliver=r  r   )_send_to_platformr  Tr8   wrap_responser#   zCronjob Response: z

(job_id: z)
-------------

zI

To stop or manage this job, send me a new message (e.g. "stop reminder z").)r  )apply_media_policy_envc                 6    g | ]\  }}t          |          |fS rm   r  r  s      r    r   z#_deliver_result.<locals>.<listcomp>
  s&    ;;;tq!A{;;;rf   z media attachment(s) dropped by media path policy (missing file, denied prefix, or strict-mode miss); see gateway.strict / media_delivery_allow_dirs in config.yamlFzfailed to load gateway config: r  r  r#  zUJob '%s': origin has thread_id=%s but delivery target lost it (deliver=%s, target=%s)z*Job '%s': delivering to %s:%s thread_id=%sr*  zunknown platform '')resolve_delivery_transport)PlatformConfigr'  z
platform 'z' not configured/enabled
is_runningc                      dS r  rm   rm   rf   r    <lambda>z!_deliver_result.<locals>.<lambda>%  s    E rf   r  +supports_inchannel_continuable_for_platformsupports_inchannel_continuablezOJob '%s': cron_continuable_surface=in_channel not supported on %s, using threadrP  rN  D)DeliveryRouterDeliveryTarget_looks_like_int#looks_like_telegram_private_chat_id)direct_messages_topic_idra   r)  ra   rG  r8  )r  r  r#  is_explicitz)live adapter event loop scheduling failedrr  r  zlive adapter send to r  z. timed out before the coroutine was dispatchedz(Job '%s': %s, falling back to standalonezJob '%s': live adapter send to %s:%s timed out after 60s; already dispatched (in flight), assuming delivered (skipping standalone fallback to avoid duplicate)zlive adapter send failed: r  raw_response
message_idrI   r  rC   zno response from adapterNonez returned unconfirmed result (z, error=)thread_fallbackrequested_thread_idzconfigured thread_id z for z+ was not found; delivered without thread_id_relay_logical_platform)r  z	 (target z& media attachment(s) not delivered to z& (live adapter confirmation timed out)z-Job '%s': delivered to %s:%s via live adapterrE  )rE  rF  rG  )rF  r*  rE  rG  uY   Job '%s': in_channel seed did NOT land on %s:%s — a plain reply will not see this briefu   Job '%s': in_channel delivery to %s:%s is not the origin conversation (origin=%s:%s thread=%s) — seed skipped, brief not continuable here)r#  r*  r(  zlive adapter delivery to z	 failed: c              3       K   | ]}|v V  	d S r   rm   )r   r  err_msgs     r    r   z"_deliver_result.<locals>.<genexpr>  s'      CCc7c>CCCCCCrf   zrelay delivery to z failedzdelivery to u)    skipped — interpreter is shutting down)r#  r  r=   r  r:  r  r  zdelivery error: warningszdelivery warning: zJob '%s': delivered to %s:%s; )Ur  r  r@   r   r   r   r  r  rR  r  rJ  r   r   r  r  gateway.media_policyr  extract_mediar  r\   r!  r%   rI   r  rK  r&  rW   r'   rT  rx   gateway.deliveryr  r  r=  	platformsis_relayr  r(  r(   r  r;  r   rw   rX   rD  r%  r&  r'  r(  TELEGRAMr  
setdefaultrC   r<  r9  _deliver_to_platformr   r  r  rB   r  r  ri   rm  r  r  r*  rG  r
  r]  r`  r5  r   r  r  asynciorunRuntimeErrorr   r  r  r  submitr  rj  )^r4   r  adaptersr>  r  r  r	  r  r  rJ  r  user_cfgr@  ra   delivery_contentr  r  r  cleaned_delivery_contentrequested_media_policy_droppedpolicy_drop_errorsmirror_enabledr  r)  r  rO  delivery_errorsr  r"  r  r#  r  r%  origin_targetmirror_this_targetorigin_user_idr  r  	transportr  r  r  live_adapter_ready	deliveredtarget_errorssurface_modein_channel_surfaceper_platform_checksurface_supportedorigin_chat_typeis_dm_targetinchannel_seededthread_seededopened_thread_idrC  r%  r&  r'  r(  is_ambiguous_telegram_topicroute_via_dm_topicroute_thread_idroute_metadatamedia_metadatatext_to_send
adapter_ok	timed_outdelivered_message_idr9  routerroute_targetr   r  timeout_handled	cancelledexsend_successsend_raw_responser  shaper0  routed_media_metadatalogical_home_media_errors_me_msgrB  r   run_errpool_sender_warnings_wr3  s^                                                                                                @r    _deliver_resultrv  \
  s    (,,G 0G1L1LMMG##4 H$$KKBc 2 233  
 4HHH~s4y#666
999999<<<<<<<<
 MH== VR0044_dKK     #GGFCI..	r""g g gg g g g Xa	g g g 	 # ;::::: <;;;;;8$$$,?,M,MN^,_,_)K);;{;;;O%AA+NNK /**S-=-==O Q	  L L L	
 	
  6sHEE    )66w??NA{$"++--K$$&&   333^SY444






 O \
 \
z*#JJ{++	 !%%+

;// 
	 
	NN*D	=#'')W*E*Ev   
  	LL<D	='9   /v}gyYY+= 3@II...T	x 3 3 5 566HHH% 	 	 	7}777CNN>3t9c:::""3'''H		 	@?????..xJJ	 &G'/OO &**844G"O Y%7  999999(.666 	 	F}FFFCNN>3t9c:::""3''' 4' =D =:lMM::<< 	
 	 2'=II)\9 	+/"= ")!N" " *++ .(,-?-?-N-N(O(O%%  . . .(-%%%. %)#%Eu* * % %! % + 'GGD#&&  
 &+" !	- !	4F !	B I vzz+66<"==CCEE'4/ 
  AS\\%<%<S%A%A 	 !& *.	1&	1  +  ! :_gt M  1 *	#0  U	            H-- 4T)477GEE4 $OC	NN33	 ( "= "AU$D	B B " Q #'03I!$i" " #=c)nn!M 5>4I#i...t"*CI!6" B2AN;/=F!P+y!9!9D  OJ!7!7 O))*c&:L6M6MNNN!%n&:!;!;))*c&:L6M6MNNNP  8==??!
!	'+$ I<JJJJJJ+^FH==F#1>!) #G"1$(	$ $ $L 6533((* 
  F ~%*
%,,-XYYYY&**/0"*0---*C*CKK+ (" (" ("$ )/I( "%TM %T %TG %T %T %T !$ !'$N$'Is!" !" !" !. 4 4S 9 9 9-2
26,0	26 &%: %(I}g!" !" !"  ) " " " *001Rb1R1RSSS!" + :< !  *+t<< `/3KOOIu4U4U/V/V4?OON4S4S 17B|7T7T 4 4/H/U/U4;KY]4^4^ 17>{LZ^7_7_ 4#/ "<#-k4#@#@ !3*5//'9*M*MC,2EE%0%<*1+w*M*MC,0,=,=,FEE*DC,2E%ZM %Z %ZG %Z %ZDI%Z %ZSV%Z %Z %Z !$ $-#8Y=O#8$*NN>3t9c$R$R$R$R$*NN(R(+D	3%& %& %& !. 4 4S 9 9 9-2

 1<$-< %6$9$9:K$L$L<
 7H6K6KLa6b6b6ofo 3%l<O %l %l'4%l %l7>%l %l %l !$ !'~s4y# N N N / 6 6s ; ; ;  0i 0K 0,01E2,F,F) ,1C,KS>-.GH'-'>'>x'H'H'38LPW8W8W+3 XCOCW 5i @+4 ZDPDY 5j A$;'#-5!)% % %M  - 5 5"%JJJJJJJ'..t44445  0; 0{++ [ [([ [+2[ [ [  NN>3t9cBBB#**3/// DKK OQTUYQZ\ikrsss $I ( - -1-,k&,jj&=&=".%+ZZ
%;%;    )- * &m &M &+E-'|$2&,jj&=&=%+ZZ
%;%;, , ,(  0 "NN!L #D	='   0 5 #_mW #$8 9 9;*0**[*A*A&2)/J)?)?    , M B  I}g"JJz22FJJy4I4I"JJ{33   0]G["+^ 2 a=7H aQaMa   
  
 
 
[m[[g[[XY[[CCCC]CCCCC 2!((111(Y-?(NN>3t9gFFFFNNBD	7  
  k	$);$ % !((M]MMWMMM    &&}555 *++ g]ggWggg~s4y#>>>$$S)))&&}555$$XwAYen  }H  I  I  ID3 T** + + +
 

 .g66 kkkkkkCNN>3t9cBBB!((---#**=999HHHH%-@@Q@OOD2!%W[:K:KHV]_f  iA  MV  do  ;p  ;p  ;p  "q  "q!'r!:!:51111511111     2!44 !o]ooWooo~s4y#FFF%,,S111'..}=== NNNNN1NNCLLTC$LOOO!((#///#**=999HHHHHHHH 21111    J]JJWJJqJJ^SYdKKK$$cU+++&&}555  	&**W-- 	
 ^]]=]]SZ]]]^SY<<<$$cU+++&&}555 +5VT*B*BL

:&&&   ' , ,R2RRRRRRR^SY<<<&&s++++KK6D	=RYZZZ']G[#^*@=/@     31222 *yy)))4s=  8C> >
D
DG# #G21G2$
H/ /
I,9(I'!I,'I,M22AOOS''S65S6Bs2%^<;s2<Ba1s2
	a1a,,a11R s22
v<B	v

v
y  AC+A8AA,* ~A}'~'}??~AA,
AA$A$AA1AA,;AAAAAA,AAA$A$AA,A,ACA9AACCACi  r;   c                  2   t           t          k    r[	 t          t          t                               } | dk    r| S n0# t          $ r# t
                              dt                      Y nw xY wt          j        dd          	                                }|rQ	 t          t          |                    } | dk    r| S n+# t          $ r t
                              d|           Y nw xY w	 t                      pi }t          |t                    r|                    di           ni }|                    d          }|$t          t          |                    } | dk    r| S n2# t          $ r%}t
                              d	|           Y d}~nd}~ww xY wt          S )
zOResolve cron pre-run script timeout from module/env/config with a safe default.r   z<Invalid patched _SCRIPT_TIMEOUT=%r; using env/config/defaultHERMES_CRON_SCRIPT_TIMEOUTr#   z;Invalid HERMES_CRON_SCRIPT_TIMEOUT=%r; using config/defaultr8   script_timeout_secondsNz2Failed to load cron script timeout from config: %s)_SCRIPT_TIMEOUT_DEFAULT_SCRIPT_TIMEOUTrA   r  r   r   r   r  r  r%   r   rB   rC   r@   rK  )rS   	env_valuer1   ry   
configuredr   s         r    _get_script_timeoutr~  ~  s   111	l%0011G{{  	l 	l 	lNNY[jkkkkk	l 	6;;AACCI e	e%	**++G{{  	e 	e 	eNNXZcddddd	e	Pmm!r*4S$*?*?G37762&&&R\\":;;
!%
++,,G{{ P P PI3OOOOOOOOP #"s:   (< *A)(A)#B; ;%C#"C#'A7E   
F*F

Fi,  c                  \   t          j        dd                                          } | rQ	 t          t	          |                     }|dk    r|S n+# t
          $ r t                              d|            Y nw xY w	 t                      pi }t          |t                    r|                    di           ni }|                    d          }|$t          t	          |                    }|dk    r|S n2# t
          $ r%}t                              d|           Y d}~nd}~ww xY wt          S )	ux  Resolve the per-attachment media-send timeout from env/config.

    Mirrors the ``script_timeout_seconds`` resolution pattern: the
    HERMES_CRON_MEDIA_SEND_TIMEOUT env var wins, then
    ``cron.media_send_timeout_seconds`` in config.yaml, then the default
    (300s — large attachments like long TTS audio can legitimately exceed
    the old fixed 30s upload window).
    HERMES_CRON_MEDIA_SEND_TIMEOUTr#   r   z?Invalid HERMES_CRON_MEDIA_SEND_TIMEOUT=%r; using config/defaultr8   media_send_timeout_secondsNz6Failed to load cron media-send timeout from config: %s)r  r  r%   rA   r  r   r   r   r   rB   rC   r@   rK  _DEFAULT_MEDIA_SEND_TIMEOUT)r|  rS   r1   ry   r}  r   s         r    r  r    s^    	:B??EEGGI 		%	**++G{{  	 	 	NNQ    		Tmm!r*4S$*?*?G37762&&&R\\">??
!%
++,,G{{ T T TMsSSSSSSSST '&s*   #A %A87A8<A7C5 5
D$?DD$venv_dirc                 H   | dz  }	 |                     d                                          }n# t          $ r i cY S w xY wi }|D ][}d|vr|                    dd          \  }}|                                ||                                                                <   \|S )Nz
pyvenv.cfgr=  r>  rg  r=   )	read_text
splitlinesrf  r  r%   rW   )r  cfg_pathlinesr   r  rl  rm  s          r    _read_windows_pyvenv_cfgr    s    ,&H""G"44??AA   			  F 4 4c>>YYsA&&
U&+kkmmsyy{{  ""##Ms   (0 ??
python_exec                 d   t           j        dk    r| i fS t          |           }|j        j        }i }|j                                        dk    r+|                    d          }|                                r|}t          |          }|	                    dd          }|dz  dz  }d|v r|rt          |          dz  }|                                r|                                r|}t          |          |d	<   t          t          t                                                    j        d
                   t          |          g}	t          j        	                    dd          }
|
r|	                    |
           t          j                            |	          |d<   t          |          |fS )a  Return an output-capable hidden Python invocation for Windows scripts.

    Cron scripts capture stdout/stderr, so using ``pythonw.exe`` directly can
    lose script output.  uv-created venv ``python.exe`` launchers are also a
    problem: even with CREATE_NO_WINDOW, the launcher can re-exec the base
    console interpreter and flash a visible window.  For uv venvs, bypass the
    launcher and run the base ``python.exe`` directly with the venv paths
    overlaid in the environment.
    win32zpythonw.exez
python.exer  r#   Libsite-packagesuvVIRTUAL_ENVr=   
PYTHONPATH)r  r  r   rE  r>   rW   	with_nameexistsr  r@   rw   __file__resolver:  r  environrx   pathseprj  )r  interpreterr  env_overlaysiblingr1   r  site_packagesbase_pythonpythonpath_entriesexisting_pythonpaths              r    _windows_cron_python_invocationr    s    |w2~z""K!(H"$K=00''55>> 	"!K
"8
,
,C7762Du$6Ms{{t{4jj</ 
	LM$8$8$:$: 
	L%K),XK&DNN**,,4Q788M""" #%*..r"B"B" ?"))*=>>>(*
8J(K(KK%{[((rf   procc                 2   |                                  dS t          j        dk    rp	 t          j        ddt          | j                  ddgddt                      d	
           nZ# t          t          j	        f$ r | 
                                 Y n-w xY w	 t          j        | j                  }n# t          t          f$ r d}Y nw xY w|	 t          j        |t          j                   n# t          t"          t          f$ r d}Y nw xY w|	 |                     d           n# t          j	        $ r Y nw xY w	 t          j        |d           n# t          t          f$ r d}Y nw xY w|R	 t          j        |t'          t          dt          j                             n# t          t"          t          f$ r Y nw xY w	 |                     d           dS # t          j	        $ r. | 
                                 |                     d           Y dS w xY w)zBBest-effort hard stop of a cron script and every child it spawned.Nr  taskkillz/PIDz/Tz/FTr  F)capture_outputrS   creationflagscheck      ?r  r   SIGKILL)pollr  r  
subprocessr@  rw   pidr   rf  TimeoutExpiredkillr  getpgidProcessLookupErrorkillpgr   SIGTERMPermissionErrorr  r(   )r  process_groups     r    _terminate_cron_script_processr    su   yy{{
|w		NVS]]D$?#022     23 	 	 	IIKKKKK		!+-:dh+?+?MM"G, 	! 	! 	! MMM	!$%	-8888&A % % % $%(IIcI****!0   D)ImQ////*G4 ) ) )$(MMM) ,	-FN1[1[\\\\.I   		#	$   				#	s}   <A& &*BBB2 2CCC. .D
	D
D' 'D98D9=E E)(E)/3F# #F=<F=G 9HHc                 x   	 |                      d           dS # t          j        $ r Y nw xY w	 |                                  n# t          $ r Y nw xY w| j        | j        fD ])}	 ||                                 # t          $ r Y &w xY w	 |                     d           dS # t          j        $ r Y dS w xY w)ue  Reap a terminated script process without ever blocking indefinitely.

    A descendant that survived the tree kill can hold the pipe write ends
    open, so a bare ``communicate()`` would wait for EOF forever.  Bound the
    drain, then abandon the pipes — the caller only needs the process reaped
    and the worker thread unblocked, not the output.
          @r  N)	communicater  r  r  rf  stdoutstderrr   r  )r  streams     r    _drain_script_pipesr  1  s   %%%$   		   ;,  	! 	 	 	D			#	$    	sA    ,,A 
AA&A==
B
	B
B& &B98B9r  script_pathc                     t          |                    dd                    dz  dz  }|                                st                              d|           | |gS dt          |          d}| d||gS )	u  Bootstrap a cron script under the base interpreter with ``.pth`` support.

    The uv-venv overlay mode runs the base ``python.exe`` (to avoid the
    launcher re-execing a console interpreter and flashing a window) and
    re-attaches the venv via ``PYTHONPATH``.  But ``PYTHONPATH`` entries are
    plain ``sys.path`` additions — Python's site initialization never
    processes ``.pth`` files for them (only ``site.addsitedir()`` does) — so
    editable installs (``pip install -e``, ``__editable__*.pth`` links) are
    invisible to cron script jobs.

    Bootstrap with ``site.addsitedir()`` on the venv ``site-packages``, then
    exec the script as ``__main__``.  ``runpy.run_path`` keeps ``__file__``
    correct; ``sys.path[0]`` is set to the script's directory to preserve the
    ``python script.py`` import semantics.  Note: ``runpy`` does not set
    ``__package__``/``__spec__`` the way a direct invocation does, so
    package-relative imports (``from . import x``) may behave differently.
    Falls back to a plain invocation if the venv layout is unresolvable —
    the pre-existing PYTHONPATH behaviour is strictly better than failing
    to run at all.
    r  r#   r  r  z}Windows cron script: venv site-packages %s not found; running without .pth processing (editable installs may be unimportable)z,import os, runpy, site, sys;site.addsitedir(z);script = sys.argv[1];sys.argv = [script] + sys.argv[2:];sys.path.insert(0, os.path.dirname(os.path.abspath(script)));runpy.run_path(script, run_name='__main__')z-c)r   r@   is_dirr   r   rw   )r  r  r  r  	bootstraps        r    _windows_cron_bootstrap_argvr  P  s    2 ;;<<uDVM!! ) 	N	
 	
 	

 K((	6}--	6 	6 	6  i55rf   workdircancel_eventc           	      	   t                      dz  }|                    dd           |                                }dt          |           v rdd| fS 	 t	          |                                           }n$# t          t          t          f$ r
 dd| fcY S w xY w|	                                r|                                }n||z                                  }	 |
                    |           n# t          $ r dd| d	| fcY S w xY w|                                sdd
| fS |                                sdd| fS t                      }|j                                        }|dv rZt!          j        d          p"t$          j                            d          rdnd}	|	dd|j        dfS |	t          |          g}
i }nNt-          t.          j                  \  }}|rt3          ||t          |                    }
n|t          |          g}
	 ddlm} ddi}t.          j        dk    r)t;                      t=          t>          dd          z  ddd} |            }|                     |           |pt          |j!                  }t?          j"        |
ft>          j#        t>          j#        d||d|}tI          j%                    |z   }	 |4|&                                r tO          |           tQ          |           dS |tI          j%                    z
  }|dk    r(tO          |           tQ          |           dd| d| fS 	 |)                    tU          d|                    \  }}n# t>          j+        $ r Y w xY w|pd ,                                }|pd ,                                }	 dd!l-m.}  ||          } ||          }n6# t^          $ r)}t`          1                    d"|           d#}d#}Y d}~nd}~ww xY w|j2        dk    rVd$|j2         g}|r|3                    d%|            |r|3                    d&|            dd'4                    |          fS d|fS # t^          $ r}dd(| fcY d}~S d}~ww xY w))u  Execute a cron job's data-collection script and capture its output.

    Scripts must reside within HERMES_HOME/scripts/.  Both relative and
    absolute paths are resolved and validated against this directory to
    prevent arbitrary script execution via path traversal or absolute
    path injection.

    Supported interpreters (chosen by file extension):

    * ``.sh`` / ``.bash`` — run with ``/bin/bash``
    * anything else — run with the current Python interpreter
      (``sys.executable``), preserving the original behaviour for
      Python-based pre-check and data-collection scripts.

    Shell support lets ``no_agent=True`` jobs ship classic bash watchdogs
    (the `memory-watchdog.sh` pattern) without wrapping them in Python.

    Subprocess environment is passed through ``_sanitize_subprocess_env`` so
    provider credentials and other Hermes-managed secrets are not inherited
    (SECURITY.md §2.3), matching terminal and MCP child processes.

    Args:
        script_path: Path to the script.  Relative paths are resolved
            against HERMES_HOME/scripts/.  Absolute and ~-prefixed paths
            are also validated to ensure they stay within the scripts dir.
        workdir: Optional absolute path to use as the script's cwd.
            When set, the subprocess runs in this directory instead of
            the scripts-dir parent.  The Python process cwd is NEVER
            mutated, avoiding the global-side-effect bug where a cron
            job's ``os.chdir()`` leaks into concurrent gateway sessions
            (#69396).

    Returns:
        (success, output) — on failure *output* contains the error message so the
        LLM can report the problem to the user.
    scriptsTr9   Fz*Blocked: script path contains a NUL byte: z5Blocked: script path is not a valid filesystem path: z=Blocked: script path resolves outside the scripts directory (z): zScript not found: zScript path is not a file: >   .sh.bashbashz	/bin/bashNzCannot run .sh/.bash script z{: bash not found on PATH. On Windows, install Git for Windows (which ships Git Bash) or rewrite the script as Python (.py).r   )build_subprocess_envstart_new_sessionr  CREATE_NEW_PROCESS_GROUPr=  ri  )r  r?  r   )r  r  r_   cwdenv)Fz5Script cancelled because cron fire ownership was lostzScript timed out after zs: g?r  r#   )redact_sensitive_textz/Failed to redact sensitive text from output: %sz[REDACTED - redaction failed]zScript exited with code zstderr:
zstdout:
r@  Script execution failed: )5rD  rF  r  rw   r   
expanduserr'   rA  rf  is_absoluterelative_tor  is_filer~  r  rW   shutilwhichr  rM  isfiler>   r  r  
executabler  tools.environments.localr  r  r   r(   r  r  rE  PopenPIPEr   r  r   r  r  r  minr  r%   agent.redactr  r   r   r   
returncoderx   rj  )r  r  r  scripts_dirscripts_dir_resolvedr  rM  script_timeoutr  _bashargvr  r  r  popen_kwargsr  _script_cwdr  r  r  
stdout_raw
stderr_rawr  r  r  rO  r  r   s                               r    _run_job_scriptr  ~  s   R #$$y0KdT222&..00 [!!!!R;RRRR	^;**,,g. ^ ^ ^ ]k]]]]]]^  -{{}}c!**,,
-.... 
 
 
9$9 9)49 9
 	
 	
 	

 ;;== 2141111<<>> ;:D::::(**N [  F!!! V$$ 
7>>+66@KKD 	 =9ty 9 9 9
 

 s4yy!&("A#."Q"Q
K 	+ 0
KTSSDDD		*DD8AAAAAA(;T'B<7""!3!5!5*&@!DD"E##	 L #"$$

;
 1T[!1!1
??
 
 
 
 >##n4	'L,?,?,A,A'.t444#D)))UU 4>#3#33IA~~.t444#D)))QQQ4QQQQ)-)9)9#c9BUBU)9)V)V&
J,    "))++"))++	5::::::**622F**622FF 	5 	5 	5NNLaPPP4F4FFFFFF	5
 ?aAAABE 3111222 3111222$))E****V| 8 8 87#7777777778s   !A8 8BBC3 3D
	D
C/Q/ AQ/ 'M5 4Q/ 5NQ/ N/Q/ 7O Q/ 
PP=Q/ PA#Q/ +Q/ /
R
9R?R
R
c                   	
 |                      d          }|                      d          }t          |t                    r$t          |                     d          pd          nd
t          |t                    r|                     d          dk    r
st	          |||          S t          |                      d          pd          	t          j                    t          j                    }d	
fd}t          j	        |j
        |fdd          }	 |                                 n?# t          $ r2 t                              d	d           t	          |||          cY S w xY w	 t	          |||                                           |                    d           S #                                  |                    d           w xY w)a  Run a cron script while keeping its owned one-shot claim fresh.

    Script execution is synchronous and may legitimately outlive the stale
    claim TTL.  Without a concurrent heartbeat, another scheduler process can
    mistake the live run for a dead owner and dispatch the same one-shot again.
    Recurring jobs and unclaimed/manual runs have no durable one-shot claim and
    therefore use the ordinary script path without starting a thread.

    The claim owner is captured from the dispatched job and never re-read from
    storage.  ``heartbeat_run_claim`` compares that stable owner before every
    refresh, so a stale runner cannot extend a replacement owner's claim.
    r6   	run_claimbyr#   r7   oncer  r  r?   r   Nc                                           t                    s\	 t                      n-# t          $ r  t                              d d           Y nw xY w                     t                    Zd S d S )Nexpected_ownerz+Job '%s': script run_claim heartbeat failedTr  )r  _RUN_CLAIM_HEARTBEAT_SECONDSr   r   r   rK  )ra   r  stops   r    _heartbeat_loopz=_run_job_script_with_claim_heartbeat.<locals>._heartbeat_loop^  s    ))899 	#F5AAAAA   A!       ))899 	 	 	 	 	s   / 'AAzcron-script-claim-heartbeatTr  argsr>   daemonz4Job '%s': could not start script run_claim heartbeatr  r  r  r   )r@   rB   rC   rw   r  r  Eventcontextvarscopy_contextThreadr@  startr   r   rK  r   rj  )r4   r  r  r  r6   claimheartbeat_contextr  heartbeat_threadra   r  r  s            @@@r    $_run_job_script_with_claim_heartbeatr  >  s   $ wwz""HGGK  E*4UD*A*AIC		$%2&&&rE8T""XLL  F** + {G,WWWW$"%%F?D#022	 	 	 	 	 	 	 	 !' $*	  X     X X XB 	 	
 	
 	

 {G,WWWWWWX+{G,WWW


 	c**** 	


 	c****s   D1 19E-,E-1F- -,Gscript_outputc                 F   | sdS d |                                  D             }|sdS |d                                         }	 t          j        |          }n# t          j        t
          f$ r Y dS w xY wt          |t                    sdS |                    dd          duS )u  Parse the last non-empty stdout line of a cron job's pre-check script
    as a wake gate.

    The convention (ported from nanoclaw #1232): if the last stdout line is
    JSON like ``{"wakeAgent": false}``, the agent is skipped entirely — no
    LLM run, no delivery. Any other output (non-JSON, missing flag, gate
    absent, or ``wakeAgent: true``) means wake the agent normally.

    Returns True if the agent should wake, False to skip.
    Tc                 :    g | ]}|                                 |S rm   r  )r   r  s     r    r   z$_parse_wake_gate.<locals>.<listcomp>  s%    RRRtTZZ\\RdRRRrf   	wakeAgentF)	r  r%   rI  loadsJSONDecodeErrorr'   rB   rC   r@   )r  stripped_lines	last_linegates       r    _parse_wake_gater    s      tRR}'?'?'A'ARRRN tr"((**Iz)$$ *-   ttdD!! t88K&&e33s   A A10A1prerun_scriptextra_promptc                 j   t          |                     d          pd          }|r| d| }|}|                     d          }d}|                     d          }|r5||\  }}	nt          |          \  }}	|r|	rd|	 d	| }d
}ndS d|	 d	| }d
}|                     d          }
|
rddlm}  |            }t          |
t                     r|
g}
|
D ]}d}t          |t                     rQ|                                                                dk    r't          |                     d          pd          }d
}n||                     d          k    rd
}|rt          d |D                       sRt          
                    d||                     d          |                     d          t          |                      	 ||z  }|                                st          |                    d          d d
          }|s9|d                             d                                          }d}t!          |          |k    r|d|         dz   }|r|r	d| d	| }nd| d| d	| }d
}n# t"          t$          f$ r'}t          
                    d||           Y d}~d}~ww xY wddlm} |                    t          |                     d          pd                    }|r| | }d
}d }||z   }||                     d!          }|r|gng }nt          |t                     r|g}d" |D             }|st-          || d||#          S dd$lm} dd%lm} dd&lm}m} dd'lm} g }g }|D ]}  ||                      d(                    }!|!r ||!dt          |                     d          pd          pd)          }"|"r3|"\  }#}$}%|r|!                    d           |!                    |#           t          
                    d*|                     d|                     d                    |            |!                    |            	 tE          j#         | ||                               }&nu# tD          j$        tJ          f$ r\ t          
                    d+|                     d|                     d                    |            |!                    |            Y |w xY w|&                    d,          su|&                    d-          pd.|  d/}'t          
                    d0|                     d|                     d                    |'           |!                    |            
	  || t          |                     d          pd          pd1           n-# tL          $ r  t          '                    d2| d
3           Y nw xY wt          |&                    d4          pd                                          }(|r|!                    d           |(                    d5|  d6d|(g           |rEd7d8)                    |           d9d8)                    |           d:})|*                    d|)           d}*|r'dd;l+m,}+ |!                    d            |+||          }*t-          d<)                    |          | d
=          },|*rFt!          |,          t!          |*          k    r&|,-                    |*          rdd>l.m/}-  |-|*           |,S )?u
  Build the effective prompt for a cron job, optionally loading one or more skills first.

    Args:
        job: The cron job dict.
        prerun_script: Optional ``(success, stdout)`` from a script that has
            already been executed by the caller (e.g. for a wake-gate check).
            When provided, the script is not re-executed and the cached
            result is used for prompt injection. When omitted, the script
            (if any) runs inline as before.
        extra_prompt: Optional per-run context (from ``cronjob(action='run')``,
            #57331 — salvaged from #57342 by @liuhao1024). Appended to the
            stored prompt under a ``## Run Context`` header for this single
            fire only — never persisted to the job definition.
    promptr#   z

## Run Context
skillsFscriptNzq## Script Output
The following data was collected by a pre-run script. Use it as context for your analysis.

```
z
```

TzQ## Script Error
The data-collection script failed. Report this to the user.

```
context_fromr   )get_cron_output_dirr   r?   c              3      K   | ]}|d v V  	dS )0123456789abcdefNrm   )r   cs     r    r   z$_build_job_prompt.<locals>.<genexpr>  s(      +[+[A1C,C+[+[+[+[+[+[rf   z@context_from: skipping invalid job_id %r for job_id=%r name=%r%sr>   z*.mdc                 4    |                                  j        S r   )statst_mtime)r  s    r    r!  z#_build_job_prompt.<locals>.<lambda>  s    !&&(("3 rf   )rl  reverser=  r>  i@  z

[... output truncated ...]z## Your previous run's output
The following is this job's most recent output from its previous run. Use it for continuity: avoid repeating what was already reported, and continue where the last run left off.

```
z## Output from job 'zo'
The following is the most recent output from a preceding cron job. Use it as context for your analysis.

```
z2context_from: failed to read output for job %r: %s)notepadu  [IMPORTANT: You are running as a scheduled cron job. DELIVERY: Your final response will be automatically delivered to the user — do NOT use send_message or try to deliver the output yourself. Just produce your report/output as your final response and the system handles the rest. SILENT: If there is genuinely nothing new to report, respond with exactly "[SILENT]" (nothing else) to suppress delivery. Never combine [SILENT] with content — either report your findings normally, or say [SILENT] and nothing more.]

skillc                     g | ]D}t          |                                          #t          |                                          ES rm   r  r   r>   s     r    r   z%_build_job_prompt.<locals>.<listcomp><  =    MMM3t99??;L;LM3t99??$$MMMrf   
has_skillshas_injected_datauser_prompt
skill_view)bump_use)build_bundle_invocation_messageresolve_bundle_command_key)normalize_skill_lookup_name/)user_instructiontask_idz>Cron job '%s': bundle '%s' could not load any skills, skippingz9Cron job '%s': skill '%s' returned invalid JSON, skippingr  rI   zFailed to load skill 'r  u/   Cron job '%s': skill not found, skipping — %s)r)  z-Cron job: failed to bump skill usage for '%s'r  r  z&[IMPORTANT: The user has invoked the "zf" skill, indicating they want you to follow its instructions. The full skill content is loaded below.]ze[IMPORTANT: The following skill(s) were listed for this job but could not be found and were skipped: , un   . Start your response with a brief notice so the user is aware, e.g.: '⚠️ Skill(s) not found and skipped: z'])append_user_instructionr@  )r  )register_stable_prefix)0rw   r@   r  r  r  rB   r%   rW   r  r   r   rn  r  r   globr  r\   rf  r  r8   r  render_notepad_section_scan_assembled_cron_prompttools.skills_toolr"  tools.skill_usager#  agent.skill_bundlesr$  r%  rv   r&  lstriprx   rI  r  r  r  r   rK  r  rj  insertagent.skill_commandsr+  rX   agent.prompt_cache_boundaryr,  ).r4   r	  r
  r   r  r  r  r  r  r  r  r  
output_dirsource_job_idis_selfjob_output_diroutput_fileslatest_output_MAX_CONTEXT_CHARSrO  cron_notepadnotepad_section	cron_hintr  skill_namesr"  r#  r$  r%  r&  r  skipped
skill_name
bundle_keybundle_payloadbundle_message_loaded_bundle_skills_missing_bundle_skillsloadedrI   r  noticestable_prefixr+  	assembledr,  s.                                                 r    _build_job_promptrM    s	   & cggh''-2..K I$HH,HHFWWXF  ''(##K %$%2"G]]%4[%A%A"G] 	%   *    	     %)!! t%     !% 77>**L Ag111111((**
lC(( 	*(>L) <	g <	gM
 G--- -2E2E2G2G2M2M2O2OSY2Y2Y #CGGDMM$7R 8 8#''$--//  +[+[]+[+[+[([([ V!GGDMMGGFOO/44   'g!+m!;%,,.. %"''//33      
 $  ,Q 9 97 9 K K Q Q S S%)"}%%(:::$12E3E2E$FIi$iM   (
 %2( (  &( ( (= ( ( %2( (  &	( (  )-%% & _- g g gSUbdeffffffffg -,,,,,"99#cggdmm>Qr:R:RSSO !#-V-- 
	D  F~!!#+&	FC	 	  MMMMMK 
*/#
 
 
 	
 -,,,,,******________======EG! 5
 5


 0/
0A0A#0F0FGG
 	<<!#CGGDMM/R008D  N
  P^M 57M %LL$$$^,,,NNP..  
 NN:&&&	Z

+F+Fz+R+R S STTFF$i0 	 	 	NNVX[X_X_`fhkhohopthuhuXvXv  yC  D  D  DNN:&&&H	 zz)$$ 	JJw''Q+QJ+Q+Q+QENNLcggV\^a^e^efj^k^kNlNlnstttNN:&&&	eHZSWWT]]-@b)A)A)ITJJJJJ 	e 	e 	eLLH*_cLddddd	e fjj++1r2288:: 	LL l  l  l  l	
 	
 	
 	
   N!%7!3!3N N 8<yy7I7IN N N 	 	QM ?@@@@@@R
 0/v>>+DIIe,<,<cdSSSI .Y#m*<*<<<AUAUVcAdAd< 	GFFFFF}---sJ   	I=$(I=A+I==J5J00J5#&R

A.S<;S<
1V<<'W&%W&r  rL  r  r  r   c                *   ddl m}m} |s|r" ||           \  }}|} |s|s|r ||          }n ||           }|rW|                    d          p|                    d          pd}	t                              d|	|           t          |          | S )u  Scan the fully-assembled cron prompt for injection patterns. Raises
    ``CronPromptInjectionBlocked`` when a match fires so ``run_job`` can
    surface a clear refusal to the operator.

    Plugs the #3968 gap: ``_scan_cron_prompt`` runs on the user-supplied
    prompt at create/update, but skill content is loaded from disk at
    runtime and was never scanned. Since cron runs non-interactively
    (auto-approves tool calls), a malicious skill carrying an injection
    payload bypassed every gate.

    Two pattern tiers, selected by what the assembled prompt CONTAINS,
    not just whether skills are attached:

    - When the assembled prompt is essentially the user prompt + the cron
      hint (no skills, no injected data), the STRICT ``_scan_cron_prompt``
      patterns apply: a bare ``rm -rf /`` in a small directive prompt is a
      smoking gun, not prose.
    - When the assembled prompt includes runtime-loaded content — skill
      markdown (``has_skills=True``) or DATA injected from a job script's
      stdout / an upstream job's output (``has_injected_data=True``) — the
      LOOSER ``_scan_cron_skill_assembled`` pattern set is used: only
      unambiguous prompt-injection directives block; command-shape
      patterns are dropped and invisible unicode is sanitized (stripped +
      logged) rather than blocked, to avoid false-positives that
      permanently kill a job. Skill bodies are vetted at install time by
      ``skills_guard.py``; script output is produced by operator-authored
      code, the same trust class — and data feeds (e.g. a triage bot
      ingesting bug reports) legitimately quote dangerous commands.

    When the looser tier is selected because of injected data only,
    ``user_prompt`` (the raw, pre-assembly prompt) is additionally scanned
    with the STRICT set so the user-authored surface keeps the full
    create/update-time guarantee at runtime (defense-in-depth for legacy
    jobs that predate the create-time scanner).
    r   )_scan_cron_prompt_scan_cron_skill_assembledr>   r?   z	<unknown>uC   Cron job '%s': assembled prompt blocked by injection scanner — %s)tools.cronjob_toolsrO  rP  r@   r   r   rh   )
rL  r4   r  r  r   rO  rP  rd   
scan_error	job_labels
             r    r/  r/    s    V RQQQQQQQ 2& 2 98CC	 	8* 	8 	8 +*;77J&&y11
 5GGFOOCswwt}}C	Q	
 	
 	

 )444rf   c                    	 ddl m}  ||                     d          |                     d                    }nB# t          $ r5}|                     d          rd|j        j         d| d}nd}Y d}~nd}~ww xY w|rF|                     d	          }t                              d
||           t          d| d|           dS )u4  Fail closed if a job's stored provider/base_url pair would exfiltrate a
    credential (F8 runtime backstop; CWE-200/CWE-522).

    The model-callable cron tool validates this on create/update, but a job
    persisted before that guard — or written directly to the jobs store —
    reaches the scheduler's provider-resolution sink unchecked. Re-validate the
    EFFECTIVE stored pair with the same guard the tool uses, so a named
    provider's stored key is never paired with an off-host base_url at fire
    time. Raises ``RuntimeError`` (caught by the run_job failure path → the run
    is aborted and reported) when the pair is unsafe; returns ``None`` otherwise.

    Fallback providers come from operator config, not the model-callable job, so
    they are trusted and validated by the caller, not here.
    r   )_validate_cron_base_urlproviderbase_urlz+could not validate provider/base_url pair (r  z=); refusing to run a job with an unverified base_url overrideNr?   ud   Job '%s': refusing to run — unsafe provider/base_url pair could exfiltrate a stored credential: %s
Cron job 'z' blocked for safety: )	rQ  rU  r@   r   	__class__ri   r   rI   rA  )r4   rU  r  r   ra   s        r    _guard_job_credential_exfilrZ    s   ??????%%cggj&9&9377:;N;NOO    77: 	2M*2 2.12 2 2 C C#$  M1C	
 	
 	

 KKKcKKLLLM Ms   8; 
A:+A55A:z[blocked_config]z[blocked_config:silent]z[drift_skip]z[drift_skip:silent]c                 8  	 t                      }| }|t          |          |vrr|                    t          |                     t          |          j        }t          |          j        pd}t          |                                          	|dv rdS d|v sd|v sd|v rt          	fdd	D                       rdS t          |t                    rd
dl}d
dlt          |j                  rfddD             }|j        |v rdS nAt          |dd          }||j        |j        |j        |j        |j        |j        |j        hv rdS t          	fddD                       rdS d	v sd	v rdS |j        p|j        }|t          |          |vrdS )u3  True when primary provider resolution failed for a transient network reason.

    Agent crons resolve OAuth credentials (token refresh / discovery) before the
    agent loop starts. A short DNS outage (Cloudflare WARP / macOS resolver blip)
    surfaces as httpx/httpcore ConnectError or raw OSError errno 8 ("nodename nor
    servname provided") and must be eligible for ``fallback_providers`` the same
    way AuthError already is — otherwise a healthy XAI_API_KEY / Anthropic rung
    never gets tried and the whole job dies before the first model call.
    Nr#   >   PoolTimeoutReadTimeoutConnectErrorNetworkErrorWriteTimeoutClientOSErrorConnectTimeoutTimeoutExceptionServerTimeoutErrorClientConnectorErrorClientConnectorDNSErrorThttpxhttpcoreaiohttpc              3       K   | ]}|v V  	d S r   rm   r   needler	  s     r    r   z7_is_transient_provider_resolve_error.<locals>.<genexpr>I  s;         #     rf   )nodename nor servnamename or service not known$temporary failure in name resolutionzfailed to resolvezconnection refusednetwork is unreachablerQ   rS   r   c                 P    h | ]"}t          |          t          |          #S rm   )r  r(   )r   _n_sockets     r    r  z7_is_transient_provider_resolve_error.<locals>.<setcomp>a  sC     " " "w++"GR((" " "rf   )
EAI_NONAME	EAI_AGAINEAI_FAIL
EAI_NODATAr  c              3       K   | ]}|v V  	d S r   rm   rk  s     r    r   z7_is_transient_provider_resolve_error.<locals>.<genexpr>t  s;         #     rf   )rm  rn  ro  rp  rm  rn  F)r   r?   r   r  ri   rj   rw   rW   r   rB   rf  r  socketgaierrorr(   ECONNREFUSED
ECONNRESETEHOSTUNREACHENETUNREACHENETDOWN	ETIMEDOUTr  	__cause____context__)
r   r  curr>   module_errno_eai_transienterr_nors  r	  s
           @@r    $_is_transient_provider_resolve_errorr  '  sR    UUD#&C
/bggT11CCyy!c%+#hhnn 
 
 
 4f
f 4 4	V8K8K    	      tc7## &	 #"""$$$$#w/00  " " " "S" " "
 9..4 / !gt44'%'&O$M    4         	 t #c))-HC-O-O4m.s] /bggT11^ 5rf   c                     | pi                      d          }t          |t                    sdS |                     dd          duS )zWhether cron pre-dispatch configuration validation is enabled.

    Default ON; only the literal boolean ``false`` under ``cron.preflight``
    opts out (mirrors ``cron_model_drift_guard_enabled`` semantics).
    r8   T	preflightFr  )r1   ry   s     r    _cron_preflight_enabledr    sI     	rv&&Hh%% t<<T**%77rf   c                    	 t          |          rdS n# t          $ r Y dS w xY wt          |                    d          t                    r|                    d          ni }|                     d          p9t          |pi                     d          pd                                          pd}|                     d          pt          j        d          pd}dd	l	m
} 	 dd
lm} ||d}|                     d          r|                     d          |d<    |di | n:# |$ r&}d| d|                     d           dcY d}~S d}~wt          $ r Y dS w xY wdS )u  READ-ONLY probe: would provider resolution fail for lack of a key?

    Mirrors the effective requested-provider computation from run_job's
    resolution block without any side effects on the run. When a fallback
    chain is configured the check is skipped entirely — the existing
    auth-fallback path may legitimately rescue a missing primary key, so
    blocking here would break that contract (and burning zero LLM calls is
    already guaranteed by the fallback resolution being config-local).
    Nr8   rV  model_providerr#   r   HERMES_MODELr   	AuthError)resolve_runtime_providerr  target_modelrW  explicit_base_urlzprovider credential missing: zh. Set the provider API key in .env (or `hermes setup`), or pin a working provider via `hermes cron edit r?   z --provider <p>`.rm   )r   r   rB   r@   rC   rw   r%   r  r  hermes_cli.authr  hermes_cli.runtime_providerr  )	r4   r1   	_cron_cfgr  r   r  r  kwargsr   s	            r    _preflight_check_provider_keyr    s   c"" 	4	   tt $.cggfoot#D#DL"I
 		R$$%566<"==CCEE	 
 GGG?	. 9 9?RE))))))HHHHHH(%@@77: 	>*-''**=*=F&'  **6**** 
 
 
0C 0 0 wwt}}0 0 0	
 	
 	
 	
 	
 	
     tt	
 4s1    
""%A D& &E+EEEEc                    t          |                     dd                    }g }|                    d          D ]p}|                                }|r|                                dv r/|                    |                    dd          d                                                    q|sdS d}|D ]}t          |          sd	| d
c S |p	 ddlm}  |            }d |	                                D             }|t          |          z  }n.# t          $ r! t                              dd           Y  dS w xY w|                                |vrd	| dc S dS )un  Check the job's delivery target(s) resolve to configured platforms.

    ``local``/``origin`` (and the ``all`` routing token) need no gateway
    credentials and are never checked — a deliver=local job must not pay a
    gateway-config load. For concrete platform targets, an unknown platform
    always blocks; a known platform additionally blocks when the gateway
    config is loadable and reports it unconnected (enabled + credentials —
    the same source `cron_delivery_targets` uses). Gateway-config load
    failures fail OPEN so a transient config hiccup never wedges delivery
    that would have worked.
    r  r  r  >   r  r  r  r  r=   r   Nzdelivery platform 'zu' is not a known cron delivery target. Fix the job's `deliver` value or configure the platform's gateway credentials.r  c                     h | ]	}|j         
S rm   r  r  s     r    r  z,_preflight_check_delivery.<locals>.<setcomp>  s'        !AG  rf   uL   preflight: gateway config unavailable — skipping delivery credential checkTr  z~' has no gateway credentials configured (not connected). Configure it via `hermes setup` or change the job's `deliver` target.)r  r@   r  r%   rW   rx   rz  rR  r  r  r  r   r   rK  )r4   r  platform_partsr  r  r"  r  r  s           r    _preflight_check_deliveryr    s    -SWWY-H-HIIM "N##C(( = =zz|| 	tzz||'AAAdjja00399;;<<<< t#I'  *=99 	6m 6 6 6  
 >>>>>>!4!4!6!6 %3%K%K%M%M  	 >yIII		   0:>     ttt   	11Gm G G G   2 4s   A D&D<;D<c                    |                      d          }||                      d          }|r|gng }nt          |t                    r|g}d |D             }|sdS ddlm} |D ]}	 t          j         ||                    }n# t          $ r Y .w xY wt          |t                    r|                     d          s]|                     d          s|                     d	          dk    rd
 |                     d          pg D             }|d |                     d          pg D             z  }|d |                     d          pg D             z  }d	                    |          pd}d| d| dc S dS )u  Check attached skills report ready (no missing required env/commands).

    Consults the same ``readiness_status`` payload ``skill_view`` computes
    for interactive use. Skills that fail to load at all are left to the
    existing skipped-skill handling in ``_build_job_prompt`` (fail-open):
    this check only blocks on an affirmative "setup needed" verdict, i.e.
    the skill exists but its required environment is missing — a run that
    is guaranteed to misfire.
    r  Nr  c                     g | ]D}t          |                                          #t          |                                          ES rm   r  r  s     r    r   z+_preflight_check_skills.<locals>.<listcomp>  r  rf   r   r!  r  setup_neededreadiness_statusc                     g | ]}d | S )zenv $rm   r  s     r    r   z+_preflight_check_skills.<locals>.<listcomp>  s.          rf   &missing_required_environment_variablesc                     g | ]}d | d	S )z	command 'r  rm   r  s     r    r   z+_preflight_check_skills.<locals>.<listcomp>  s3        $D###  rf   missing_required_commandsc                     g | ]}d | S )zcredential file rm   r  s     r    r   z+_preflight_check_skills.<locals>.<listcomp>#  s0        *4))  rf   missing_credential_filesr*  zrequired setup incompletezattached skill 'z' is not ready: missing zF. Provide the missing prerequisites or detach the skill from this job.)
r@   rB   rw   r0  r"  rI  r  r   rC   rj  )	r4   r  r  rA  r"  rC  payloadrn  details	            r    _preflight_check_skillsr    s.    WWXF~!!#+&	FC	 	  MMMMMK t,,,,,,!  
	jJ!7!788GG 	 	 	H	'4(( 	I0F0F 	KK''	{{-...@@ #KK<  	  G   #KK(CDDJ   G   #KK(BCCIr   G YYw''F+FF': ' '' ' '  # A, 4s   )B
BBc                      d fdfd fdfd fdffD ]D\  }}	  |            }n-# t           $ r  t                              d|d	           Y :w xY w|r|c S Ed
S )u  Pre-dispatch configuration validation (T1-26).

    Returns a human-readable reason when the job's configuration cannot
    produce a successful run — missing provider API key, unconfigured
    delivery platform, or an attached skill with missing required env —
    so the caller can refuse the run BEFORE any agent machinery is
    constructed and no LLM call is burned. Returns ``None`` when the
    configuration validates (or when a check cannot be evaluated: every
    check fails open, so preflight can only ever block on an affirmative
    misconfiguration verdict).

    Same fail-before-spend spirit as the #44585 drift guard and the
    fail-loud-on-hidden-tools direction in #27948; alert dedup follows the
    alert-once pattern from the dead-pin auto-pause (#73506).
    provider_keyc                  $    t                     S r   )r  )r1   r4   s   r    r!  z'_preflight_job_config.<locals>.<lambda>A  s    !>sC!H!H rf   r  c                  "    t                     S r   )r  r4   s   r    r!  z'_preflight_job_config.<locals>.<lambda>B  s    2377 rf   deliveryc                  "    t                     S r   )r  r  s   r    r!  z'_preflight_job_config.<locals>.<lambda>C  s    6s;; rf   u*   preflight check %s raised — failing openTr  N)r   r   rK  )r4   r1   r>   r  rc   s   ``   r    _preflight_job_configr  0  s    " 
HHHHHI	77778	;;;;<  e
	UWWFF 	 	 	LL<dT     H		
  	MMM	4s   
)'AAc                  L   d} 	 ddl m}  |            pi }t          |t                    r|                    di           ni }|                    d          }|t          |          }|dk    r|S n2# t          $ r%}t                              d|           Y d}~nd}~ww xY w| S )z6Return the wall-clock bound for cron post-run cleanup.      $@r   r   r8   cleanup_timeout_secondsNz3Failed to load cron cleanup timeout from config: %s)	hermes_cli.configr   rB   rC   r@   r  r   r   rK  )defaultr   r1   ry   r}  rS   r   s          r    _cron_cleanup_timeout_secondsr  Q  s    GQ111111kmm!r*4S$*?*?G37762&&&R\\";<<
!J''G!|| Q Q QJCPPPPPPPPQNs   A,A2 2
B!<BB!)timeout_secondslabelr  c                4   	 |t                      nt          |          }|dk    rI	                dS # t          t          f$ r(}t                              d|||           Y d}~dS d}~ww xY wt          j                    g 	d 	fd}t          j        |d| d	          }|	                                 
                    |          st                              d
|||           dS 	r%t                              d||	d                    dS dS )zDRun fallible post-run cleanup without permanently wedging a cron ID.Nr   TzJob '%s': %s failed: %sFr   c                      	               n,# t           $ r}                     |            Y d } ~ nd } ~ ww xY w                                 d S #                                  w xY wr   )BaseExceptionrx   r   )r   cleanupr\  rI   s    r    _runnerz/_run_cron_cleanup_with_timeout.<locals>._runner{  sz    	GIIII 	 	 	LL	 HHJJJJJDHHJJJJs%   
 A 
72A 7A A&zcron-cleanup-)r  r>   r  zRJob '%s': %s exceeded %.1fs; abandoning cleanup so future runs remain dispatchabler   )r  r  r   KeyboardInterruptr   rK  r  r  r  r  r  rI   )
r  ra   r  r  rS   r   r  workerr\  rI   s
   `       @@r    _run_cron_cleanup_with_timeoutr  c  sw    " 	&'''?## 
 !||	GIII4,- 	 	 	LL2FE3GGG55555	 ?D!#E        %V%%  F
 LLNNN99W `		
 	
 	
 u .uQxHHHu4s   
6 A/A**A/c                   $    e Zd ZdZdefdZd ZdS )_BoundedCronSessionDBa!  Proxy SessionDB cleanup calls through the cron cleanup timeout.

    After the first failed or timed-out operation the proxy fails subsequent
    calls immediately. A damaged SQLite connection should leak at most one
    abandoned cleanup worker, not one worker per finalization step.
    ra   c                 0    || _         || _        d| _        d S r  )_session_db_job_id	_disabled)r   r)   ra   s      r    r   z_BoundedCronSessionDB.__init__  s    %rf   c                 f     t           j                  t                    sS  fd}|S )Nc                      j         rt          d          i  fd}t          |j        d d          }|s3                    d          }||d_         t          d d	                              d
          S )Nz9session finalization disabled after prior cleanup failurec                  P    	  i d<   d S # t           $ r} | d<    d } ~ ww xY w)Nrm  rI   )r  )r   r  r  r   r  s    r    _callzB_BoundedCronSessionDB.__getattr__.<locals>._bounded.<locals>._call  sO    &,fd&=f&=&=F7OOO$   &)F7Os    
% %zsession finalization (r.  )ra   r  rI   Tzsession finalization method z
 timed outrm  )r  rA  r  r  r@   r  )	r  r  r  r4  rI   r   r>   r   r  s	   ``   @r    _boundedz3_BoundedCronSessionDB.__getattr__.<locals>._bounded  s    ~ `"#^___F        0|6t666  B
  T

7++$K "&"#R$#R#R#RSSS::g&&&rf   )r(   r  r;  )r   r>   r  r  s   `` @r    __getattr__z!_BoundedCronSessionDB.__getattr__  sV    )400 	M	' 	' 	' 	' 	' 	' 	': rf   N)ri   rj   rk   rl   rw   r   r  rm   rf   r    r  r    sH         3    
" " " " "rf   r  defer_agent_teardownr
  r  r  c                2]   | d         t          |                     d          p|                     d          ppd          |                     d          r$	 ddlm}  |t	                                 n-# t
          $ r  t                              d	d
           Y nw xY w|                     d          }|s$d}t                              d|           ddd|fS |                     d          pd	                                pd}|r?t          |                                          st                              d|           d}	 t          | ||          \  }}	n9# t
          $ r,}
t                              d           dd|
 }	}Y d}
~
nd}
~
ww xY wt                                          d          }|s d d|	 d| }d d d| d|	 d	}d|||	fS t#          |	          s2t                              d            d d d| d!}d
|t&          dfS |		                                s2t                              d"           d d d| d#}d
|t&          dfS d d d| d$|	 d	}d
||	dfS dd%lm}m} d} ||           r ||           }t                                          d          }|j        sPt                              d&|j                   d d d| d'|j         d	}d( d)|j         d| }d|||j        fS |j        s2t                              d*           d d d| d+}d
|t&          dfS |j        }|r|r| d,| n|}dd-lm} d}	 dd.lm} d}t=          j        d/d          	                                }|rC	 tA          |          }n2# tB          tD          f$ r t                              d0|           Y nw xY w|	 dd1l#m$}  |            pi }tK          |tL                    r|                    d2i           ni }|                    d3          }|tA          |          }n2# t
          $ r%}
t                              d4|
           Y d}
~
nd}
~
ww xY w|d5}|dk    rtN          j(        )                    d67          }|*                    |          } 	 | +                    |8          }n3# tN          j(        j,        $ r | -                    t\                      w xY w	 |/                    d9           n%# |/                    d9           w xY w |            }n# tN          j(        j,        $ r3 t                              d:|                     dd;          |           Y nFt
          $ r:}!t                              d<|                     dd;          |!           Y d}!~!nd}!~!ww xY wd}"|                     d          }|rzt          | |=          }"|"\  }#}$|#rat#          |$          sRt                              d>           d d dt                                          d           d?}d
|t&          dfS 	 ta          | |"|@          }%nu# tb          $ rh}&t                              dA|&           d d dt                                          d           dB|& dC	}'d|'dt          |&          fcY d}&~&S d}&~&ww xY w|%&t                              dD           d
dt&          dfS dE dFt                                          dG           }(t                              dH           t                              dI|%ddJ                    dddKl2m3})m4}*m5}+ |                     d          pd	                                pd}|r?t          |                                          st                              d|           d} |)dddd|pdL          },dM}-|-D ]}.|+|.         6                    d           t<          j7                            dNdO          }/|du}0tq                      }1d
}2|0rtr          :                    |18          sd}2ntr          ;                    |18          sd}2|+dP         }3d}4d}5	 |2stY          dQ|0rdRndS dT|1dUdV          |36                    dW          }4ty                      }5|r+|t<          j7        dN<   t                              dX|           ddYlm}m=}6  |6              |t	                                 t}          |           }7|7r|+dZ         6                    |7d[                    |+d\         6                    t          |7d]                              |+d^         6                    |7                    d_          dnt          |7d_                              |                     d`          pt=          j        da          pd}8d}9d}:i }i };	 ddbl#m?}< t          t	                      dcz            }=t<          j@        A                    |=          r3 |<t          |=                    }	 dddlBmC}> |>D                    |          }n# t
          $ r Y nw xY wt          |          }|                    d`          pi };|                    d2          pi }?tK          |?tL                    rlt          |?                    d`          pd          	                                }9t          |?                    de          pd          	                                }:|                     d`          s|9r|9}8nt          |          \  }@}A|Ar|A}8n3# t
          $ r&}!t                              df|!           Y d}!~!nd}!~!ww xY wtK          |8t                     r|8	                                sBt          dg dh|                     d`          dit=          j        dad          dj dk	          	 ddllHmI}B |                    dmi           }CtK          |CtL                    r!|C                    dn          r |Bd
o           n# t
          $ r Y nw xY wd}DtK          |                    dpi           tL                    r|                    dpi           ni }Et=          j        dqd          p+|                    drd          p|E                    drd          }F|Frt          |F          J                                }G|GK                                st	                      |Gz  }G|GA                                r	 t          |Gdsdtu          5 }Ht          jN        |H          }Dddd           n# 1 swxY w Y   tK          |Dt                    sd}Dn6# t
          $ r)}!t                              dv|G|!           d}DY d}!~!nd}!~!ww xY wddwl#mP}I |                    dpi                               dx          }J|J|                    dx          }J |I|J          }K|                    dy          pi }LddzlQmR}MmS}N dd{lTmU}O t          |            d}P	 t          |          rJt          | |          }P|Ps8|                     d|          r#	 dd}lYmZ}Q  |Q           n# t
          $ r Y nw xY wn/# t
          $ r" t                              d~d
           d}PY nw xY w|Prt                              d|P           d}R	 ddlYm[}S  |S          }Rn-# t
          $ r  t                              dd
           Y nw xY w|Rrt          nt          }Td d dt                                          d           d|P d	}'d|'d|T d|P f|r8|2r6|/dOk    r!t<          j7        ^                    dNd           n|/t<          j7        dN<   |2r5|0rtr          _                                 ntr          `                                  |*|,           |4|3a                    |4           |5t          |5           |-D ]}.|+|.         6                    d           |r`t          |          }|(}U	 |d                    |(          }V|Vr|V}Unh# t
          t          f$ rT}!	 t          dd          }W|Wr|W}Un# t
          t          f$ r Y nw xY wt                              d|!           Y d}!~!nd}!~!ww xY w	 dg                    h                                          dd         	                                pd }X|X dt                                          d           }Yt          ||U|Y          st          ||Ud            n# t
          t          f$ r}!t                              d|!            t          |dd           d           d d|Udd          fD ]-}Z	 t          ||U|Z          r n# t
          t          f$ r Y *w xY wY d}!~!nd}!~!ww xY w	 |j                    |Ud           n:# t
          t          f$ r&}!t                              d|!           Y d}!~!nd}!~!ww xY w	 |k                                 n:# t
          t          f$ r&}!t                              d|!           Y d}!~!nd}!~!ww xY w||l                               S S t                     S |8}[tK          |;tL                    rHt          |;                    d          pd          	                                n                                nd}\t          |                     d          pd          	                                n                                p|\pd}]	 |                     d          p|:pd|8d}^|                     d          r|                     d          |^d<    |Md	i |^}_t          |_                    d          pd          	                                n                                p|]}]n%# t
          $ r}`tK          |`|O          }at          |`          }b|as|bst           |N|`                    |`t          t          |`dd          pd          	                                n                                p|]}]|ardnd}ct                              d|c|`           t          |          }dd}_|dD ]8}etK          |etL                    st          |e                    d          pd          	                                }ft          |e                    d`          pd          	                                }g|fr|gs	 ddlqmr}h |f|gd}i|e                    d          r|ed         |id<    |h|e          }j|jr|j|id<    |Md	i |i}_|g}8t                              d|_                    d          |g            n5# t
          $ r(}kt                              d|f|k           Y d}k~k2d}k~kww xY w|_t           |N|`                    |`Y d}`~`nd}`~`ww xY wt          | tK          |tL                    r|ni t          |8                    }lt          |          r6g }mt          |]p|_                    d          pd          	                                n                                }nt          |[pd          	                                n                                }ot          | |n|o|          D ]u}pt          |                     |p d          pd          	                                n                                }q|pdk    r|nn|o}r|ml                    |p d|q d|r d           v|mr)dg                    |m          }stK          |                     d          tL                    r|                     d          ni }ttK          |                     d          tL                    o7| d                             d          dk    o|t                    d          d6k    }u|urd}vnd d}vt                              d|s|v           d}w	 ddlYmv}x  |x          }wn# t
          $ r Y nw xY w|wrt          nt          }yt          |y d|s d|v d          t          |          pd}zd}{t          |_                    d          pd          	                                n                                }|||r	 ddlymz}}  |}||          }~|~{                                r>|~}{t                              d||t          |~}                                                     n4# t
          $ r'}!t                              d|||!           Y d}!~!nd}!~!ww xY w	 ddl~m}  |            }|r)t                              dt          |                     n3# t
          $ r&}t                              d|           Y d}~nd}~ww xY w |d	i d`|8d|_                    d          d|_                    d          d|_                    d          d|_                    d          d|_                    d          d|_                    d          d|_                    d          d|Kd|ld|Dd|zd|{d|L                    dæ          d|L                    dŦ          d|L                    dǦ          d|L                    dɦ          d|                    d˦          pi                     d̦          d͐t          | |          dΐt          |          dd
dАt          |           dd
dddd
d[d2d|(d|t                      }|dk    r|nd}d}|                     d          }tK          |tL                    o|                    d          dk    |                     d֦          }tK          |tL                    r$t          |                    dצ          pd          ndt	          j                    d
fdل}fdڄ}tN          j(        )                    d67          }t          j                    }t          j                    j        }t	          j                    }|*                    |j        j        |%          }d}	 |xs_d}	 tN          j(                            |h|8          \  }}@|r |             |+                                }n |              |             [n|+                                }nd}	 tN          j(                            |h|8          \  }}@|r |             |+                                }nm |              |             d}t          dܦ          r<	                                 }|                    ddۦ          }n# t
          $ r Y nw xY w||k    rd
}nn&# t
          $ r |/                    dd
ަ            w xY w|/                    dd
ަ           n# |/                    dd
ަ           w xY w|ri }t          dܦ          r&	                                 }n# t
          $ r Y nw xY w|                    dd          }|                    dd          }|                    d          }|                    dd          }|                    dd          }t                              d㉫||||||pd           t!          d           tY          dg dt#          |           dt#          |           d|           tK          |tL                    s(t          dt%          |          j         d|          t          |                    d          pd          }|                    d          pd	                                }|                    d          d
uo;|                    d          du o$|                    d          ot          |          }|                    d          d
u s|                    d          du r*|s(|                    d          p|pd}t          |          |rt                              d򉫦           |                    dd          pd}|	                                dk    rd}|	                                r|rg }	 ddlm} n# t
          $ r d}Y nw xY wdg|R D ]}	 |                    ||          }nF# tD          $ r, 	 |                    |          }n# t
          $ r d}Y nw xY wY nt
          $ r d}Y nw xY w|r'|l                    |	                                           |	                                |v rt                              d|           d}|r|nd}d d dt                                          d           d|                     dd           d|% d| d}	t                              d           t#          t	          j                    |z
  dz            }t/          |pd          }t1          t3                      ||                    d          |                    d          |                    d           ||                     d          |8pd|dd           d
|	|df|r8|2r6|/dOk    r!t<          j7        ^                    dNd           n|/t<          j7        dN<   |2r5|0rtr          _                                 ntr          `                                  |*|,           |4|3a                    |4           |5t          |5           |-D ]}.|+|.         6                    d           |r`t          |          }|(}U	 |d                    |(          }V|Vr|V}Unh# t
          t          f$ rT}!	 t          dd          }W|Wr|W}Un# t
          t          f$ r Y nw xY wt                              d|!           Y d}!~!nd}!~!ww xY w	 dg                    h                                          dd         	                                pd }X|X dt                                          d           }Yt          ||U|Y          st          ||Ud            n# t
          t          f$ r}!t                              d|!            t          |dd           d           d d|Udd          fD ]-}Z	 t          ||U|Z          r n# t
          t          f$ r Y *w xY wY d}!~!nd}!~!ww xY w	 |j                    |Ud           n:# t
          t          f$ r&}!t                              d|!           Y d}!~!nd}!~!ww xY w	 |k                                 n:# t
          t          f$ r&}!t                              d|!           Y d}!~!nd}!~!ww xY w||l                               S S t                     S # t
          $ rl}!t%          |!          j         dt          |!           }t                              d|           dt5                      v rht#          t	          j                    |z
  dz            }t1          t3                      |dddd|                     d          |8pd||d           d d dt                                          d           d|                     dd           d|% d| d}	d|	d|fcY d}!~!|r8|2r6|/dOk    r!t<          j7        ^                    dNd           n|/t<          j7        dN<   |2r5|0rtr          _                                 ntr          `                                  |*|,           |4|3a                    |4           |5t          |5           |-D ]}.|+|.         6                    d           |r`t          |          }|(}U	 |d                    |(          }V|Vr|V}Unh# t
          t          f$ rT}!	 t          dd          }W|Wr|W}Un# t
          t          f$ r Y nw xY wt                              d|!           Y d}!~!nd}!~!ww xY w	 dg                    h                                          dd         	                                pd }X|X dt                                          d           }Yt          ||U|Y          st          ||Ud            n# t
          t          f$ r}!t                              d|!            t          |dd           d           d d|Udd          fD ]-}Z	 t          ||U|Z          r n# t
          t          f$ r Y *w xY wY d}!~!nd}!~!ww xY w	 |j                    |Ud           n:# t
          t          f$ r&}!t                              d|!           Y d}!~!nd}!~!ww xY w	 |k                                 n:# t
          t          f$ r&}!t                              d|!           Y d}!~!nd}!~!ww xY w||l                               S S t                     S d}!~!ww xY w# |r8|2r6|/dOk    r!t<          j7        ^                    dNd           n|/t<          j7        dN<   |2r5|0rtr          _                                 ntr          `                                  |*|,           |4|3a                    |4           |5t          |5           |-D ]}.|+|.         6                    d           |r`t          |          }|(}U	 |d                    |(          }V|Vr|V}Unh# t
          t          f$ rT}!	 t          dd          }W|Wr|W}Un# t
          t          f$ r Y nw xY wt                              d|!           Y d}!~!nd}!~!ww xY w	 dg                    h                                          dd         	                                pd }X|X dt                                          d           }Yt          ||U|Y          st          ||Ud            n# t
          t          f$ r}!t                              d|!            t          |dd           d           d d|Udd          fD ]-}Z	 t          ||U|Z          r n# t
          t          f$ r Y *w xY wY d}!~!nd}!~!ww xY w	 |j                    |Ud           n:# t
          t          f$ r&}!t                              d|!           Y d}!~!nd}!~!ww xY w	 |k                                 n:# t
          t          f$ r&}!t                              d|!           Y d}!~!nd}!~!ww xY w||l                               w w t                     w xY w(  u  
    Execute a single cron job.

    ``defer_agent_teardown``: when a caller passes a list, ``run_job`` skips
    the agent's async-resource teardown (``agent.close()`` +
    ``cleanup_stale_async_clients()``) in its ``finally`` block and instead
    appends the live agent to that list. The caller is then responsible for
    calling ``_teardown_cron_agent(agent)`` AFTER it has delivered the result.
    This closes the ordering window in #58720 where delivery ran against a
    torn-down async client (defense-in-depth alongside the interpreter-shutdown
    guard). When ``None`` (the default) teardown happens inline as before, so
    every existing caller is unchanged.

    ``extra_prompt``: optional per-run context from ``cronjob(action='run',
    prompt=...)`` (#57331). Appended to the stored prompt for this fire only —
    never persisted to the job definition.

    Returns:
        Tuple of (success, full_output_doc, final_response, error_message)
    r?   r>   r  rK   rM   r   )load_hermes_dotenv)r  z%Job '%s': no_agent .env reload failedTr  r  z/no_agent=True but no script is set for this jobr  Fr#   r  NuG   Job '%s': configured workdir %r no longer exists — running without itr  z.Job '%s': script execution raised unexpectedlyr  z%Y-%m-%d %H:%M:%Su   ⚠ Cron watchdog 'z' script failed

z

Time: z# Cron Job: z

**Job ID:** z
**Run Time:** z8
**Mode:** no_agent (script)
**Status:** script failed

r@  u8   Job '%s' (no_agent): wakeAgent=false gate — silent runzB
**Mode:** no_agent (script)
**Status:** silent (wakeAgent=false)
u0   Job '%s' (no_agent): empty stdout — silent runz?
**Mode:** no_agent (script)
**Status:** silent (empty output)
z#
**Mode:** no_agent (script)

---

)check_monitorjob_has_monitorz#Job '%s': monitor source failed: %sz6
**Mode:** monitor
**Status:** monitor source failed

u   ⚠ Cron monitor 'z' source failed

u<   Job '%s': monitor output unchanged — suppressing agent runz@
**Mode:** monitor
**Status:** no_change (agent run suppressed)
z

)AIAgent)	SessionDBHERMES_CRON_SESSION_DB_TIMEOUTz?Invalid HERMES_CRON_SESSION_DB_TIMEOUT=%r; using config/defaultr  r8   session_db_timeout_secondsz>Failed to load cron.session_db_timeout_seconds from config: %sr  r=   r  r  r4  u   Job '%s': SessionDB init did not return within %.0fs — proceeding without a session store for this run instead of blocking it foreverr   z0Job '%s': SQLite session store not available: %s)r  z6Job '%s' (ID: %s): wakeAgent=false, skipping agent runu<   

Script gate returned `wakeAgent=false` — agent skipped.
)r	  r
  u=   Job '%s' (ID: %s): blocked by prompt-injection scanner — %sz
**Status:** BLOCKED

The assembled prompt (user prompt + loaded skill content) tripped the cron injection scanner and the agent was NOT run.

**Scanner result:** a  

Audit the skill(s) attached to this job for prompt-injection payloads or invisible-unicode markers. If the skill is legitimate and the match is a false positive, rephrase the content to avoid the threat pattern (`tools/cronjob_tools.py::_CRON_THREAT_PATTERNS`).z6Job '%s': script produced no output, skipping AI call.cron_r  z%Y%m%d_%H%M%SzRunning job '%s' (ID: %s)z
Prompt: %sd   )set_session_varsclear_session_vars_VAR_MAP)r  r  rE  async_deliveryr  )!HERMES_CRON_AUTO_DELIVER_PLATFORM HERMES_CRON_AUTO_DELIVER_CHAT_ID"HERMES_CRON_AUTO_DELIVER_THREAD_IDTERMINAL_CWD_UNSET_HERMES_CRON_SESSIONz'Timed out waiting for the TERMINAL_CWD rH  readz lock after z.0fu   s — another cron job (a workdir writer, or long-running readers) has held it for longer than the cron inactivity limit. If a workdir job is the holder, stagger its schedule or remove its workdir to unblock this job (#79768).1zJob '%s': using workdir %s)r  reset_secret_source_cacher  r  r  r  r  r#  r   r  )read_user_config_rawzconfig.yaml)managed_scoper  z8Job '%s': failed to load config.yaml, using defaults: %srX  z%' has no model configured (job.model=z, HERMES_MODEL=zY, config.yaml model.default missing or empty). Set a per-job model via `hermes cron edit z= --model <name>` or set a default with `hermes model <name>`.)apply_ipv4_preferencenetwork
force_ipv4)forcers   HERMES_PREFILL_MESSAGES_FILEprefill_messages_filerr=  r>  z8Job '%s': failed to parse prefill messages file '%s': %s)resolve_turn_limit	max_turnsprovider_routing)r  format_runtime_provider_errorr  preflight_alerted)clear_preflight_alertedu7   Job '%s': preflight validation errored — failing openuZ   Job '%s' (ID: %s): BLOCKED by pre-dispatch config validation — %s (no LLM call was made))mark_preflight_alertedz2Job '%s': could not persist preflight alert markerz
**Status:** BLOCKED (configuration)

Pre-dispatch validation found a configuration problem and the agent was NOT run (no tokens spent).

**Reason:** z

The job will stay blocked (without re-alerting) until the configuration is fixed; the next healthy run clears this state. Set `cron.preflight: false` in config.yaml to disable this validation.rT   r*   z4Job '%s': failed to resolve cron compression tip: %srr  zcron u    · z%b %d %H:%Mz.Job '%s': failed to set cron session title: %sr$   c                     | S r   rm   )bs    r    r!  zrun_job.<locals>.<lambda>;  s    PQ rf   icron_completez#Job '%s': failed to end session: %sz2Job '%s': failed to close SQLite session store: %srV  r  rW  r  authztransient networkzCJob '%s': primary provider resolve failed (%s: %s), trying fallback)resolve_entry_api_keyexplicit_api_keyz*Job '%s': fallback resolved to %s model %sz Job '%s': fallback %s failed: %s)current_providercurrent_modelr  	_snapshotz 'z' -> 'r  r6  rp  r6   r7   r  rq  zThis finite one-shot job is consumed by this attempted run; create a new one-shot job at a future time with an explicit provider and model.zZTo run on the new config, on the host running Hermes pin it explicitly: `hermes cron edit zR --provider <provider> --model <model>` (or pin the original values to keep them).u   Job '%s': SKIPPED — global inference config drifted since creation (%s) and this job is unpinned. Skipped to prevent unintended spend. %s)mark_drift_alertedzb Skipped to prevent unintended spend: global inference config drifted since this job was created (z9), and this job is unpinned. No inference call was made. zc This alert is sent once; the job stays skipped until the config is pinned or restored. See #44585.)	load_poolz@Job '%s': loaded credential pool for provider %s with %d entriesz3Job '%s': failed to load credential pool for %s: %s)discover_mcp_toolsz"Job '%s': %d MCP tool(s) availablez3Job '%s': MCP initialization failed (non-fatal): %sapi_keyrequested_providerapi_modeacp_commandcommandacp_argsr  max_iterationsreasoning_configprefill_messagesfallback_modelcredential_poolproviders_allowedonlyproviders_ignoredignoreproviders_orderorderprovider_sortsortopenrouter_min_coding_score
openroutermin_coding_scorer   ru   
quiet_modeskip_context_filesload_soul_identityskip_memoryskip_background_reviewr)   r  r  r  r   c                                                       sd S  %t           d          r                     d           t          d d          )N	interruptz"Cron fire claim ownership was lostrX  z'' lost its durable fire claim ownership)r   r  r!  rA  )rs   r  r^   s   r    _abort_if_fire_claim_lostz*run_job.<locals>._abort_if_fire_claim_lost  sg    #<+>+>+@+@# WUK%@%@  DEEENXNNN  rf   c                      rsd S t          j                    } | z
  t          k     rd S | 	 t                     d S # t          $ r! t
                              dd           Y d S w xY w)Nr  z$Job '%s': run_claim heartbeat failedTr  )r   r  r  r   r   r   rK  )_mono_is_oneshot_last_claim_heartbeat_run_claim_ownerra   r^   s    r    _heartbeat_run_claim_if_duez,run_job.<locals>._heartbeat_run_claim_if_due  s     &6 N$$E,,/KKK$)!#F;KLLLLLL   :Ht       s   A 'A,+A,g        get_activity_summaryseconds_since_activityr  last_activity_descr  current_toolapi_call_countz_Job '%s' idle for %.0fs (inactivity limit %.0fs) | last_activity=%s | iteration=%s/%s | tool=%snonezCron job timed out (inactivity)z' idle for z	s (limit u   s) — last activity: z agent.run_conversation returned z instead of dict: turn_exit_reasonfinal_responsefailed	completedzmax_iterations_reached(rI   zagent reported failurezJob '%s' reached the iteration limit but produced a final fallback response; delivering the response instead of failing the cron runz(No response generated))PERSISTENCE_ERROR_CAUSES)lockeddiskr  uN   Job '%s': abnormal empty turn (%s) — suppressing explainer for cron deliveryz
**Schedule:** schedule_displayzN/Az

## Prompt

z

## Response

zJob '%s' completed successfullyr  prompt_tokenscompletion_tokenstotal_tokensr  )tsra   fire_idr7  r8  r9  response_silentdeliver_targetr   duration_msrI   r  zJob '%s' failed: %s_audit_fire_idz (FAILED)

**Job ID:** z

## Error

```
z
```
rm   r   )rw   r@   hermes_cli.env_loaderr  rD  r   r   rK  rI   r%   r   r  r   r  	exceptionrA  r  r  r   SILENT_MARKERcron.monitorr  r  r4  changedcontext_block	run_agentr  hermes_stater  r  r  r  r'   r  r  r   rB   rC   r  r  r  rB  r   r  add_done_callbackr!   r  rM  rh   gateway.session_contextr  r  r  r   r  r  _terminal_cwd_lockr  r  r   r  r  r  rM  r  
hermes_clir  apply_managed_overlayr   r   rA  r   r  r  r  rG  rI  loadr   r  r  r  r  r  r  rZ  r  r  r  r  r  BLOCKED_CONFIG_SILENT_MARKERBLOCKED_CONFIG_MARKERr  r  r  resetr   r  get_compression_tipr  r(   rj  r  r/   end_sessionr   rx   _teardown_cron_agentrW   r  r   hermes_cli.fallback_configr  r   r   r   r  DRIFT_SKIP_SILENT_MARKERDRIFT_SKIP_MARKERagent.credential_poolr  has_credentialsr\   entriestools.mcp_toolr  r   r}   r   r  r   r  r  r  uuiduuid4hexr@  run_conversationr  r  r)  r   rA   r  ri   rX   r3  #_format_turn_completion_explanationr   r  r  locals)r4   r  r
  r  r  r  r  _job_workdirr4  outputr   now_isoalertdoc
silent_docr  r  _monitor_context_mon_mon_now_mon_doc
_mon_alertr  r  r  _session_db_timeout_raw_env_timeoutr   _cfgr  _configured_session_db_pool_session_db_futurerO  r	  _ran_ok_script_outputr  	block_excblocked_doc_cron_session_idr  r  r  _ctx_tokens_cron_delivery_vars	_var_name_prior_terminal_cwd_holds_cwd_write_cwd_lock_timeout_cwd_lock_acquired_cron_session_var_cron_session_token_non_dispatcher_tokenr  delivery_targetr   _cron_default_model_cron_default_provider
_model_cfgr  	_cfg_pathr  _cron_cfg_for_modelr  _global_modelr  _net_cfgr  r{   prefill_filepfpath_pf_resolve_turn_limit_mtr  prr  r  r  
_pf_reasonr  already_alertedr  marker_final_cron_session_id_compression_tip_agent_session_id_title_base_cron_title	_fallbackprimary_model_for_driftconfigured_provider_for_driftprimary_provider_for_driftruntime_kwargsruntimeresolve_excis_authis_transient_netrc   fb_listrL  fb_providerfb_modelr  	fb_kwargs
fb_api_keyfb_excr  _drift_current_provider_current_model_axisr  _current_changes_repeat_finite_oneshot_remediation_drift_already_alertedr  _drift_markerr  r  runtime_providerr  rs  r  
_mcp_tools_mcp_exc_cron_timeout_cron_inactivity_limit_POLL_INTERVAL_job_schedule
_run_claimr"  r(  
_cron_pool_cron_contextr?  _audit_t_start_cron_future_inactivity_timeoutr   r\  
_idle_secs_act	_activity
_last_desc	_secs_ago	_cur_tool_iter_n	_iter_maxr/  final_response_textmax_iteration_summary	_err_textr0  _explainer_variants_causes_cause_variantlogged_response_audit_duration_ms_audit_response_silent	error_msgr%  r&  r'  rs   ra   r^   s      `                                                                                                                                                                  @@@@@@r    run_jobr    s5?   6 YF3776??Ocggh&7&7O6OZPPH( wwz d'	@@@@@@+;+=+=>>>>> 	 	 	LL7$      	
 ggh'' 	&CCLL555"b#%% 	**0b7799AT 	 \ 2 2 9 9 ; ; 	 NNY    L	B=[,\  JB  	B 	B 	B@&    AC A ABBBBBB		B --(()<== 	-
#h # ## # # # x  % !( 
     #uf,,  '' 	9KKJF  :x : :%: :!(: : :  ]D88||~~ 		9KKJFSSS7x 7 7%7 7!(7 7 7  ]D888  ! $ 
    	 S&$&& <;;;;;;;&*s -}S!!==))*=>>w 	; LL>
SSS"x " "%" "!)" "
 :" " " $X $ $:$ $!$ $ 
 (J
::| 	7 KKN  
Bx B B%B B!)B B B  =$66  - 	;G]#77777M]  "!!!!! K<`****** -19%ErJJPPRR 	&+,<&=&=##	*   U$    
 &999999"{}}*4>tT4J4JRDHHVR000PR	'mm,HII**/*<*<'   T       
 &"&"")1DDQRDSS!1!8!8!C!C6077@S7TT%2    #445RSSS  !))u)5555 ))u)5555 $)++K* 
 
 
 GGD# 3		
 	
 	
 	
 	
  ` ` `GQUWZI[I[]^________` M''(##K 9<<
 
 
 #0 	9+N;; 	9KKH&  
Nx N N%N N!,!7!78K!L!LN N N  ]D886"}<
 
 
 & 6 6 6
 	Kfi	
 	
 	


T8 
T 
T!
T 
T(]]334GHH
T 
T $-
T 
T 
T 	 k2s9~~5555555-6. ~LhWWWR,,QvQQ(>(>(O(OQQ
KK+Xv>>>
KKfTcTl+++E WVVVVVVVVV6 GGI&&,"3355=L D..5577 UL	
 	
 	
 "" B%  K(
 ) $ $	####4 *..CC#4/133 '!//8I/JJ 	'!&!..7H.II 	'!& !!67 E0! 	 ..:77F. .$L. . .   033C88( !C D D 	L)5BJ~&KK4flKKK	
 	
 	
 	
 	
 	
 	
 	
 	"!###'7'9'9::::2377 	89==oj>YZZZ78<<SQZA[=\=\]]]9:>>"&&{33; 566     CBIn$=$=C !!# 
&	b>>>>>>,..>??Iw~~i(( !2++DOO<<
888888(>>tDDDD    D'-- "XXg..4"
&*hhv&6&6&<"#1488 *-+//88>B+ +egg ( .1+//0@AAGR. .egg + www'' 
2* 	2 !4 ,MT+R+R(=( 2$1E 	b 	b 	bNNUW]_`aaaaaaaa	b
 5#&& 		5;;== 		6X 6 6!ggg..6 6 "	." = =6 6
 &,6 6 6  	>>>>>>xx	2..H(D)) 2hll<.H.H 2%%D1111 	 	 	D	  -7"8M8Mt-T-T\DHHWb)))Z\	I4b99 :xx/44:}}4b99 	
  	,,''2244F%%'' 5)++f4}} ,,fcG<<< :+/9S>>(: : : : : : : : : : : : : : :%&6== 0+/(  , , ,NN#]_egmopqqq'+$$$$$$, 	POOOOOhhw##''44;((;''C,,S11 XX())/R	
 	
 	
 	
 	
 	
 	
 	
 	.----- 	$C(((" 
	&t,, 	23==
! cgg.A&B&B EEEEEE//7777$    	 	 	LLI     JJJ	   	DNN;&*  
 $O<<<<<<"8"8"@"@   HT       1@ +,,* 

+x 
+ 
+%
+ 
+!,!7!78K!L!L
+ 
+  *
+ 
+ 
+  +rf+C+Cz+C+CCJ  	A. 	A"i//
~t4444-@
>*  	2 2"002222"//111 	;'''*##$7888 ,-.CDDD, 	( 	(IY##B'''' H	^ 0VDDK &6"#.#B#B$$ $  $ >-=*01   (/|T(J(J%( C1B.!#45   DJ       $!!hhx~~'7'788"=CCEEYIYQWIYIY!,YY+--2H2H2W2WYY.!7   ,#%;=MV=M=M   01 ! ! !Dfa   SGK)DkkRR(((  CFBB%;BCC%@BB	" ! !I!2')?  " "E" &'89 ! ! ! !#!&O''*O    01 O O OBFANNNNNNNNO^!!####01 ^ ^ ^QSY[\]]]]]]]]^  + $++E2222 ! !////] #( *d++C
z**0b117799??AAA 	& 
##)r**002288:: , 	#
T	` !WWZ00R4JRd
 !&
 
N wwz"" J69ggj6I6I23..@@@@GGKK
++1r2288::@@BB .- '&  <	` <	` <	` !i88GCKPP `/ `"#@#@#M#MNNT__ GKR88>B??EEGGMMOO .- '  '?VV,?FNNU	   )..GG  b b!%.. !%))J"7"7"=2>>DDFFuyy117R88>>@@" ( bPPPPPP &1(0! !I yy,, K9>z9J	"56!6!6u!=!=J! C8B	"4566CCCCG$EKKDJ// 	   E  b b bLL!CV[Z`aaaaaaaab"#@#@#M#MNNT__ w<	`| 9D$//7RU
 
6 *$// G	 "F #*Kgkk*.E.EK! !eggeegg  !!8!>B??EEGGMMOON.!2,	   H H  5(;(;(; < < BCCIIKKQQSS	050C0C,,FF)FF8FFFGGGG 899V,,
 0:#''(:K:KT/R/RZ#''(+++XZswwz22D99 2J++F33v=2G,,1  
 # . !L!-3! ! ! ! +     */&<<<<<<-?-?-G-G**    D 1G +,,*  #$ A A A A ".A A A   ,D119Tw{{:66<"==CCEEKKMM 	qq;;;;;; y!122'')) &*OKKZ(DLLNN++	    q q qRTZ\lnoppppppppq	999999++--J 8C
OO    	 	 	NNE       	  %
 %
 %
%%
KK	***%
 [[,,,%
 [[,,,	%

  '{{+?@@@%
 [[,,,%
  I...%
 [[(((%
 *>%
 .-%
 .-%
 *>%
 ,O%
 !ffVnnn%
 !ffX...%
  FF7OOO!%
" &&...#%
$ *.,)?)?)E2(J(JK](^(^(^%%
& <CFFF'%
( >dCCC)%
* t+%
4 $(#5#5555%
6  $t7%
@ A%
B $(4C%
D VE%
F ('G%
H #{I%
^ 1222?!2C2C 
++}d++S0A0A&0I0IV0S 	 WW[))
/9*d/K/KSC
t$$*+++QS 	 !% 0 0	 	 	 	 	 	 	 		 	 	 	 	 	 	 	 	  '::q:II
 $022)))!(():E<RTZ[[#-	A%-  3,":!F	6","4"9"9)NN #: # #a   "55777%1%8%8%:%:F!1133333555	6 " *0022FF(055% 6  GD!  11333!-!4!4!6!6--/////111!$Ju&<== !!#(#=#=#?#?D)-2JC)P)PJJ( ! ! ! D!!%;;;.2+)*  	 	 	U4@@@	 U4@@@@JU4@@@@ 	Iu455  % : : < <II    D"';YGGJ!&>BBI!n55Imm$4a88G!&6::ILLA)%;GY#V   #5*KLLL3X 3 3y>>3 3,/0F,G,G3 3&03 3   &$'' 	f4<<3Hff\bff   vzz*<==CDD%zz*:;;ArHHJJJJx  , *

;''50* ++,EFF* ())	 	 ::h4''FJJ{,C,Cu,L,LUj,L

7## ,&,+ 
 y)))  	NNJ    $4b99?R!!%>>>N !! !	$&6 !	$
 #%8LLLLLLL 8 8 878*'** A A"&JJ(&   HH ! & & &&#*#N#N,$ $ % & & &#%&  " " "!HHH" A'..x~~/?/?@@@##%%)<<<d$  
 "$ -;Y..@Y(    }}%%&9::  ww)511	       	5x@@@ !$."2"2^"Ct!KLL!:>;OR!P!P ""%#ZZ88!',?!@!@"JJ~665!ggi00]d-
 
 	 	 	 V^T1`  	A. 	A"i//
~t4444-@
>*  	2 2"002222"//111 	;'''*##$7888 ,-.CDDD, 	( 	(IY##B'''' H	^ 0VDDK &6"#.#B#B$$ $  $ >-=*01   (/|T(J(J%( C1B.!#45   DJ       $!!hhx~~'7'788"=CCEEYIYQWIYIY!,YY+--2H2H2W2WYY.!7   ,#%;=MV=M=M   01 ! ! !Dfa   SGK)DkkRR(((  CFBB%;BCC%@BB	" ! !I!2')?  " "E" &'89 ! ! ! !#!&O''*O    01 O O OBFANNNNNNNNO^!!####01 ^ ^ ^QSY[\]]]]]]]]^  + $++E2222 ! !////s  &, &, &,Aww'333q6633	.)DDD vxx''!$dn&6&6&G4%O!P!P$&& )!%%) $#("%'')"4"4$1"      (    }}%%&9::  ww)511	        fb)++++++  	A. 	A"i//
~t4444-@
>*  	2 2"002222"//111 	;'''*##$7888 ,-.CDDD, 	( 	(IY##B'''' H	^ 0VDDK &6"#.#B#B$$ $  $ >-=*01   (/|T(J(J%( C1B.!#45   DJ       $!!hhx~~'7'788"=CCEEYIYQWIYIY!,YY+--2H2H2W2WYY.!7   ,#%;=MV=M=M   01 ! ! !Dfa   SGK)DkkRR(((  CFBB%;BCC%@BB	" ! !I!2')?  " "E" &'89 ! ! ! !#!&O''*O    01 O O OBFANNNNNNNNO^!!####01 ^ ^ ^QSY[\]]]]]]]]^  + $++E2222 ! !////s&,\  	A. 	A"i//
~t4444-@
>*  	2 2"002222"//111 	;'''*##$7888 ,-.CDDD, 	( 	(IY##B'''' H	^ 0VDDK &6"#.#B#B$$ $  $ >-=*01   (/|T(J(J%( C1B.!#45   DJ       $!!hhx~~'7'788"=CCEEYIYQWIYIY!,YY+--2H2H2W2WYY.!7   ,#%;=MV=M=M   01 ! ! !Dfa   SGK)DkkRR(((  CFBB%;BCC%@BB	" ! !I!2')?  " "E" &'89 ! ! ! !#!&O''*O    01 O O OBFANNNNNNNNO^!!####01 ^ ^ ^QSY[\]]]]]]]]^  + $++E2222 ! !////s  "B 'B+*B+E- -
F#7"FF#31U
 %N5 4U
 5,O$!U
 #O$$U
 *A%Q U
 
Q?Q:5U
 :Q??AU
 S T$ 0T		T$ U
 $T<<U
 
AW	W0WW*Y= =
[/A[*$[/*[/*E/B[ An 7j n 
j n j  C1n B[ 
on=8B[ =oA.B[ 1Ar B[ 
rB[ rC#B[ 5w v(w (v,,w /v,0w 
B[ 
w>w94B[ 9w>>B
B[ 	6{#  { {# 
{{# {{# "B[ #)|B[ |%B[ 5} B[ '}1.B[ 0}11AB[ BAB0B0ADCACCADCAC,C)ADC+AC,C,ADDADDBAF+F+AH>F<AAH9H	AHHAH9HAH1	H.AH9H0AH1	H1AH9H9AH>IAIIAJI*AJJAJJAJ)J)AK J:AKKAK LB-B[ N=BAQ QB[ QAY=Q&EAY8V)A8AX#X!AY8X#
AYX-AYY
AY8YAYYAY8Y3B[ Y8AY=Y=H3B[ b1Ac cB[ c
AccB[ cAccBB[ eA#Af= f<B[ f=
Ag.gAg)g$B[ g)Ag.g.B[ g2;Ah. h-B[ h.
Aih8AiiB[ iAiiKB[ t:C*Ay+ x%*Ay yAy+ y
AyyAy+ yAyyAy+ y*Az) y+#AzzAz) zB[ z)A{{B[ {A{1 {0B[ {1
A{>{;B[ {=A{>{>I7B[ E6BE= E<B[ E=BFF	B[ FBFFB[ FBF/F.B[ F/
BG2F:BGGBG2GBGGBG2GBGGBG2G"B[ G$BG2G/B[ G1BG2G2FB[ QBQ)Q)BSQ;BRRBS	RBR%R"BS	R$BR%R%BS	S	BSSBBU$U$BW7U5ABW2WBWWBW2WBW*	W'BW2W)BW*	W*BW2W2BW7W;BXXBY	X#BYYBY	YBY"Y"BZY3BZZBZ[Bl?[DBl:_"Bl?_#Bm cBccBe c-BddBd;dBddBd;dBddBd;d;Be eBBggBi)g'ABi$h4BiiBi$iBi	iBi$iBi	iBi$i$Bi)i-BjjBj;jBj6j6Bj;j?BkkBlk%BllBll:Bl?l?Bm mCBzpBp7p6Bzp7Brq	BqqBrqBq3	q0Brq2Bq3	q3BrrBzrBrrBzr BBt2t1Bzt2BwuABw vBv$v!Bw v$Bv8v5Bw v7Bv8v8Bw v;Bzw BwwBzw	Bw wBzw Bxw1BxxBzxBxxBzxBx0x/Bzx0By'yBy"yBzy"By'y'/Bzc                >     d fd}t          |d|           dS )um  Release an ephemeral cron agent's async resources within a hard bound.

    Split out of ``run_job``'s ``finally`` so a caller that defers teardown
    (to deliver first — #58720) can invoke the identical cleanup AFTER delivery.
    The timeout matters because this executes after ``run_conversation`` has
    returned, outside the agent inactivity watchdog.
    r   Nc                  6   	                                   n:# t          t          f$ r&} t                              d|            Y d } ~ nd } ~ ww xY w	 ddlm}  |             d S # t          $ r'} t                              d|            Y d } ~ d S d } ~ ww xY w)Nz-Job '%s': failed to close agent resources: %sr   )cleanup_stale_async_clientsz4Job '%s': failed to reap stale auxiliary clients: %s)r   r   r  r   rK  agent.auxiliary_clientr  )rO  r  rs   ra   s     r    _cleanup_agentz,_teardown_cron_agent.<locals>._cleanup_agentj  s    	U ,- 	U 	U 	ULLH&RSTTTTTTTT	U
	\JJJJJJ''))))) 	\ 	\ 	\LLOQWYZ[[[[[[[[[	\s,    AAAA' '
B1BBzagent resource teardown)ra   r  r  r   )r  )rs   ra   r  r  s   ``  r    rS  rS  `  s]    \ \ \ \ \ \ \ #''	     rf   c                 F   	
                       d          }t          |t                    r$t          |                     d          pd          nd

s |d          S t                                d          pd          t	          j                    t	          j                    	t          j                    }dt          ddf fd}	 t          
	          }n9# t          $ r, t                              d
d            |d           Y dS w xY w|du r(t                              d            |d           dS d	
fd}t	          j        |j        |fdd          }	 |                                 n9# t          $ r, t                              dd            |d           Y dS w xY w	  |	                                           |                    d           S #                                  |                    d           w xY w)zDRun ``run`` while keeping this job's owned durable fire claim fresh.
fire_claimr  r#   Nr?   rI   r   c                                          d          }|sd S 	 t          |d|            d S # t          $ r! t                              dd           Y d S w xY w)Nexecution_idFr  rI   z8Job '%s': failed to close unstarted execution ledger rowTr  )r@   r   r   r   r   )rI   r  r4   ra   s     r    _finish_unstartedz9_run_with_fire_claim_heartbeat.<locals>._finish_unstarted  s    ww~.. 	F	\5FFFFFF 	 	 	NNJ       	s   0 'AAr  z.Job '%s': initial fire_claim validation failedTr  zEFire claim ownership could not be validated before execution started.Fz@Job '%s': fire claim ownership was already lost before executionz3Fire claim ownership lost before execution started.c                  P   t          j                    }                     t                    s	 t	                    s1                                 t                              d           d S t          j                    } n# t          $ rw t          	                    dd           t          j                    | z
  t          k    r8                                 t                              dt                     Y d S Y nw xY w                    t                    d S d S )Nr  z;Job '%s': fire claim ownership lost; interrupting stale runz%Job '%s': fire_claim heartbeat failedTr  zRJob '%s': fire_claim could not be renewed within %.1fs; interrupting uncertain run)r   r  r  r  r   r   r   r   r   rK  #_FIRE_CLAIM_HEARTBEAT_GRACE_SECONDS)last_confirmedra   lost_ownershipr  r  s    r    r  z7_run_with_fire_claim_heartbeat.<locals>._heartbeat_loop  sT   ))))899 	+F5III "&&(((NNU   F!%!1!1   ;!     N$$~5:; ; #&&(((NN5;	   FF; ; ))899 	 	 	 	 	s   A B 2B A;DDzcron-fire-claim-heartbeatr  z.Job '%s': could not start fire_claim heartbeatzAFire claim heartbeat could not be started; execution was not run.r  r  r   )r@   rB   rC   rw   r  r  r  r  r   r   r   r   r  r@  r  r   rj  )r4   r@  r  r  r  owns_fire_claimr  r  ra   r  r  r  s   `       @@@@r    _run_with_fire_claim_heartbeatr    s   GGL!!E*4UD*A*AIC		$%2&&&rE s4yy$"%%F?D_&&N#022        .veLLL 	 	 	< 	 	
 	
 	

 	S	
 	
 	
 tt	 %N	
 	
 	
 	OPPPt        > !' $(	       	 	 	< 	 	
 	
 	

 	O	
 	
 	
 tt	+s>""


c**** 	


c****s0   C# #2DD0F 2F;:F;?
G4 4,H )rC  r>  verboser
  r  r  c          
         
                       d          }t          |t                    r$t          |                     d          pd          nd}t	                      
t                                                      }t          5  |pd|ft          	                     d         i           
<   ddd           n# 1 swxY w Y   	 t           
 fd          t          5  t                                d                   }	|	9|	                    
d           |	s!t                               d         d           ddd           S # 1 swxY w Y   S # t          5  t                                d                   }	|	9|	                    
d           |	s!t                               d         d           ddd           w # 1 swxY w Y   w xY w)u   Run ONE due job end-to-end: execute → save output → deliver → mark.

    This is the shared firing body extracted from ``tick``'s per-job closure so
    that BOTH the built-in ticker and an external provider's ``fire_due`` (e.g.
    Chronos) run the identical sequence — no duplicated correctness.

    It does NOT decide whether the job is due or acquire the initial claim —
    both the ticker and external providers use the same store CAS before
    calling it. It does keep an acquired claim alive for the full execution.

    Returns True if the job was processed (even if the job itself failed —
    failure is recorded via ``mark_job_run``), False only if processing raised.

    ``cancel_event``: optional transport-level cancellation source (dashboard
    webhook drain, API server shutdown). It is OR-combined with the internal
    fire-claim heartbeat's lost-ownership event, so either trigger stops the
    run cooperatively — agent interruption AND script process-tree kill —
    through the single fenced completion path.
    r  r  r#   Nr?   c                 T    t          t          |           n|           S )NrC  r>  r  r
  fire_claim_lostexecution_token)_run_one_job_bodyr   )r  rC  r  r  r
  r4   r>  r  s    r    r!  zrun_one_job.<locals>.<lambda>  sF    #4!) $/ )FFF' /$ $ $ rf   )r@   rB   rC   rw   objectrD  r  r   r   r=  r  r  )r4   rC  r>  r  r
  r  r  r  r  r  r  s   ``````    @r    run_one_jobr    s   8 GGL!!E/9%/F/FNUYYt__*+++BJhhO#%%--//L	 
 
$K
''D	266G
 
 
 
 
 
 
 
 
 
 
 
 
 
 

>-         
 
"  	> 	>-11#d)<<J%555! >(,,SY===	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>] 	> 	>-11#d)<<J%555! >(,,SY===	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>sV   )CC	CE! -AEE	E	!G)AGGGGGGr  r  r  c                ^   1                       d          }t          |t                    r$t          |                     d          pd          nd 1 G d dt                    }1 fd}	dt
          f1 fd}
                      d	          }|st           d
         d          d
         }d}d }	 t           d
                   sIt          	                    d                      d d
                              t          |dd           dS t          |           ddlm}m}m}  | |t!                                          }g }	 t#           ||          \  }}}}nt#           ||          \  }}}}n*# t$          $ r |D ]}t'          | d
                     w xY w ||           n#  ||           w xY w |
            rt|D ]}t'          | d
                    1Ct)           d
         1          r,t+           d
         dd1           t          |dd           nt          |dd           dS d}d}	  |	            5 }|s|t-           d
         |          }d d d            n# 1 swxY w Y   |rt          	                    d|           |rt/           d
         |          rd}d}t          |          ot0          t          |          v }|p$t          |          ot2          t          |          v }t          |          ot4          t          |          v }|p$t          |          ot6          t          |          v }|r\|sZt9          j        ddt          |                                                    }d                      d          p d
          d| d } n|r|nt?           |          tA                     z   } |rZ|sXt9          j        d!dt          |                                                    }!d"                      d          p d
          d#|! } t          |                                           }"|s|rd}"d}#|"r:|r8tC          |           r)t          	                    d$ d
         tD                     d}"|"r- |
            r#d}"t          #                    d% d
                    |"rtI                                d&d'                    d(k    otK                      }#	  |	            5 }$|$s|d}tM           | ||)          }d d d            n# 1 swxY w Y   nY# t          $ rL}%t          |%|          r t          |%          }t          '                    d* d
         |%           Y d }%~%nd }%~%ww xY wn# |$ r d}Y nw xY w|D ]}t'          | d
                    n # |D ]}t'          | d
                    w xY w|s
 |
            rY1Ct)           d
         1          r,t+           d
         dd1           t          |dd           nt          |dd           dS |r|                                sd}d+}tQ           d
         |          }&|&rk|rU	 dd,l)m*}'  |' d
         d-|i           n9# t          $ r,}(t          +                    d. d
         |(           Y d }(~(nd }(~(ww xY wt          |dd/           dS d0|i})11|)d1<   |rd2|)d3<   t+           d
         ||fi |)}*1|*st          |dd4           dS tI                                d&d'                    }+|rd5},n|"r|#rd6},n|"r	|+d'k    rd7},nd8},t          ||||,9           dS # t$          $ r,}-t          |-          ptY          |-          j-        }.t          '                    d: d
         |.           d8},t          |-t                    r|st          |-|          s |
            stI                                d&d'                    }+d}#	 d}tM           t?           |.          ||)          }nH# t          $ r;}/t          |/          }t          '                    d* d
         |/           Y d }/~/nd }/~/ww xY w|s|+d(k    rtK                      }#|rd5},n|#rd6},n|+d'k    rd7},	 tQ           d
         |          s8i })11|)d1<   t          |-t                    r||)d0<   t+           d
         d|.fi |) n9# t          $ r,}0t          '                    d; d
         |0           Y d }0~0nd }0~0ww xY w	 t          |d|.|,9           n9# t          $ r,}0t          '                    d< d
         |0           Y d }0~0nd }0~0ww xY wt          |-t                    s Y d }-~-dS d }-~-ww xY w)=Nr  r  r#   c                       e Zd ZdS )9_run_one_job_body.<locals>._FireClaimLostDuringSideEffectN)ri   rj   rk   rm   rf   r    _FireClaimLostDuringSideEffectr  4  s        rf   r  c                  ^     t          j        d          S t          d                    S )NTr?   r  )
contextlibnullcontextr   )r  r4   s   r    _side_effect_fencez-_run_one_job_body.<locals>._side_effect_fence7  s1    )$///D	*EEEErf   r   c                  
                                      rdS dS 	 t          d                   rdS n4# t          $ r' t                              dd         d           Y dS w xY w                                   dS )NTFr?   r  z0Job '%s': fire_claim ownership validation failedr  )r   r   r   r   rK  r   )r  r  r4   s   r    _fire_claim_ownership_lostz5_run_one_job_body.<locals>._fire_claim_ownership_lost<  s    &?+A+A+C+C&45		#CIjIII u 	 	 	LLBD	    
 55	 &!!!ts   9 -A*)A*r  r?   directsourceFu6   Job '%s': one-shot dispatch limit reached — skippingr>   z3Dispatch claim rejected; execution was not started.r  Tr   )build_profile_secret_scopereset_secret_scopeset_secret_scope)r  r
  r  r  z3Interrupted by shutdown before terminal completion.r  z6Fire claim ownership lost; stale result was discarded.zOutput saved to: %sz`Interrupted by gateway shutdown before the run finished (tool subprocess was killed mid-flight).z\[blocked_config[^\]]*\]\s*u
   ⛔ Cron 'z>' blocked by configuration validation (no LLM call was made): zQ This alert is sent once; the job stays blocked until the configuration is fixed.z\[drift_skip[^\]]*\]\s*rL   z' skipped: u1   Job '%s': agent returned %s — skipping deliveryz;Job '%s': skipping delivery after fire claim ownership lossr  r  r  )rC  r>  zDelivery failed for job %s: %szWAgent completed but produced empty response (model error, timeout, or misconfiguration))
update_joblast_delivery_errorz:Failed recording delivery_error for interrupted job %s: %sz;Interrupted by gateway shutdown before terminal completion.delivery_errorr  blocked_configrk  z5Fire claim ownership lost before terminal completion.r1  not_configuredrQ  
suppressed)r  rI   delivery_outcomezError processing job %s: %sz/Failed to record interrupted run for job %s: %sz0Failed to finish execution record for job %s: %s).r@   rB   rC   rw   r   r   r   r   r   r   r   r   agent.secret_scoper  r  r  rD  r  r  rS  r   r   r   r  rN  rO  rU  rV  rY   r[   r%   re   rH   r   rB  r   r  r  rv  rI   r  r  r  rK  r  ri   )2r4   rC  r>  r  r
  r  r  r  r  r  r  r  delivery_attemptedr  r  r  r  _scope_token_deferred_agentsr  rb  r0  rI   _deferred_agentr  side_effect_ownership_lostowns_outputoutput_fileblocked_config_silentdrift_skip_silent
drift_skip_pf_textdeliver_content_drift_textshould_deliverunresolved_originowns_deliverydeinterruptedr  _rec_errmark_kwargsr  normalized_deliverr  rO  r  delivery_exc
record_errr  s2   `    `                                           @r    r  r  '  s    GGL!!E/9%/F/FPUYYt__*+++DJ       F F F F F F
        & 77>**L J'D	(CCCDIN[ c$i(( 
	KKHD	**   K   
 4 	|,,,	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 ('&&'7'9'9::
 
 "$	-&9@)9!-: : :6 :A)9!-!0	: : :6  	 	 	 $4 A A$_c$i@@@@	 |,,,,|,,,,%%'' 	#3 A A$_c$i@@@@ %*>D	*+ + +% II(2	    ! !O     ! !R   
 4 %*"B	A##%% A" 988-c$i@@A A A A A A A A A A A A A A A  @1;???  ?3t9oFF ?  UJ <E

 J " 3 UC 5U C  UF 8CJJ F  + U? 1SZZ ?   g 
 62BE

 %'' 2!=CI 2 22 2 2   5< #..8eDD+C001    g 
 #%&2BE

# #egg  )(D3t9 ) )&) ) $ "/"7"7"9"9::N$ '(9 '!& %  '' '.G.X.X 'OQTUYQZ\ijjj!& "<"<">"> !&QI  
  R,SWWY-H-HIIXU ;5c::: "R++-- 	, A"@@-1*)8+%-!%	* * *		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ! R R R!"&DEE %(WWNLL!A3t9bQQQQQQQQ	R
 . 	. 	. 	.)-&&&	. $4 A A$_c$i@@@@A#3 A A$_c$i@@@@A & 	)C)C)E)E 	 %*>D	*+ + +% II(2	    ! !O     ! !R   
 4
  	n>//11 	nGmE/D	?KK 	 444444Js4y+@.*QRRRR    LLTD	8       
 S   
 4'8!1;K-. 	5$4K!c$i%GG;GG!&!M   
 45cggi6Q6QRR 	,' 	, 1 	,/ 	, 2g = =*+-		
 	
 	
 	
 t M M M FF.d1gg.	2CIyIII' q)$$	/&	/ q"@AA	/ /.00		/ ":	7++" " !&%)"!08iHH%	" " "    !$\!2!24c$i       
 " G&8H&D&D(A#(F(F$F! /#+  " /#3  #w..#. 	,SYHH I )9CK 56a++ C4BK 01SYyHHKHHH 	 	 	LLAD	:       		!1	      	 	 	LLBD	:       	
 !Y'' 	uuuuu[Ms  5A]5 ;]5 2F F8 'F))F8 ,]5 8GA?]5 ]5 
V I<0V <J  V J I?V 
U  T4(U  4T88U  ;T8<U  ?V  
V
AVV VV W V$!W #V$$W ']5 W  A&]5 4]5 =Z ]5 
[""[	]5 	[]5 %=]5 $A]5 5f, Bf'#af'
b1b=f'b-f'5Adf'
d:"d50f'5d::f'>ef'
f"f>f'ff''f,c                      	 ddl m}   |                                              dS # t          $ r&}t                              d|           Y d}~dS d}~ww xY w)u@  Best-effort: tell the active scheduler provider the job set changed.

    Called by the consumer surfaces (model tool / CLI / REST) AFTER a
    successful store mutation (create/update/remove/pause/resume) so an external
    provider (Chronos) can re-provision/cancel the affected one-shot via NAS.
    No-op for the built-in (it re-reads jobs.json each tick), so the default
    path is unchanged. Lives here (not in cron/jobs.py) to keep the store free
    of provider imports — avoids an import cycle and keeps jobs.py low-coupling.
    Never raises into the caller.
    r   resolve_cron_schedulerz!on_jobs_changed notify failed: %sN)cron.scheduler_providerr  on_jobs_changedr   r   rK  )r  rO  s     r    _notify_provider_jobs_changedr     s    =BBBBBB  0022222 = = =8!<<<<<<<<<=s   "& 
AAAc                   H     e Zd ZdZdededdf fdZdefdZdefdZ	 xZ
S )	CronSchedulerRegistrationErrorzFA job was persisted but its first external trigger was not registered.r4   causer   Nc                     || _         || _        t                                          d|d          dt	          |          j         d           d S )NrX  r?   z:' was saved, but its first scheduler registration failed (zS). Do not create a duplicate. Pause/resume or update the job to retry registration.)r4   r#  superr   r  ri   )r   r4   r#  rY  s      r    r   z'CronSchedulerRegistrationError.__init__  sm    
OT O O$(KK$8O O O	
 	
 	
 	
 	
rf   c                 \    | j                             d          p| j         d         }d| dS )zEHuman-facing variant for chat/CLI surfaces (no exception class name).r>   r?   zSaved cron job 'u   ', but couldn't register it with the external scheduler yet. The job is kept — don't re-create it; pause/resume or edit it (e.g. via /cron) to retry registration.)r4   r@   )r   r  s     r    user_messagez+CronSchedulerRegistrationError.user_message  s?    V$$6Nu N N N	
rf   c                 B    t          |           | j        d         ddddS )zDReturn the public partial-failure contract without provider details.r?   TF)rI   ra   	job_savedscheduler_registeredretry_create)rw   r4   r   s    r    to_dictz&CronSchedulerRegistrationError.to_dict  s.     YYhtn$)!
 
 	
rf   )ri   rj   rk   rl   rC   r   r   rw   r'  r,  __classcell__)rY  s   @r    r"  r"    s        PP
D 
 
t 
 
 
 
 
 

c 
 
 
 

 
 
 
 
 
 
 
 
rf   r"  c                      ddl m} ddlm}  |di | }	  |                                |           n## t
          $ r}t          ||          |d}~ww xY w|S )zHPersist one job and register its first trigger with the active provider.r   )
create_jobr  Nrm   )r  r/  r  r  register_jobr   r"  )r  r/  r  r4   r   s        r    &create_job_with_scheduler_registrationr1  $  s    $$$$$$>>>>>>
*

v

C@  --c2222 @ @ @,S#66C?@Js   4 
AAAg     r@_last_dead_owner_reap_atT)can_dispatchsyncc                   &'( t                      \  }}|                    dd           d}	 t          |dd          }t          r-t          j        |t          j        t          j        z             n9t          r2t          j        |	                                t          j
        d           n# t          $ r}|Ut          |          rFt                              d           	 |                                 n# t          $ r Y nw xY wY d}~d	S |&	 |                                 n# t          $ r Y nw xY wt!          |          rt                              d
|           nt                              d|            d}~ww xY w	 	 d	dlm}	  |	dt                    r	 t          r8	 t          j        |t          j                   ni# t          t*          f$ r Y nVw xY wt          rK	 t          j        |	                                t          j        d           n# t          t*          f$ r Y nw xY w|                                 d	S n# t.          $ r Y nw xY w| |            st                              d           	 t          r8	 t          j        |t          j                   ni# t          t*          f$ r Y nVw xY wt          rK	 t          j        |	                                t          j        d           n# t          t*          f$ r Y nw xY w|                                 d	S t1          j                    }
t4          |
t4          z
  t6          k    rc|
a	 d	dlm}  |            }|rt                              d|           n2# t>          $ r%}t                              d|           Y d}~nd}~ww xY wtA                      }tB          r|}	 tE          tB                    }d |D             }||k    sd	dl#m$}  |            }n# t>          $ r Y nw xY w	 tK          |           n2# t>          $ r%}t                              d|           Y d}~nd}~ww xY w|s( r:t          &                    dtO                      (                    d                     	 d	dl)m*}  |             n2# t>          $ r%}t                              d|           Y d}~nd}~ww xY w	 t          r8	 t          j        |t          j                   ni# t          t*          f$ r Y nVw xY wt          rK	 t          j        |	                                t          j        d           n# t          t*          f$ r Y nw xY w|                                 d	S  rHt          &                    dtO                      (                    d          tW          |                     tY          d |D                        d}	 t[          j.        dd          /                                }|rta          |          pd}n1# tb          td          f$ r t                              d           Y nw xY w|u	 tg                      pi }ti          |tj                    r|6                    di           ni 6                    d          }|ta          |          pd}n# t>          $ r Y nw xY w r-t          &                    dtW          |          |r|nd            d!tj          d"tn          f fd#&d$ |D             }d% |D             }g }g }d!tj          d&tp          j9        j:        f&fd'}|rNtw                      }|D ]=}  || |          }!|!|<                    |!           |s|<                    d           >|rOt{          |          }"|D ]=}  || |"          }!|!|<                    |!           |s|<                    d           >d-d((|rOtp          j9        >                    |          D ]q}#	 |<                    |#?                                           +# t>          $ r:}t                              d)|           |<                    d*           Y d}~jd}~ww xY w (             t          |          t          r8	 t          j        |t          j                   ni# t          t*          f$ r Y nVw xY wt          rK	 t          j        |	                                t          j        d           n# t          t*          f$ r Y nw xY w|                                 S |rEtW          |          g'd+tp          j9        jA        d"df'(fd,}$|D ]}%|%B                    |$           n
 (             t          |          t          r8	 t          j        |t          j                   ni# t          t*          f$ r Y nVw xY wt          rK	 t          j        |	                                t          j        d           n# t          t*          f$ r Y nw xY w|                                 S # t          r8	 t          j        |t          j                   ni# t          t*          f$ r Y nVw xY wt          rK	 t          j        |	                                t          j        d           n# t          t*          f$ r Y nw xY w|                                 w xY w).uj  
    Check and run all due jobs.
    
    Uses a file lock so only one tick runs at a time, even if the gateway's
    in-process ticker and a standalone daemon or manual tick overlap.
    
    Args:
        verbose: Whether to print status messages
        adapters: Optional dict mapping Platform → live adapter (from gateway)
        loop: Optional asyncio event loop (from gateway) for live adapter sends
        can_dispatch: Optional synchronous gate; false leaves due jobs untouched
            for the next allowed tick

    Returns:
        Number of jobs executed (0 if another tick is already running)
    Tr9  Nwr=  r>  r=   u0   Tick skipped — another instance holds the lockr   uj   Cron tick could not acquire tick lock: %s — scheduler will attempt fd reclamation and retry with backoffz)Cron tick could not acquire tick lock: %s)check_pausedr8   z7Cron dispatch paused while gateway drains existing work)recover_interrupted_executionsziReclaimed %d cron execution(s) whose owner process died before reaching a terminal state (marked unknown)z'Dead-owner execution reclaim failed: %sc                 b    h | ],}t          |t                    |                    d           -S rT  rU  rV  s     r    r  ztick.<locals>.<setcomp>  s1    QQQAZ4=P=PQAEE$KKQQQrf   )	load_jobsz Stale in-flight sweep failed: %sz%s - No jobs duez%H:%M:%S_kill_orphaned_mcp_children'Post-tick MCP orphan cleanup failed: %sz%s - %s job(s) duec                     g | ]
}|d          S rT  rm   )r   r4   s     r    r   ztick.<locals>.<listcomp>  s    9993t9999rf   HERMES_CRON_MAX_PARALLELr#   z?Invalid HERMES_CRON_MAX_PARALLEL value; defaulting to unboundedmax_parallel_jobsz.Running %d job(s) in parallel (max_workers=%s)	unboundedr4   r   c                    t          | d         d          }|st          | d         dd           dS t          |t                    rt          |          nt          |           }| d         |d<   t	          |          S )	u   Run one due job end-to-end. Thin wrapper around the shared
            module-level ``run_one_job`` so ``tick`` and external providers
            (Chronos ``fire_due``) use the identical execute→save→deliver→mark
            body.r?   T)
return_jobr  Fz+Fire claim lost; execution was not started.r  )rC  r>  r  )r   r   rB   rC   r  )r4   claimedclaimed_jobrC  r>  r  s      r    _process_jobztick.<locals>._process_job  s     )TtDDDG  '!G   
 t ,6gt+D+DS$w---$s))K*-n*=K'!	   rf   c                 d    g | ]-}|                     d           pd                                +|.S r  r#   r@   r%   rV  s     r    r   ztick.<locals>.<listcomp>#  s:    SSS1553C3C3Ir2P2P2R2RS1SSSrf   c                 d    g | ]-}|                     d           pd                                +|.S rH  rI  rV  s     r    r   ztick.<locals>.<listcomp>$  s:    UUUqQUU95E5E5K4R4R4T4TUUUUrf   rs  c                    
  d         
d 
fd}t                      r;t                              d                     d
                      |             dS t	          
          s1t                              d                     d
                     dS 	 t          
d	          }t           |d         
          }t          j	                    }na# t          $ rT}t          
            |             t                              d                     d
          |           Y d}~dS d}~ww xY w||ffd	}	 |                    |          }n# t          $ r}	t          
            |             t          |d         dd|	            t          |	t                     rDt          |	          r5t                              d                     d
                     Y d}	~	dS t                              d                     d
          |	           Y d}	~	dS d}	~	ww xY wt$          5  
t&          v r
|t(          
<   ddd           n# 1 swxY w Y   |S )a'  Submit a job fire-and-forget with the in-flight dedup guard.

            Returns the future, or None if the job was skipped because a prior
            tick's run of the same job is still in flight.  The running-set
            membership is released in the worker's finally block.
            r?   r   Nc                  B                        d          } t          | t                    r|                      d          dk    sdS 	 t                     dS # t          $ r;}t
                              d                     d          |           Y d}~dS d}~ww xY w)uX  Best-effort claim cleanup on the dispatch-failure paths.

                Only one-shot jobs carry a ``run_claim`` (stamped by
                get_due_jobs, #59229), so recurring jobs skip the call
                entirely — clear_run_claim acquires _jobs_lock (blocking
                cross-process flock) and does a full load_jobs read, and the
                dispatch-failure paths fire exactly when the process can
                least afford N pointless lock/read round-trips (interpreter
                shutdown, EMFILE).  clear_run_claim itself does
                load_jobs/save_jobs file I/O; on those degraded paths it can
                raise, and these early-exits exist precisely to skip cleanly
                — a stale claim expiring at the TTL is a better outcome than
                crashing the tick (#86522).
                r6   r7   r  Nz\Could not clear run_claim for job '%s' after dispatch failure: %s (claim will expire at TTL)r>   )r@   rB   rC   r   r   r   r   )	_schedule	claim_errr4   ra   s     r    _clear_run_claim_best_effortzFtick.<locals>._submit_with_guard.<locals>._clear_run_claim_best_effort2  s      GGJ//	"9d33 	f8M8MQW8W8WF#F+++++    NNA//!	        s   A 
B#0BBu8   Job '%s' not dispatched — interpreter is shutting downr>   u%   Job '%s' already running — skippingbuiltinr  )r  z6Job '%s' not dispatched: execution creation failed: %sc                     	 |                     |           t          | d                    S # t          | d                    w xY w)Nr?   )r@  r  )rW  ctxrF  s     r    _run_and_releasez:tick.<locals>._submit_with_guard.<locals>._run_and_releaser  sG    177<33'$0000'$0000s	   . AFzExecutor dispatch failed: r  zJob '%s' not dispatched: %sr   )r  r   r   r@   r  r   r   rC   r  r  r   r  rA  rB  r   rB   rA  rI   r   r   r   )r4   rs  rO  	executiondispatched_job_ctxexecution_errrS  r]  
submit_errra   rF  s   `         @r    _submit_with_guardz tick.<locals>._submit_with_guard)  s1    YF      B *++ NGGFF++   -,...t+F33 CSWWVU[E\E\]]]t,VIFFF	!%c	$!H!H!H"/11    $F+++,,...  LGGFF++!  
 ttttt $2t 1 1 1 1 1 1kk"233   #F+++,,... dO!CzCC    j,77  <VWa<b<b  NNR//    444441GGFF++  
 ttttt+2  3 3---/2$V,3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 JsJ   ;C 
D7#A	D22D7E 
H+#BH&00H&&H+5IIIc                      	 ddl m}   |              d S # t          $ r&}t                              d|           Y d }~d S d }~ww xY w)Nr   r;  r=  )rZ  r<  r   r   rK  )r<  _es     r    _sweep_mcp_orphansz tick.<locals>._sweep_mcp_orphans  sy    LFFFFFF++----- L L LFKKKKKKKKKLs    
A?AzCron job future failed: %sF_fc                    dxx         dz  cc<   	 |                                  }|2t                              d|t          |          ||j        f           n# t
          $ r Y nw xY wd         dk    r              d S d S )Nr   r=   z(Cron job future failed in async mode: %sr  )rA  r   rI   r  __traceback__r   )r]  _exc
_remainingr\  s     r    _on_doneztick.<locals>._on_done  s    1"<<>>D'%OQUaefjakakmqsw  tF  aG  H  H  H    Da=A%%&&((((( &%s   AA 
A)(A)r   )Cr  rF  rG  r  flockLOCK_EXLOCK_NBr  lockingfilenoLK_NBLCKrf  r  r   rK  r   r	  rI   agent.estopr7  LOCK_UNIOErrorLK_UNLCKImportErrorr   r  r2  !_DEAD_OWNER_REAP_INTERVAL_SECONDSru  r8  r   r   r   r   r   r  r:  r  r   rA  r  rZ  r<  r\   r   r  r  r%   rA   r'   r  r   rB   rC   r@   r   r  r  r  r  rx   r  as_completedr   sumFuturerH  ))r  rC  r>  r4  r3  r  	lock_filelock_fdr   _estop_check_paused	_reap_nowr8  
_reclaimed	_reap_excrQ  _sweep_jobs_inflight_ids_due_ids_load_all_jobsrO  r<  r[  _max_workers_env_parr  _cfg_parsequential_jobsparallel_jobs_results_all_futuresrY  seq_poolr4   r]  rs  r  rb  r]  rF  ra  r\  s)   ```                                   @@@r    tickr  9  sM   0 *++HiNN4$N/// G"y#888 	AK!>???? 	AN7>>++V_a@@@   #<S#A#ALLKLLL   11111    S!! 	K
 LL@    LLDcJJJ9<s
	GGGGGG""6622 B  		GU]3333W%    	w~~//!DDDDW%   Y 	 	 	D	 #LLNN#LLRSSSv
  		GU]3333W%    	w~~//!DDDDW%   o
 N$$	$,337XXX'0$SJJJJJJ;;==
 NNL"  
  S S SF	RRRRRRRRS  >>  	F"K #$4 5 5QQQQQ$00EEEEEE"0."2"2K   F$[1111 F F FA1EEEEEEEEF  	  T.0F0Fz0R0RSSSLFFFFFF++---- L L LFKKKKKKKKLf  		GU]3333W%    	w~~//!DDDDW%   w  	aKK,kmm.D.DZ.P.PRUV^R_R_``` 	99999::: '+	^y!;R@@FFHHH 5"8}}4I& 	^ 	^ 	^NN\]]]]]	^#+-7t-D-DLEIIfb)))"#)**  '#&x==#8DL     	KK@H ,=+  	d 	t 	 	 	 	 	 	 	 	D TShSSSUUHUUUm	D m	
0B0U m	 m	 m	 m	 m	 m	j  	*+--H& * *((h77;##C((( *OOD)))  	*%l33D$ * *((d33;##C((( *OOD)))	L 	L 	L 	L  
	!  '44\BB + ++OOAHHJJ////  + + +LL!=sCCCOOE********+    x==6  		GU]3333W%    	w~~//!DDDDW%   A  	!l++,J	)Z/6 	)4 	) 	) 	) 	) 	) 	) 	) # / /$$X..../    8}} 		GU]3333W%    	w~~//!DDDDW%     		GU]3333W%    	w~~//!DDDDW%   s  A?B2 2
F<+F(C=<F=
D
F	D

FFD+*F+
D85F7D88A
FFI ,G G G +2H H21H2i 
Ii I)i J+ +J?>J?
2K= =LL*/i -N i 
N7N2-i 2N77i 6P	 i 	
Pi Pi P* )i *
Q4Qi QAi R- ,i -
S7Si Si (T TT'2U U.-U.A%i -:X( 'i (+Yi Yi A#[  ?i  
[
i [Ei 'a i 
b0b ;i  bi (ccc'2dd.-d.Ai -gg! g!,2hh32h3k2i43k24jk2j
k22kk2kk2kk2__main__)r  )r   r   r   N)r   r   r   r   )NN)NFN)TNNT)rl   r?  atexitconcurrent.futuresr  r  r  r  rI  loggingr  rY   r  r   r  r  r  r   r[  r   r   r  rm  r  r  r   typingr   r   r	   r
   rM  r4  rw   r  rE  r   r   hermes_cli._subprocess_compatr   r  r   r   r   r   r   rT  r   hermes_timer   rA  agent.interrupt_compatr   agent.delegation_contextr   r   	getLoggerri   r   r!   r/   r3   rC   rH   re   r   rh   r   r}   r   r   r   r   ry  r|  r  r  r   r   r   r   r   r   r   r   r   r   r   ru  r   r   r   rB  r   r   r   r  r  __annotations__r   rA   r   r   r   r  r  r  r   r   r   r  r   r   rC  r  r   r   r   r   r  r  r  r  r  r$  r(  r4  rP  r  r  r  r  r  r  rJ  r  r  r  r  r  r  r  registerr  r  r  r  r  r  rD  r  rf  r  r  r	  r  r  r  r!  r&  r5  rD  r]  r`  rn  rw  rz  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  rv  r{  rz  r  r  r~  r  r  r  r  r  r  r  r  r  r  r  rM  r/  rZ  rO  rN  rV  rU  r  r  r  r  r  r  r  r  r  r  rS  r  r  r  r   rA  r"  r1  rn  r2  r  rm   rf   r    <module>r     s                      				 				       



       ' ' ' ' ' ' ' 'LLLL   E   	       0 0 0 0 0 0 0 0 0 0 0 0
 33ttH~~,344 5 5 5 , , , , , , < < < < < <              : 9 9 9 9 9 * * * * * * 9 9 9 9 9 9       
 
	8	$	$   $$ $ $N    8't ' ' ' ' 'TT9d T93: T9# T9 T9 T9 T9n
 
 
 
 
 
 
 
 $s)    Dd3i d tCy    > 4 DI<L    B'Vt 'V$ 'Vs 'Vtd{ 'V 'V 'V 'VX &I ' ' '    ' % !	
 # + 
 ! ' # ! # - ! '  3! 0 +                            W V V V V V V V V V
 0C 0D 0 0 0 0* CG+>? F F F,0 HSM 0 0 0 #   LN d3VU8C=$3F-G%G HHI N N N	       $    &((  s    $     #'   CEE c ! ! !    x          (I I I I&S T    <+ + + + + + +  +  +  +  +L   d    %    Dt     >
$ 
 
 
 
(F3 Fc F FZ_ Fdh F F F F*E! E!8D> E!T E! E! E! E!V "&R R RR #R 
	R R R Rj. .C .(8 .D . . . ., c (62B d    . EI (:-@A H H HQ$ Q$ Q$ Q$ Q$ Q$ Q$ Q$l $^%% " #( #'  %    $5    HSM j6H6[    j0C    "	  	  	  	  ' ( ( (=4 = = = =R R R R R:t : : : : :  H]$; t    B !dTk      	-$ 	- 	- 	- 	--tTz* - - - -7 t     C C C C C C	,= 	,T 	, 	, 	, 	,   :'s 's ' ' ' 'T (4.    ,% %t %(4. %D % % % %P4  c &.sm8<   L  $!C
 C
 C
 C
	C
C
 C
 	C

 }C
 c]C
 C
 
C
 C
 C
 C
L"	" "
 c]" " " "X  $"x
 x
	x
 x
 	x

 x
 x
 }x
 x
 smx
 
x
 x
 x
 x
H  $"l l l	l l 	l
 l l c]l }l sml 
l l l l^4T 4c 4 4 4 40     &
,s 
,t 
, 
, 
, 
,
& 
& 
& 
& 
& 
&C    6
C 
C 
 
 
 
C C     c hsm    D  $     0'tDz ' ' ' 'TLD LT L L L L,#D # # # #d dc dhtn d d d dN    0 )UG$$ c    $4 DJ    B+$ +8D> + + + + iIIIJJiBBBCC Y YY Y Tk	Y 
Y 
Y Y Y Yx1d 1 1 1 1*555 5 	5
 
5 5 5 5pX X X X(SV- X X X Xv  )# &BQ&F ##S # # # #B "  '  '  '  '  'Ft S#X     ') ')c4S>>Q8R ') ') ') ')T/)9 /d / / / /dj. 4    >+6+6c3h+6 +6 
#Y	+6 +6 +6 +6` "/3}8 }8}8c]}8 +,}8 49	}8 }8 }8 }8F "/3	A+ A+	A+A+ c]A+ +,	A+
 49A+ A+ A+ A+H4C 4D 4 4 4 48 &*"&C C	CE?C 3-C 		C C C CT #!%C C CC	C 	C
 C #C 	C C C CL+MT +Md +M +M +M +Mf + 8  # 0 \m \ \ \ \ \~	8 	8$ 	8 	8 	8 	8-t -$ -8C= - - - -`34 3HSM 3 3 3 3l5 5(3- 5 5 5 5pt $ 8C=    Bu    . (,5 5 5 5 	5
 e_5 
5 5 5 5p/ / / / / / / /j ,0"&/3P0 P0 P0	P0 #4.P0 3-	P0
 +,P0 4c8C=()P0 P0 P0 P0h, ?C  ,4UO	   Bd+ d+d d+ d+ d+ d+T 	"&/3<> <> <>	<>
 <> 3-<> +,<> 
<> <> <> <>D 	"&26(,H H H	H
 H 3-H ./H f%H 
H H H HV= = = =$
 
 
 
 
\ 
 
 
@
 
 
 
 
" %* !,0 (5/ 0 0 0 		y y y yy 	y y y yx zD s6   A A3A#"A3#A-*A3,A--A32A3