
    sjUR                       d Z ddlmZ ddlZddlmZmZmZmZ ddl	m
Z
 ddlmZ ddlmZ  ej        e          ZdZd	Zd
Zd8dZddd9dZ	 d:d;dZdddd<dZd Zd!Zd=d$Zd>d&Zd?d'Zeed(dfd@d0Zd:dAd3ZdBd4Z dCd7Z!dS )Dua	  Native OpenAI Responses server-side compaction — gpt-5.6 on direct OpenAI routes only.

OpenAI's Responses API supports server-side compaction: include
``context_management=[{"type": "compaction", "compact_threshold": N}]`` in a
``/v1/responses`` request and, when the rendered input crosses N tokens, the
server summarizes older context into an opaque ``compaction`` output item
(``encrypted_content``, sealed to the issuing endpoint). Replaying that item
as an input item on later requests stands in for the pruned history, so the
model keeps long-horizon recall without the client ever seeing a summary.
Docs: https://developers.openai.com/api/docs/guides/compaction

Hermes' support is deliberately narrow (live verification, Aug 2026):

* **gpt-5.6 family only.** gpt-5.6 and its variants compact correctly.
  Sending the field to gpt-5.1 / gpt-5.2 reliably fails server-side —
  HTTP 500 on the blocking path and a permanent stall on the streaming
  path (90s watchdog x 3 retries = a dead turn). There is no structured
  "unsupported" rejection to downgrade on, so the only safe gate is an
  explicit model-family check.
* **Direct OpenAI routes only:** api.openai.com (API key) or the ChatGPT
  Codex backend (subscription OAuth). Every other Responses surface
  (xAI, GitHub/Copilot, relays, local servers) never sees the field —
  most would 400 on the unknown parameter, and none can mint or decrypt
  the compaction blob.

Ownership model: Hermes' local compression stays fully armed as the
fallback owner. The native threshold is clamped safely below the local
compressor's trigger so the server compacts first; if it doesn't (native
disabled mid-session, provider hiccup, non-eligible route), the local
summarizer fires exactly as before. There is no new custody state — the
captured compaction items ride the existing ``codex_reasoning_items``
sidecar, which already handles persistence (state.db), gateway session
replay, cross-issuer stamping, and the encrypted-replay kill switch.

This module stays free of transport/adapter dependencies so the transport,
adapter, and conversation loop can share the gate without import cycles. The
two exceptions — ``agent.context_compressor`` and ``agent.message_content`` —
sit below this module in the dependency graph (neither imports
``native_compaction``), so importing their provenance/text primitives here
introduces no cycle.
    )annotationsN)AnyDictListOptional)urlsplitis_compaction_summary_message)flatten_message_texti    i@ zgpt-5.6modelOptional[str]returnboolc                <    t           | pd                                v S )z-True when the model is in the gpt-5.6 family. )_ELIGIBLE_MODEL_MARKERlower)r   s    ;/home/agent/.hermes/hermes-agent/agent/native_compaction.pyis_native_compaction_modelr   A   s    !ekr%8%8%:%:::    Fis_codex_backendbase_urlr   c                   |rdS 	 t          | pd          j        pd                                }n# t          $ r Y dS w xY w|dk    S )uF   True for api.openai.com or the ChatGPT Codex backend — nothing else.Tr   Fzapi.openai.com)r   hostnamer   
ValueError)r   r   r   s      r   is_direct_openai_router   F   sk      tX^,,5;BBDD   uu'''s   *1 
??configured_thresholdr   local_trigger_tokensintc                   	 t          |           }n# t          t          f$ r
 t          }Y nw xY wt	          | t
                    s|dk    rt          }d}	 |$t	          |t
                    st          |          }n# t          t          f$ r d}Y nw xY w||dk    r|S |t          k    r|t          z
  }n t          dt          |dz                      }t          dt          ||                    S )ae  Clamp the configured native threshold below the local compressor trigger.

    Without the clamp a native threshold above the local trigger would let the
    local summarizer fire first every time, making native compaction dead
    config. ``local_trigger_tokens`` is ``ContextCompressor.threshold_tokens``
    when a compressor is attached, else None.
    r   Ni   g?)	r    	TypeErrorr   DEFAULT_COMPACT_THRESHOLD
isinstancer   LOCAL_TRIGGER_SAFETY_MARGINmaxmin)r   r   
configuredlocaluppers        r   resolve_compact_thresholdr+   U   s   /-..

z" / / /.


/&-- /q.
E+J?SUY4Z4Z+,--Ez"   }

***33E3us{++,,uc*e,,---s    --&A< <BB)is_xai_responsesis_github_responsesagentr,   r-   Optional[List[Dict[str, Any]]]c                  t          t          | dd                    sdS t          t          | dd                    sdS |s|rdS t          t          | dd                    sdS t          t          | dd          |          sdS t          | d	d          }t	          t          | d
t
                    |t          |dd          nd          }d|dgS )u  Return the ``context_management`` payload for this request, or None.

    None means "do not send the field" — the request is byte-identical to
    pre-feature behavior. All gates are re-checked per request so a
    mid-session model switch or the in-session kill switch
    (``agent.codex_responses_native_compaction = False``, set by the
    conversation loop's rejection recovery) takes effect on the next call.
    !codex_responses_native_compactionFNcompression_enabledTr   r   r   context_compressor!codex_responses_compact_thresholdthreshold_tokens
compaction)typecompact_threshold)r   getattrr   r   r+   r#   )r.   r   r,   r-   
compressor	thresholds         r   $native_compaction_context_managementr<   w   s    BEJJKK t 4d;;<< t . t%geWd&C&CDD t!z4((;K    t 4d;;J):<UVV9C9O
.555UY I "	BBCCr   i   i }  textstrc                B    t          dt          |           dz            S )uF   Cheap chars//4 token estimate — same shape Codex uses for retention.      )r&   len)r=   s    r   _approx_tokensrC      s    q#d))q.!!!r   itemc                   t          | t                    sdS |                     d          }|d| v r|                     d          }t          |t                    r|                                r|ndS t          |t
                    rg }|D ]}t          |t                    r<|                                r'|                    |                                           Tt          |t                    r(|                    d          p)|                    d          p|                    d          }t          |t                    r;|                                r'|                    |                                           |                    d          }t          |t                    rot          |                    d          t                    rG|d                                         r-|                    |d                                                    d                    |          }|                                r|ndS dS )zExtract measurable text from string, list content, output_text, or nested metadata text.

    Returns None when the item carries no measurable text.
    Handles string content, multipart lists (input_text/text/output_text), and fallback keys.
    Ncontentoutput_textr=   
input_textmetadata )r$   dictgetr>   striplistappendjoin)rD   rF   partspart	part_text	part_metar=   s          r   _extract_item_textrU      s    dD!! thhy!!G=D00((=))'3 4!--//3wwt3'4   . 	@ 	@D$$$ 
@::<< /LL...D$'' @ HHV,,a0F0Fa$((S`JaJa	i-- 4)//2C2C 4LL!2!2333 HHZ00	i.. @:immF>S>SUX3Y3Y @ (..00 @Yv%6%<%<%>%>???xxzz||-tt-4r   c                     t          |           S )u  True when *item* is a canonical Hermes compression-summary message.

    Delegates entirely to
    ``agent.context_compressor.is_compaction_summary_message`` — the single
    authoritative provenance check already used by every other summary
    consumer (memory providers, frontends, the compactor itself). It prefers
    the exact, truthy ``COMPRESSED_SUMMARY_METADATA_KEY`` marker and falls
    back to the canonical prefix classifier (``SUMMARY_PREFIX`` /
    ``LEGACY_SUMMARY_PREFIX`` / historical prefixes, including the
    merge-into-tail shape) for the case where the underscore-prefixed key
    was already stripped by a wire sanitizer.

    Deliberately NOT a second heuristic: no arbitrary underscore-key scan, no
    inference from a falsy or unrelated metadata key, and no matching on
    ad-hoc content headings like ``"## Summary"`` in ordinary text — any of
    those can promote a normal user/assistant message (or adversarial
    content) to durable retained history (#90975 review).
    r	   )rD   s    r   _is_summary_itemrW      s    & )...r   TitemsList[Dict[str, Any]]retained_user_token_budgetretained_summary_token_budgetenable_summary_retentionitem_sourcesOptional[List[Any]]c                   t          | t                    r| s| S d}t          |           D ]5\  }}t          |t                    r|                    d          dk    r|}6|| S |}|dk    rt          | |dz
           t                    rm| |dz
                               d          dk    rK|dz  }|dk    r@t          | |dz
           t                    r"| |dz
                               d          dk    K| d|         }	| ||dz            }
| |dz   d         }t          |t                    r+t          |          t          |           k    r|d|         }ndgt          |	          z  }g }t          dt          |                    }t          dt          |                    t                      dfd
}t          t          |	          t          |                    D ]0\  }}t          |t                    s|rt          |t                    rt          |          rt          |t                    r"t          |                    d                    nd}|                                r|nd} ||          }|r>|                    d          }|                    |dv r|nd|d           |d         z  d|v r|                    d          dk    r|ot          |          }|                    d          dk    }|s|s-t          |          }|@|s|sF|r/ ||          }|r |                    |           |d         z  w|r|dk    rt!          |          }||k    r|                    |           ||z  }t          |                    d          t"                    rVt          |          }|d         d|dz           |d<   |d                                         r|                    |           d}2t          t          |                    }|
|z   |z   }t$                              dt          |           t          |          |           |S )u
  Restructure Responses input around the newest compaction checkpoint.

    The server drops every input item that precedes a replayed ``compaction``
    item (live-verified Aug 2026), so sending pre-checkpoint history is dead
    weight AND silently erases the user's plaintext asks — including any
    local-compression summary the agent already produced, which previously
    vanished here because it carries ``role="assistant"``, not ``"user"``
    (#90975). When a checkpoint is present, rebuild the wire as::

        [checkpoint run] + [retained user & summary messages (newest-first budget)] + [post]

    - The NEWEST contiguous run of checkpoints wins.
    - Retained user messages are kept verbatim within
      ``retained_user_token_budget``; the boundary message is head-truncated
      when it only partially fits (string content only) — goals are usually
      stated up front, so the head is the valuable end.
    - Compression summary messages (``_is_summary_item``, the canonical
      ``agent.context_compressor`` provenance check) are retained whole
      within ``retained_summary_token_budget``. A summary is never
      byte/character-sliced: Hermes summaries carry structural framing
      (handoff prefix, end marker, merge-into-tail delimiters) that a blind
      slice can corrupt, so one that doesn't fit whole is dropped instead.
      A summary already retained once (identical text) is never duplicated,
      so repeated checkpoints stay idempotent.
    - ``enable_summary_retention`` is a function-level override (used by
      tests and callers that need the pre-#90975 behavior back); it is not
      wired to a user-facing config surface.
    - Original relative chronological order between user messages and
      summaries is preserved.
    - ``item_sources`` (optional, parallel to ``items``) is the raw chat
      message each Responses item was converted from. By the time a summary
      reaches this function as a converted ``item`` it can already be lossy:
      a merge-into-tail tool-result carrier becomes a typed
      ``function_call_output`` (no ``content``/``role`` survives the
      conversion at all), and a merge-into-tail assistant carrier can be
      shadowed by a stale exact ``codex_message_items`` replay captured
      before the merge rewrote its content. When a source is provided and is
      itself a canonical summary carrier (``is_compaction_summary_message``),
      its content is read directly from the source — never from the
      converted item — and it is retained as a synthesized
      ``role="assistant"`` message regardless of what shape the original
      item took. Without ``item_sources`` (default), retention only sees
      what survived conversion, matching pre-#90976 behavior (#90976).
    Nr7   r6   r   r@   r=   r   r   Optional[Dict[str, Any]]c                    | r
dk    s| v rdS t          |           }|k    rdS                     |            d|iS )z@Check budget/dedup/cost for a summary; return cost info or None.r   Ncost)rC   add)r=   rb   seen_summary_textssummary_remainings     r   _try_retain_summaryz7prune_pre_checkpoint_items.<locals>._try_retain_summary:  sd     	(A--9K1K1K4d##### 4t$$$~r   rF   r   role)user	assistantri   )rg   rF   rb   messagerh   rA   zRPruned pre-checkpoint items: %d input -> %d retained (user_rem=%d, summary_rem=%d))r=   r   r   r`   )r$   rN   	enumeraterK   rL   rB   r&   r    setzipreversedrW   r   rM   rO   rU   rC   r>   loggerdebug)rX   rZ   r[   r\   r]   last_cpirD   first_cpprecheckpoint_runpostpre_sourcesretained_reverseduser_remainingrf   sourcer=   result	_src_role
is_summaryis_userrb   	truncatedretained_orderedrd   re   s                            @@r   prune_pre_checkpoint_itemsr      s   f eT"" % GU##  4dD!! 	dhhv&6&6,&F&FG H1uX\*D11 	(Q,##F++|;;A	 	1uX\*D11 	(Q,##F++|;; 		
C8gk12N1D,%% (#l*;*;s5zz*I*I!-ixi!8fs3xx'.0C :;;<<NAs#@AABB!ee
 
 
 
 
 
 
 HSMM8K+@+@AA ?# ?#f$%% 	 $ 	
64(@(@ 	EUV\E]E] 	BLVUYBZBZb'

9(=(=>>>`bD::<<144TD((..F 4"JJv..	!(()26K)K)KIIQ\#* *    "VF^3!
 T>>dhhv..);;-H2B42H2H
((6""f, 	z 	!$''<
  	G 	 	#((..F 4!((...!VF^3! 	#""!$''D~%%!((...$&DHHY//55 # JJ	'+I7K!9K7K'L	)$Y'--// 8%,,Y777!"H%67788..5F
LL\E

F   Mr   errorstatus_codec                    t          | pd                                          dvrdvrdS |.	 t          |          dk    rdS n# t          t          f$ r Y nw xY wd}t          fd|D                       S )	u  True when a provider error is a STRUCTURED rejection of the
    context_management field.

    Used by the conversation loop's one-shot recovery: strip the field,
    disable native compaction for the rest of the session, retry. Matching
    is deliberately narrow — a transient 5xx/timeout whose body merely
    ECHOES the request (and therefore contains the field name) must NOT
    permanently downgrade native compaction for the session (#82777).

    Two conditions, both required when a status is known:

    * ``status_code`` is 400 (or unknown/None — some transports surface
      only a message string; field-name matching alone is then the best
      available signal, preserving pre-#82777 behavior for them), and
    * the error text names ``context_management`` / ``compact_threshold``
      alongside rejection language ("unknown", "unsupported", "invalid",
      "unexpected", "not permitted"...). A bare field-name echo without
      rejection language does not match.
    r   context_managementr8   FNi  )unknownunsupportedinvalid
unexpectedznot permittedznot allowedunrecognizedzextra fieldzno suchzbad requestznot supportedc              3      K   | ]}|v V  	d S N ).0markerr=   s     r   	<genexpr>z1is_native_compaction_rejection.<locals>.<genexpr>  s'      >>&v~>>>>>>r   )r>   r   r    r"   r   any)r   r   rejection_markersr=   s      @r   is_native_compaction_rejectionr     s    ( u{!!##D4'',?t,K,Ku	;3&&u ':& 	 	 	D	
 >>>>,=>>>>>>s   A AAc                b    t          d t          | t                    r| ndD                       S )u  Does this ``codex_reasoning_items`` sidecar carry a compaction checkpoint?

    A ``type: "compaction"`` item is the server-side stand-in for history that
    has already been pruned — cumulative context, not per-turn reasoning. It
    rides the same sidecar as ordinary reasoning items, so anything that
    rewrites or discards that sidecar (or the message carrying it) has to ask
    this question first: the checkpoint exists in exactly one place, and the
    request that loses it loses the compacted history with it.
    c              3  r   K   | ]2}t          |t                    o|                    d           dk    V  3dS )r7   r6   Nr$   rK   rL   r   rD   s     r   r   z,has_compaction_checkpoint.<locals>.<genexpr>  sV         	4C488F#3#3|#C     r   r   )r   r$   rN   )rX   s    r   has_compaction_checkpointr     sE       (55=UU2     r   prior_items	new_itemsc                    d t          | t                    r| ng D             }t          |t                    rt          |          ng }t          |          s|s|S ||z   S )a  Merge ``codex_reasoning_items`` across Codex incomplete-continuation
    dedup, preserving native compaction checkpoints.

    The incomplete-retry path updates a visually-duplicate interim assistant
    message in place with the newer response's replay payload. A checkpoint
    captured on the EARLIER response is a cumulative context carrier the
    continuation won't re-emit (the replayed checkpoint keeps the server
    render under threshold), so a blind overwrite drops the only copy and the
    next request balloons back to full history. Rule: newer items win, but
    prior checkpoints are prepended unless the newer payload carries its own.
    c                n    g | ]2}t          |t                    r|                    d           dk    0|3S )r7   r6   r   r   s     r   
<listcomp>z1merge_interim_reasoning_items.<locals>.<listcomp>  sM       dD!! '+hhv&6&6,&F&F 	&F&F&Fr   )r$   rN   r   )r   r   kept_checkpointsnew_lists       r   merge_interim_reasoning_itemsr     s|     $.{D$A$AI[[r  
 #-Y"="=EtI2H ** 2B h&&r   )r   r   r   r   )r   r   r   r   r   r   r   )r   r   r   r   r   r    )
r.   r   r   r   r,   r   r-   r   r   r/   )r=   r>   r   r    )rD   r   r   r   )rD   r   r   r   )rX   rY   rZ   r    r[   r    r\   r   r]   r^   r   rY   )r   r   r   r   r   r   )rX   r   r   r   )r   r   r   r   r   rY   )"__doc__
__future__r   loggingtypingr   r   r   r   urllib.parser   agent.context_compressorr
   agent.message_contentr   	getLogger__name__ro   r%   r#   r   r   r   r+   r<   "RETAINED_USER_MESSAGE_TOKEN_BUDGETRETAINED_SUMMARY_TOKEN_BUDGETrC   rU   rW   r   r   r   r   r   r   r   <module>r      s  ( (T # " " " " "  , , , , , , , , , , , , ! ! ! ! ! ! B B B B B B 6 6 6 6 6 6		8	$	$ $ #  # ; ; ; ; #( ( ( ( ( (" !%. . . . .L # %#D #D #D #D #D #DR &, " !' " " " "
! ! ! !H/ / / /0 'I)F%)(,l l l l l^"? "? "? "? "?J    ' ' ' ' ' 'r   