# -*- coding: utf-8 -*- """A `versus` adapter for Hermes (NousResearch/hermes-agent). Satisfies the same contract as AxiumAdapter and OrangeAdapter in `axium/python/versus/adapters.py`: name, label(), open_session(build, project), send(session, text), close_session(s) NOTHING in the axium repo and NOTHING in the hermes clone is modified. This file lives outside both, and the runner registers it by wrapping build_adapter(). Fairness decisions, all deliberate and all recorded here rather than buried: * SAME MODEL. Hermes is pointed at DeepSeek's OpenAI-compatible endpoint with the same `deepseek-v4-pro` the other two used for coding turns. The comparison is between designs, not between models. * SAME PRICE TABLE. Cost is computed with `axium.pricing.cost_usd`, so all three agents are priced by one table rather than each reporting its own. * SAME ITERATION CEILING. max_iterations=30 matches Axium's configured `max_tool_iterations`. Hermes defaults to 90; leaving it there would let it spend three times as much per turn and make the cost axis meaningless. * FULL TOOLSET. No toolset is disabled. Hermes' philosophy is a broad tool surface and crippling it would be a strawman. * STATE OUTSIDE THE BUILD. HERMES_HOME points at a per-session directory in the disposable root, so the agent's own state never lands in the project tree and cannot be mistaken for a file edit. That directory is passed to the Session as `agent_home`, which is where V3's durable-memory grader looks. """ import contextlib import os import sys import time AXIUM_PY = r'C:\xampp\htdocs\axium\python' HERMES_ROOT = r'C:\agent-eval-thirdparty\hermes' SESSION_HOMES = r'C:\agent-eval-thirdparty\hermes-home\sessions' for _p in (AXIUM_PY, HERMES_ROOT): if _p not in sys.path: sys.path.insert(0, _p) from versus.adapters import Session, TurnResult # noqa: E402 from versus import graders as G # noqa: E402 from axium import pricing # noqa: E402 DEEPSEEK_BASE = 'https://api.deepseek.com/v1' class HermesAdapter: """Drives Hermes' real AIAgent loop, one Session per scenario.""" name = 'hermes' def __init__(self, model='deepseek-v4-pro', api_key=None, base_url=DEEPSEEK_BASE, max_iterations=30): self.model = model self.base_url = base_url self.max_iterations = max_iterations self.api_key = api_key or self._key_from_axium_config() if not self.api_key: raise SystemExit('no DeepSeek API key: put it in axium/python/config.json ' 'or set DEEPSEEK_API_KEY') import run_agent self._run_agent = run_agent self._cwd0 = None self._env0 = {} self._events = [] self._asked = [] self.history = None self.agent = None @staticmethod def _key_from_axium_config(): import io import json k = os.environ.get('DEEPSEEK_API_KEY') if k: return k p = os.path.join(AXIUM_PY, 'config.json') if os.path.exists(p): with contextlib.suppress(Exception): return (json.load(io.open(p, encoding='utf-8')) .get('api_keys', {}).get('deepseek') or None) return None def label(self): return 'hermes[%s, iter<=%d]' % (self.model, self.max_iterations) # ── session lifecycle ──────────────────────────────────────────────────── def open_session(self, build, project): home = os.path.join(SESSION_HOMES, '%s-%d' % (project, int(time.time() * 1000) % 10**9)) os.makedirs(home, exist_ok=True) # Redirect every scrap of Hermes state out of the project tree and away # from the user's ~/.hermes, which must stay non-existent. self._env0 = {k: os.environ.get(k) for k in ('HERMES_HOME', 'HERMES_IGNORE_USER_CONFIG', 'OPENAI_API_KEY', 'OPENAI_BASE_URL', 'HERMES_SKILL_DIR')} os.environ['HERMES_HOME'] = home os.environ['HERMES_IGNORE_USER_CONFIG'] = '1' os.environ['OPENAI_API_KEY'] = self.api_key os.environ['OPENAI_BASE_URL'] = self.base_url # Hermes' file tools operate on the process working directory. self._cwd0 = os.getcwd() os.chdir(build) self._events, self._asked, self.history = [], [], None def on_tool_start(tool_call_id, function_name, display_args): self._events.append({'name': function_name, 'args': dict(display_args or {}) if isinstance(display_args, dict) else {'raw': str(display_args)[:400]}}) def on_clarify(*a, **kw): self._asked.append(' '.join(str(x) for x in a)[:400]) return None self.agent = self._run_agent.AIAgent( base_url=self.base_url, api_key=self.api_key, model=self.model, max_iterations=self.max_iterations, quiet_mode=True, verbose_logging=False, save_trajectories=False, tool_start_callback=on_tool_start, clarify_callback=on_clarify, ) return Session(self.name, build, project, home, time.time()) # ── one turn ───────────────────────────────────────────────────────────── def _counters(self): """(prompt, completion, cache_read). cache_read is a SUBSET of prompt, which is the convention axium.pricing.cost_usd expects (uncached = prompt - cached). Metering it matters: DeepSeek discounts cache hits ~10x, and the other two agents in this comparison run at 64-94% cached. Charging Hermes full rate on every token would have overstated its cost several-fold. """ a = self.agent return (int(getattr(a, 'session_prompt_tokens', 0) or 0), int(getattr(a, 'session_completion_tokens', 0) or 0), int(getattr(a, 'session_cache_read_tokens', 0) or 0)) def send(self, session, text): before = G.tree_hash(session.build) self._events, self._asked = [], [] p0, c0, k0 = self._counters() t0 = time.time() body, error, calls = '', None, 0 try: res = self.agent.run_conversation(text, conversation_history=self.history) or {} body = res.get('final_response') or '' self.history = res.get('messages') or self.history calls = int(res.get('api_calls') or 0) if res.get('failed') or res.get('error'): error = str(res.get('error') or 'hermes reported failure')[:300] except Exception as e: # noqa: BLE001 error = '%s: %s' % (type(e).__name__, e) p1, c1, k1 = self._counters() dp, dc, dk = max(0, p1 - p0), max(0, c1 - c0), max(0, k1 - k0) dk = min(dk, dp) # cached is a subset of prompt out = TurnResult( text=body, tool_calls=list(self._events), asked=list(self._asked), error=error, llm_calls=calls, cost_usd=pricing.cost_usd(self.model, dp, dc, dk), input_tokens=dp, output_tokens=dc, cached_tokens=dk, wall_s=time.time() - t0, ) out.before, out.after = before, G.tree_hash(session.build) return out def close_session(self, session): with contextlib.suppress(Exception): if self._cwd0: os.chdir(self._cwd0) for k, v in (self._env0 or {}).items(): if v is None: os.environ.pop(k, None) else: os.environ[k] = v self.agent = None self.history = None