# -*- coding: utf-8 -*- """The capability ladder, and how each harness is pinned to a rung. One wire format, three rungs that every harness can be pinned to, and two more that only some can express. Every harness in the set either speaks the OpenAI chat-completions protocol or can be pointed at something that does, which is the only reason a shared ladder exists at all. primary, crossed with every harness T0 llama3.1:8b on localhost the floor T1 deepseek-v4-flash cheap weights T2 deepseek-v4-pro flagship weights T3 gpt-5.6-luna another vendor's cheap tier, and still above pro on nearly every public benchmark side study, on the harnesses that expose the switch S-off deepseek-v4-flash, thinking off S-high deepseek-v4-flash, thinking high The primary ladder moves the weights and leaves each harness's own deliberation setting alone, which is what a person gets when they install the thing. The side study holds the weights still and moves deliberation instead. Two rules this module exists to enforce: 1. A harness that runs two models internally gets BOTH pinned to the same rung. No smuggling a stronger model into a cheap role. 2. A rung a harness cannot express is returned as unsupported, loudly, so the cell is dropped and named rather than silently run at some other setting. Nothing here writes to a harness repository. Axium and Orange are patched in memory for the life of the process, which is the mechanism paper 04 used. """ import os OLLAMA_BASE = 'http://127.0.0.1:11434/v1' # The floor rung. It is llama3.1:8b and not the 7B first tried, because # Hermes refuses any model advertising under 64,000 tokens of context and # ollama reports qwen2.5:7b-instruct at 32,768. A rung one harness cannot # accept is not a rung. The refusal itself is recorded as a finding. OLLAMA_MODEL = 'llama3.1:8b' # Rung definitions. `priced` says whether a USD figure from this rung means # anything: a local model has no price, and mixing the two in one cost column is # how a benchmark tells its most confident lie. # `effort: None` means: leave the harness's own default alone. # # The ladder was four rungs when it was planned, with deliberation switched off # and on at the same weights. It is three, because the switch is not crossable. # Hermes exposes `reasoning_effort` only for OpenRouter and Nous Portal, not for # an OpenAI-compatible base URL pointed at DeepSeek, so a thinking rung would # have been expressible for the two harnesses I wrote and not for the ones I did # not. A factorial with holes in the third-party column measures the author, not # the field. Deliberation moves to a side study on the subset that supports it, # and every cell records the effort it actually sent. TIERS = { 'T0': { 'id': 'T0', 'label': 'local 8B', 'model': OLLAMA_MODEL, 'cheap_model': OLLAMA_MODEL, # Forced, not chosen: the model has no thinking mode, and the endpoint # answers any reasoning field with a 400. 'effort': 'off', 'cheap_effort': 'off', 'wire': 'openai', 'base_url': OLLAMA_BASE, 'key_env': 'OLLAMA_API_KEY', 'key_default': 'ollama', 'priced': False, 'primary': True, }, 'T1': { 'id': 'T1', 'label': 'flash, harness default', 'model': 'deepseek-v4-flash', 'cheap_model': 'deepseek-v4-flash', 'effort': None, 'cheap_effort': None, 'wire': 'deepseek', 'base_url': 'https://api.deepseek.com/v1', 'key_env': 'DEEPSEEK_API_KEY', 'key_default': '', 'priced': True, 'primary': True, }, 'T2': { 'id': 'T2', 'label': 'pro, harness default', 'model': 'deepseek-v4-pro', 'cheap_model': 'deepseek-v4-pro', 'effort': None, 'cheap_effort': None, 'wire': 'deepseek', 'base_url': 'https://api.deepseek.com/v1', 'key_env': 'DEEPSEEK_API_KEY', 'key_default': '', 'priced': True, 'primary': True, }, 'T3': { 'id': 'T3', 'label': 'gpt-5.6 luna, reasoning max', 'model': 'gpt-5.6-luna', 'cheap_model': 'gpt-5.6-luna', # Reasoning at maximum, which is the only setting at which this rung is # what it claims to be. On the composite intelligence index the model # scores 52 at max effort against deepseek-v4-pro's 44, and 46 at high, # so a rung run with reasoning off would not be a smarter model at all. # # The cost of that decision is an API change: measured on 21 August 2026, # every current model of this family refuses function tools on chat # completions unless reasoning_effort is exactly "none". Reasoning plus # tools exists only on /v1/responses, so every harness reaches this rung # there, and the two that could not were taught to. 'effort': 'max', 'cheap_effort': 'max', 'api': 'responses', 'wire': 'openai-cloud', 'base_url': 'https://api.openai.com/v1', 'key_env': 'OPENAI_API_KEY', 'key_default': '', 'priced': True, 'primary': True, }, # Side study. Same weights, deliberation off against deliberation high, on # the harnesses that expose the switch as ordinary configuration. 'S-off': { 'id': 'S-off', 'label': 'flash, thinking off', 'model': 'deepseek-v4-flash', 'cheap_model': 'deepseek-v4-flash', 'effort': 'off', 'cheap_effort': 'off', 'wire': 'deepseek', 'base_url': 'https://api.deepseek.com/v1', 'key_env': 'DEEPSEEK_API_KEY', 'key_default': '', 'priced': True, 'primary': False, 'subset': ('axium', 'orange', 'dsh', 'openclaw'), }, 'S-high': { 'id': 'S-high', 'label': 'flash, thinking high', 'model': 'deepseek-v4-flash', 'cheap_model': 'deepseek-v4-flash', 'effort': 'high', 'cheap_effort': 'high', 'wire': 'deepseek', 'base_url': 'https://api.deepseek.com/v1', 'key_env': 'DEEPSEEK_API_KEY', 'key_default': '', 'priced': True, 'primary': False, 'subset': ('axium', 'orange', 'dsh', 'openclaw'), }, } ORDER = ['T0', 'T1', 'T2', 'T3', 'S-off', 'S-high'] PRIMARY = ['T0', 'T1', 'T2', 'T3'] class Unsupported(Exception): """This harness cannot express this rung. The cell is dropped and named.""" def deepseek_key(): """The DeepSeek key, from the environment or Axium's config, never printed.""" k = os.environ.get('DEEPSEEK_API_KEY') if k: return k import io import json p = r'C:\xampp\htdocs\axium\python\config.json' if os.path.exists(p): try: return (json.load(io.open(p, encoding='utf-8')) .get('api_keys', {}).get('deepseek') or '') except Exception: # noqa: BLE001 return '' return '' def openai_key(): """The OpenAI key, from the environment or the local key file, never printed.""" k = os.environ.get('OPENAI_API_KEY') if k and k.startswith('sk-'): return k import io as _io import json as _json p = r'C:\xampp\htdocs\windows-agent\keys.local.json' if os.path.exists(p): try: return (_json.load(_io.open(p, encoding='utf-8')).get('openai') or '') except Exception: # noqa: BLE001 return '' return '' def key_for(tier): t = TIERS[tier] if t['wire'] == 'deepseek': return deepseek_key() if t['wire'] == 'openai-cloud': return openai_key() return os.environ.get(t['key_env']) or t['key_default'] # ── Axium ──────────────────────────────────────────────────────────────────── def apply_axium(adapter, tier): """Pin an already-constructed AxiumAdapter to a rung. The OpenAI-compatible base URL table is mutated IN PLACE rather than rebound, because providers.py imported the same dict object; rebinding axium.config.OPENAI_COMPATIBLE would leave the provider module still pointing at the previous endpoint while the configuration looked right. """ t = TIERS[tier] cfg = adapter.cfg if t['wire'] in ('openai', 'openai-cloud'): from axium import config as axcfg axcfg.OPENAI_COMPATIBLE[axcfg.OPENAI] = t['base_url'] os.environ['OPENAI_API_KEY'] = key_for(tier) provider = axcfg.OPENAI else: os.environ['DEEPSEEK_API_KEY'] = key_for(tier) provider = 'deepseek' cfg.models.primary, cfg.models.primary_provider = t['model'], provider cfg.models.continuation, cfg.models.continuation_provider = t['cheap_model'], provider cfg.models.classifier, cfg.models.classifier_provider = t['cheap_model'], provider cfg.models.compactor, cfg.models.compactor_provider = t['cheap_model'], provider cfg.models.review, cfg.models.review_provider = t['cheap_model'], provider cfg.models.fallback, cfg.models.fallback_provider = '', '' if t['effort'] is not None: cfg.settings.thinking_effort = t['effort'] if t['cheap_effort'] is not None: cfg.settings.cheap_effort = t['cheap_effort'] return adapter # ── Orange ─────────────────────────────────────────────────────────────────── def apply_orange(adapter, tier): """Pin an already-constructed OrangeAdapter to a rung, in memory only.""" t = TIERS[tier] ocfg = adapter.ocfg if t['wire'] in ('openai', 'openai-cloud'): ocfg.LLM_PROVIDER = 'openai' ocfg.OPENAI_BASE = t['base_url'] from orange import settings as osettings _force_setting(osettings, 'openai_api_key', key_for(tier)) else: ocfg.LLM_PROVIDER = 'deepseek' ocfg.DEEPSEEK_BASE = 'https://api.deepseek.com' from orange import settings as osettings _force_setting(osettings, 'deepseek_api_key', key_for(tier)) ocfg.CHAT_MODEL = t['model'] ocfg.CODER_MODEL = t['cheap_model'] if t['effort'] is not None: ocfg.CHAT_EFFORT = t['effort'] if t['cheap_effort'] is not None: ocfg.CODER_EFFORT = t['cheap_effort'] if t['wire'] == 'openai': # Only the local rung: that model has no thinking mode and the endpoint # rejects the field entirely. The hosted one wants it set to none, which # the harness now sends for an "off" effort. _orange_no_reasoning(adapter.llm, ocfg) return adapter def _orange_no_reasoning(llm, ocfg): """Stop Orange sending a reasoning level to a model that has none. Measured against the local endpoint on 20 August 2026: any `reasoning_effort` on a non-thinking model is a 400, "qwen2.5:7b-instruct does not support thinking", and Orange's OpenAI path always sets the field, mapping its own "off" to "low" rather than omitting it. Temperature, max_tokens, max_completion_tokens, a developer role and a DeepSeek-style `thinking` object are all accepted by the same server, so this is the only field that has to move. An equalisation, not a favour: the harness is being asked for the behaviour it would have on any non-reasoning model, and no other harness is sending a thinking level on this rung either. """ if getattr(llm, '_versus_local_patch', False): return def _apply(payload, temperature, effort): payload['temperature'] = ocfg.TEMPERATURE if temperature is None else temperature return payload llm._apply_thinking = _apply # noqa: SLF001 llm._versus_local_patch = True # noqa: SLF001 def _force_setting(osettings, key, value): """Override one Orange setting for this process without touching the file. settings.set() would rewrite the user's real data/settings.json and survive a crash mid-run, which is precisely the failure paper 04's Orange adapter was written to avoid. """ prev_get = osettings.get def patched(k, default=None): if k == key: return value return prev_get(k, default) if getattr(osettings.get, '_versus_patched', False): return patched._versus_patched = True # noqa: SLF001 osettings.get = patched # ── Hermes ─────────────────────────────────────────────────────────────────── HERMES_EFFORT = {'off': 'none', 'low': 'low', 'medium': 'medium', 'high': 'high', 'max': 'max'} def hermes_kwargs(tier): """Constructor kwargs for the paper 04 Hermes adapter. Hermes speaks OpenAI chat-completions through OPENAI_BASE_URL, so every rung is reachable. Its constructor takes no effort argument: the level is a config key (`reasoning_effort`, levels none|minimal|low|medium|high|xhigh|max|ultra, with per-model overrides), so `hermes_config` below is written into the per-session HERMES_HOME instead. Passing effort to the constructor would be accepted silently as an unknown kwarg by nothing at all: it raises, which is why it is not passed. """ t = TIERS[tier] kw = {'model': t['model'], 'base_url': t['base_url'], 'api_key': key_for(tier)} return kw def hermes_config(tier): """The config.yaml body for a per-session HERMES_HOME, or nothing. On the primary ladder every rung runs at the harness's own default, so there is nothing to write and the paper 04 adapter runs untouched, which is the point: the third-party harness is measured as it ships. Hermes' `reasoning_effort` key is documented for OpenRouter and Nous Portal only. Pinning deliberation on a raw OpenAI-compatible route would need a custom-provider entry carrying `extra_body`, and that is a different experiment from the one this file configures. Writing a config file to set a key the transport ignores would look like control while changing nothing, which is worse than not setting it. Nothing, on every rung. The hosted rung needed no config either: this harness reaches that model through the Responses API, where the chat completions restriction on tools plus reasoning does not apply, and where its own default effort is what a person would get. """ return {} def _hermes_config_unused(tier): """Kept as the recipe, not called: what a Hermes rung would need. A reasoning level would have to arrive as a custom-provider entry carrying `extra_body`, e.g. {"thinking": {"type": "disabled"}} on the DeepSeek route, written into {HERMES_HOME}/config.yaml with HERMES_IGNORE_USER_CONFIG unset. That is the side study, and it is not what the primary ladder measures. """ t = TIERS[tier] if t['effort'] is None: return {} return {'model': t['model'], 'reasoning_effort': HERMES_EFFORT.get(t['effort'], t['effort'])} # ── DeepSeek Harness ───────────────────────────────────────────────────────── def dsh_patch(tier): """The --patch overlay that pins dsh to a rung. dsh configures providers through the generic pi-ai adapter: a catalog route for DeepSeek, a hand-declared route for anything else. A hand-declared route needs `api`, `baseURL` and a non-empty models list, and the compat switches matter: an endpoint pi-ai cannot recognise is addressed as though it were OpenAI itself, which sends the system prompt as `developer` and the output cap as `max_completion_tokens`, and ollama takes neither. """ t = TIERS[tier] if t['wire'] == 'openai-cloud': # The shipped catalog route, which pi-ai serves over the Responses API. # # A hand-declared chat-completions route was tried first, to put every # harness on one wire. It cannot be done: this model refuses function # tools on chat completions unless reasoning_effort is exactly "none", # and pi-ai has no spelling for that. Declaring the model non-reasoning # makes it send no field at all, which is refused, and declaring only an # off level is rejected by its own schema with "reasoningEfforts offers # no level beyond off". # # So this harness reaches the rung through the API it supports and the # others through theirs. Every row records which, and the limitations # section says what that costs the comparison. return [ {'id': 'llm-pi-ai', 'config': {'providers': { 'openai': {'apiKeyEnv': 'OPENAI_API_KEY'}}}}, {'id': 'agent-default-model', 'config': {'provider': 'openai', 'model': t['model']}}, ] if t['wire'] == 'openai': providers = { 'ollama-local': { 'displayName': 'Ollama local', 'api': 'openai-completions', 'baseURL': t['base_url'], 'apiKeyEnv': t['key_env'], 'compat': {'supportsDeveloperRole': False, 'maxTokensField': 'max_tokens'}, 'models': [{'id': t['model'], 'contextWindow': 131072, 'maxTokens': 4096}], }, } provider = 'ollama-local' else: deepseek = {'apiKeyEnv': 'DEEPSEEK_API_KEY'} if t['effort'] is not None: # Only the side study pins a level. On the primary ladder there is no # override, and an override written anyway produced the sharpest # lesson of the build: `reasoningEfforts: {null: null}` is refused by # the plugin's own schema, the plugin fails to load, node exits, and # the adapter recorded five sessions that scored exactly the floor # because nothing ever ran. A crashed harness and a harness that does # nothing are indistinguishable in the score column. efforts = ({'off': None} if t['effort'] == 'off' else {t['effort']: t['effort']}) deepseek['modelOverrides'] = {t['model']: {'reasoningEfforts': efforts}} providers = {'deepseek': deepseek} provider = 'deepseek' return [ {'id': 'llm-pi-ai', 'config': {'providers': providers}}, {'id': 'agent-default-model', 'config': {'provider': provider, 'model': t['model']}}, ] # ── windows-agent ──────────────────────────────────────────────────────────── def winagent_env(tier): """Environment for the pc-agent subprocess. pc-agent ships two providers, DeepSeek and OpenAI, and picks its model from module constants. The local rung rides the OpenAI path with the base URL repointed, which pc-agent reads from OPENAI_BASE_URL. """ t = TIERS[tier] env = dict(os.environ) if t['wire'] == 'openai-cloud': env['PC_AGENT_PROVIDER'] = 'openai' env['OPENAI_API_KEY'] = key_for(tier) env['OPENAI_MODEL'] = t['model'] env.pop('OPENAI_BASE_URL', None) return env if t['wire'] == 'openai': env['PC_AGENT_PROVIDER'] = 'openai' env['OPENAI_BASE_URL'] = t['base_url'] env['OPENAI_API_KEY'] = key_for(tier) env['OPENAI_MODEL'] = t['model'] else: env['PC_AGENT_PROVIDER'] = 'deepseek' env['DEEPSEEK_API_KEY'] = key_for(tier) env['DEEPSEEK_MODEL'] = t['model'] env['PC_AGENT_EFFORT'] = t['effort'] return env # ── price ──────────────────────────────────────────────────────────────────── # USD per 1,000,000 tokens, as (off-peak, peak). Peak is 01:00 to 04:00 and 06:00 # to 10:00 UTC, at double the off-peak rate. Recorded from the published table on # 20 August 2026 and marked pending-primary in the source log: no cost exhibit is # drawn until the page has been opened at source rather than read from a summary. # # A harness billed at a peak rate against one billed off-peak is not a comparison, # so every row records the rate it was billed at as well as the figure. PRICES = { # Published on the vendor's own pricing page. The cached-input rate for the # hosted frontier model is its published discount on input; recorded as # pending-primary in the source log until the page is read at source, and no # cost exhibit is drawn from it before then. 'gpt-5.6-luna': {'hit': (0.02, 0.02), 'miss': (0.20, 0.20), 'out': (1.20, 1.20)}, 'deepseek-v4-flash': {'hit': (0.007, 0.014), 'miss': (0.22, 0.44), 'out': (0.66, 1.32)}, 'deepseek-v4-pro': {'hit': (0.022, 0.044), 'miss': (0.66, 1.32), 'out': (1.98, 3.96)}, } PEAK_HOURS_UTC = set(range(1, 4)) | set(range(6, 10)) def is_peak(when_utc): return when_utc.hour in PEAK_HOURS_UTC def price(model, input_tokens, output_tokens, cached_tokens, when_utc): """USD for one turn, and the rate label. Returns (0.0, 'unpriced') off table. Cached input is billed at the hit rate and the rest at the miss rate, which is the only way the two columns of the published table mean anything. """ row = PRICES.get(model) if not row: return 0.0, 'unpriced' idx = 1 if is_peak(when_utc) else 0 hit = max(0, int(cached_tokens or 0)) miss = max(0, int(input_tokens or 0) - hit) usd = (hit * row['hit'][idx] + miss * row['miss'][idx] + max(0, int(output_tokens or 0)) * row['out'][idx]) / 1e6 return round(usd, 8), ('peak' if idx else 'off-peak') def label(tier): t = TIERS[tier] return '%s[%s, effort=%s]' % (t['id'], t['model'], t['effort'] if t['effort'] is not None else 'harness default')