diff --git a/config/aipods/artificer.yaml b/config/aipods/artificer.yaml index ecca087..33dc9e4 100644 --- a/config/aipods/artificer.yaml +++ b/config/aipods/artificer.yaml @@ -16,8 +16,8 @@ spec: providerId: NC01 backendProfile: gpt-pika model: - model: gpt-5.6-terra - reasoningEffort: xhigh + model: gpt-5.6-sol + reasoningEffort: medium imageRef: kind: env-image-dockerfile repoUrl: git@github.com:pikasTech/agentrun.git diff --git a/src/selftest/cases/76-aipod-spec.ts b/src/selftest/cases/76-aipod-spec.ts index 1fad5c5..da2fefb 100644 --- a/src/selftest/cases/76-aipod-spec.ts +++ b/src/selftest/cases/76-aipod-spec.ts @@ -49,8 +49,8 @@ const selfTest: SelfTestCase = async (context) => { assert.equal(shownItem.backendProfile, "gpt-pika"); assert.equal(shownItem.providerId, "NC01"); assert.equal(shownItem.lane, "v0.2"); - assert.equal(((shownItem.model as JsonRecord).model), "gpt-5.6-terra"); - assert.equal(((shownItem.model as JsonRecord).reasoningEffort), "xhigh"); + assert.equal(((shownItem.model as JsonRecord).model), "gpt-5.6-sol"); + assert.equal(((shownItem.model as JsonRecord).reasoningEffort), "medium"); assert.deepEqual(shownItem.gitIdentity, { name: "AgentRun Codex", email: "agentrun-codex@users.noreply.github.com", @@ -91,13 +91,13 @@ const selfTest: SelfTestCase = async (context) => { assert.equal(((runnerDefaults.imageRef as JsonRecord).dockerfilePath), "deploy/container/Containerfile"); assert.equal(runnerDefaults.namespace, "agentrun-v02"); const taskPayload = task.payload as JsonRecord; - assert.equal(taskPayload.model, "gpt-5.6-terra"); - assert.equal(taskPayload.reasoningEffort, "xhigh"); + assert.equal(taskPayload.model, "gpt-5.6-sol"); + assert.equal(taskPayload.reasoningEffort, "medium"); assert.equal((taskPayload.modelConfig as JsonRecord).modelSource, "aipod-spec-default"); assert.equal((taskPayload.modelConfig as JsonRecord).reasoningEffortSource, "aipod-spec-default"); const modelResolution = rendered.modelResolution as JsonRecord; - assert.equal(modelResolution.model, "gpt-5.6-terra"); - assert.equal(modelResolution.reasoningEffort, "xhigh"); + assert.equal(modelResolution.model, "gpt-5.6-sol"); + assert.equal(modelResolution.reasoningEffort, "medium"); assert.equal(modelResolution.modelSource, "aipod-spec-default"); assert.equal(modelResolution.reasoningEffortSource, "aipod-spec-default"); assert.equal(modelResolution.backendProfile, "gpt-pika"); @@ -146,11 +146,11 @@ const selfTest: SelfTestCase = async (context) => { assert.deepEqual((modelOverrideResolution.providerCredential as JsonRecord).secretRef, disclosedProviderCredential.secretRef); assert.deepEqual(((modelOverrideTask.executionPolicy as JsonRecord).secretScope as JsonRecord).providerCredentials, providerCredentials); - const reasoningOverride = await client.post("/api/v1/aipod-specs/Artificer/render", { prompt: "override reasoning", reasoningEffort: "medium" }) as JsonRecord; + const reasoningOverride = await client.post("/api/v1/aipod-specs/Artificer/render", { prompt: "override reasoning", reasoningEffort: "high" }) as JsonRecord; const reasoningOverridePayload = (reasoningOverride.queueTask as JsonRecord).payload as JsonRecord; const reasoningOverrideResolution = reasoningOverride.modelResolution as JsonRecord; - assert.equal(reasoningOverridePayload.model, "gpt-5.6-terra"); - assert.equal(reasoningOverridePayload.reasoningEffort, "medium"); + assert.equal(reasoningOverridePayload.model, "gpt-5.6-sol"); + assert.equal(reasoningOverridePayload.reasoningEffort, "high"); assert.equal(reasoningOverrideResolution.modelSource, "aipod-spec-default"); assert.equal(reasoningOverrideResolution.reasoningEffortSource, "render-input-override"); @@ -236,8 +236,8 @@ const selfTest: SelfTestCase = async (context) => { const changedRender = await client.post("/api/v1/aipod-specs/Artificer/render", { prompt: "new yaml default" }) as JsonRecord; assert.equal(((changedRender.queueTask as JsonRecord).payload as JsonRecord).model, "gpt-5.7-terra"); const storedTask = await client.get(`/api/v1/queue/tasks/${String(durableTask.id)}`) as JsonRecord; - assert.equal((storedTask.payload as JsonRecord).model, "gpt-5.6-terra"); - assert.equal((storedTask.payload as JsonRecord).reasoningEffort, "xhigh"); + assert.equal((storedTask.payload as JsonRecord).model, "gpt-5.6-sol"); + assert.equal((storedTask.payload as JsonRecord).reasoningEffort, "medium"); assert.deepEqual((storedTask.payload as JsonRecord).modelConfig, taskPayload.modelConfig); return { name: "aipod-spec", tests: ["aipod-spec-yaml-parser-runtime-compatible", "aipod-spec-artificer-image-ref-render", "aipod-spec-artificer-v02-runtime-authority", "aipod-spec-gpt-pika-secret-namespace", "aipod-spec-model-defaults", "aipod-spec-model-reasoning-overrides", "aipod-spec-model-override-keeps-provider", "aipod-spec-control-override-fail-closed", "aipod-spec-payload-control-bypass-fail-closed", "aipod-spec-missing-gpt-pika-credential", "aipod-spec-durable-task-model-snapshot", "aipod-spec-primary-workspace-contract", "aipod-spec-artificer-github-url-render", "aipod-spec-artificer-github-ssh-required-keys", "aipod-spec-git-mirror-url", "git-fetch-stderr-classification", "resource-bundle-source-authority-validation", "resource-bundle-target-conflict-validation", "queue-submit-aipod-dry-run", "session-send-aipod-model-dry-run", "aipod-cli-help"] }; } finally { diff --git a/src/selftest/cases/80-artificer-model-selection.ts b/src/selftest/cases/80-artificer-model-selection.ts index 85b9946..2c1d0e8 100644 --- a/src/selftest/cases/80-artificer-model-selection.ts +++ b/src/selftest/cases/80-artificer-model-selection.ts @@ -33,8 +33,8 @@ const selfTest: SelfTestCase = async (context) => { const modelResolution = rendered.modelResolution as JsonRecord; assert.equal(task.backendProfile, "gpt-pika"); assert.equal(task.providerId, "NC01"); - assert.equal(modelResolution.model, "gpt-5.6-terra"); - assert.equal(modelResolution.reasoningEffort, "xhigh"); + assert.equal(modelResolution.model, "gpt-5.6-sol"); + assert.equal(modelResolution.reasoningEffort, "medium"); await assertDurableRetryModel(context.tmp, task as unknown as CreateQueueTaskInput); const runtimeExecutionPolicy = structuredClone(task.executionPolicy as JsonRecord); ((runtimeExecutionPolicy.secretScope as JsonRecord).toolCredentials) = []; @@ -63,7 +63,12 @@ const selfTest: SelfTestCase = async (context) => { codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, - env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "require-explicit-model-effort" }, + env: { + CODEX_HOME: context.codexHome, + AGENTRUN_FAKE_CODEX_MODE: "require-explicit-model-effort", + AGENTRUN_FAKE_CODEX_EXPECTED_MODEL: "gpt-5.6-sol", + AGENTRUN_FAKE_CODEX_EXPECTED_REASONING_EFFORT: "medium", + }, oneShot: true, }); assert.equal(result.terminalStatus, "completed"); @@ -71,12 +76,12 @@ const selfTest: SelfTestCase = async (context) => { const starting = events.items?.find((event) => event.type === "backend_status" && event.payload.phase === "codex-app-server-starting"); const selection = starting?.payload.modelSelection as JsonRecord | undefined; assert.equal(starting?.payload.backendProfile, "gpt-pika"); - assert.equal(selection?.model, "gpt-5.6-terra"); - assert.equal(selection?.reasoningEffort, "xhigh"); + assert.equal(selection?.model, "gpt-5.6-sol"); + assert.equal(selection?.reasoningEffort, "medium"); assert.equal(selection?.reasoningProtocolField, "turn/start.effort"); const turnStart = events.items?.find((event) => event.type === "backend_status" && event.payload.phase === "turn/start:completed"); - assert.equal(turnStart?.payload.model, "gpt-5.6-terra"); - assert.equal(turnStart?.payload.reasoningEffort, "xhigh"); + assert.equal(turnStart?.payload.model, "gpt-5.6-sol"); + assert.equal(turnStart?.payload.reasoningEffort, "medium"); assert.equal(turnStart?.payload.reasoningProtocolField, "effort"); assertNoSecretLeak({ rendered, result, events }); return { name: "artificer-model-selection", tests: ["artificer-gpt-pika-default-render", "artificer-durable-retry-model", "artificer-gpt-pika-model-runtime", "artificer-reasoning-effort-turn-start", "artificer-model-selection-visible"] }; @@ -134,8 +139,8 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin }); const command = retried.command as JsonRecord; assert.deepEqual(command.payload, task.payload); - assert.equal((command.payload as JsonRecord).model, "gpt-5.6-terra"); - assert.equal((command.payload as JsonRecord).reasoningEffort, "xhigh"); + assert.equal((command.payload as JsonRecord).model, "gpt-5.6-sol"); + assert.equal((command.payload as JsonRecord).reasoningEffort, "medium"); assert.equal(((command.payload as JsonRecord).modelConfig as JsonRecord).modelSource, "aipod-spec-default"); } diff --git a/src/selftest/fake-codex-app-server.ts b/src/selftest/fake-codex-app-server.ts index b9432e1..13f6dbb 100644 --- a/src/selftest/fake-codex-app-server.ts +++ b/src/selftest/fake-codex-app-server.ts @@ -4,6 +4,8 @@ import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; const rl = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); const mode = process.env.AGENTRUN_FAKE_CODEX_MODE ?? "success"; +const expectedModelEffortModel = process.env.AGENTRUN_FAKE_CODEX_EXPECTED_MODEL ?? "gpt-5.6-terra"; +const expectedModelEffort = process.env.AGENTRUN_FAKE_CODEX_EXPECTED_REASONING_EFFORT ?? "xhigh"; if (process.env.AGENTRUN_FAKE_CODEX_START_FILE) appendFileSync(process.env.AGENTRUN_FAKE_CODEX_START_FILE, `${process.pid}\n`); const fakeProcessAttempt = process.env.AGENTRUN_FAKE_CODEX_START_FILE ? lineCount(process.env.AGENTRUN_FAKE_CODEX_START_FILE) : 1; let threadCounter = 0; @@ -37,7 +39,7 @@ for await (const line of rl) { respond(message.id, null, { code: -32000, message: "thread/start did not include expected model" }); continue; } - if (mode === "require-explicit-model-effort" && (message.params?.model !== "gpt-5.6-terra" || Object.hasOwn(message.params ?? {}, "effort"))) { + if (mode === "require-explicit-model-effort" && (message.params?.model !== expectedModelEffortModel || Object.hasOwn(message.params ?? {}, "effort"))) { respond(message.id, null, { code: -32000, message: "thread/start expected model without turn effort" }); continue; } @@ -65,7 +67,7 @@ for await (const line of rl) { respond(message.id, null, { code: -32000, message: "thread/resume did not include expected model" }); continue; } - if (mode === "require-explicit-model-effort" && (message.params?.model !== "gpt-5.6-terra" || Object.hasOwn(message.params ?? {}, "effort"))) { + if (mode === "require-explicit-model-effort" && (message.params?.model !== expectedModelEffortModel || Object.hasOwn(message.params ?? {}, "effort"))) { respond(message.id, null, { code: -32000, message: "thread/resume expected model without turn effort" }); continue; } @@ -84,7 +86,7 @@ for await (const line of rl) { respond(message.id, null, { code: -32000, message: "turn/start did not include expected model" }); continue; } - if (mode === "require-explicit-model-effort" && (message.params?.model !== "gpt-5.6-terra" || message.params?.effort !== "xhigh")) { + if (mode === "require-explicit-model-effort" && (message.params?.model !== expectedModelEffortModel || message.params?.effort !== expectedModelEffort)) { respond(message.id, null, { code: -32000, message: "turn/start did not include expected model and effort" }); continue; }