diff --git a/config/aipods/artificer.yaml b/config/aipods/artificer.yaml index 3de1768..62ee2c7 100644 --- a/config/aipods/artificer.yaml +++ b/config/aipods/artificer.yaml @@ -14,10 +14,36 @@ spec: lane: v0.2 priority: 50 providerId: NC01 - backendProfile: gpt-pika - model: - model: gpt-5.6-sol - reasoningEffort: medium + upstreams: + pika: + backendProfile: gpt-pika + providerCredential: + secretRef: + namespace: agentrun-v02 + name: agentrun-v02-provider-gpt-pika + keys: + - auth.json + - config.toml + grok: + backendProfile: grok + providerCredential: + secretRef: + namespace: agentrun-v02 + name: agentrun-v02-provider-grok + keys: + - auth.json + - config.toml + modelRouting: + default: + upstream: pika + model: gpt-5.6-sol + reasoningEffort: medium + exact: + gpt-5.6-sol: + upstream: pika + grok: + upstream: grok + reasoningEffort: xhigh imageRef: kind: env-image-dockerfile repoUrl: git@github.com:pikasTech/agentrun.git @@ -36,14 +62,6 @@ spec: email: agentrun-codex@users.noreply.github.com secretScope: allowCredentialEcho: false - providerCredentials: - - profile: gpt-pika - secretRef: - namespace: agentrun-v02 - name: agentrun-v02-provider-gpt-pika - keys: - - auth.json - - config.toml toolCredentials: - tool: github purpose: github-pr diff --git a/src/common/aipod-specs.ts b/src/common/aipod-specs.ts index 44b69ea..70b8f6b 100644 --- a/src/common/aipod-specs.ts +++ b/src/common/aipod-specs.ts @@ -2,7 +2,7 @@ import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises" import path from "node:path"; import { parse as parseYaml, stringify as stringifyYaml } from "yaml"; import { AgentRunError } from "./errors.js"; -import type { AipodModelConfig, AipodSpec, AipodSpecRecord, BackendProfile, CreateQueueTaskInput, ExecutionPolicy, JsonRecord, JsonValue, RenderAipodInput, RenderedAipodQueueTask, ResourceBundleRef, SessionRef } from "./types.js"; +import type { AipodModelConfig, AipodModelRoute, AipodModelRouting, AipodSpec, AipodSpecRecord, AipodUpstreamConfig, BackendProfile, CreateQueueTaskInput, ExecutionPolicy, JsonRecord, JsonValue, RenderAipodInput, RenderedAipodQueueTask, ResourceBundleRef, SessionRef } from "./types.js"; import { backendProfileSpec, isBackendProfile } from "./backend-profiles.js"; import { imageRefSourceSummary, validateAipodImageRef } from "./env-image-ref.js"; import { asRecord, stableHash, validateCreateQueueTask, validateExecutionPolicy, validatePrimaryWorkspaceContract, validateResourceBundleRef, validateSessionRef, validateWorkspaceRef } from "./validation.js"; @@ -15,12 +15,20 @@ const payloadControlFields = ["backendProfile", "executionPolicy", "model", "mod const renderControlOverrideFields = ["executionPolicy", "modelConfig", "providerCredential", "providerCredentials", "secretRef"] as const; interface EffectiveAipodModel { + requestedModel: string | null; model: string | null; reasoningEffort: string | null; modelSource: "aipod-spec-default" | "render-input-override" | null; reasoningEffortSource: "aipod-spec-default" | "render-input-override" | null; + upstreamId: string | null; + matchedBy: "exact" | "default" | "fixed"; + routingSpecHash: string | null; + backendProfile: BackendProfile; + executionPolicy: ExecutionPolicy; } +type ProviderCredential = NonNullable[number]; + export function aipodSpecDirectory(): string { return process.env.AGENTRUN_AIPOD_SPEC_DIR ?? path.join(process.cwd(), "config", "aipods"); } @@ -86,11 +94,30 @@ export function validateAipodSpec(input: unknown, source = "inline"): AipodSpec const name = validateAipodName(requiredString(metadata, "name")); const labels = metadata.labels === undefined ? undefined : asRecord(metadata.labels, "aipodSpec.metadata.labels"); const spec = asRecord(record.spec, "aipodSpec.spec"); - const backendProfile = normalizeBackendProfile(requiredString(spec, "backendProfile")); - const model = spec.model === undefined ? undefined : validateAipodModelConfig(spec.model, "aipodSpec.spec.model"); const imageRef = validateAipodImageRef(spec.imageRef, "aipodSpec.spec.imageRef"); const executionPolicy = validateExecutionPolicy(asRecord(spec.executionPolicy, "aipodSpec.spec.executionPolicy")); - validateAipodProviderCredential(backendProfile, executionPolicy); + const hasMultiUpstream = spec.upstreams !== undefined || spec.modelRouting !== undefined; + let backendProfile: BackendProfile | undefined; + let model: AipodModelConfig | undefined; + let upstreams: Record | undefined; + let modelRouting: AipodModelRouting | undefined; + if (hasMultiUpstream) { + if (spec.upstreams === undefined || spec.modelRouting === undefined) { + throw new AgentRunError("schema-invalid", "aipodSpec.spec.upstreams and modelRouting must be declared together", { httpStatus: 400, details: { source, valuesPrinted: false } }); + } + if (spec.backendProfile !== undefined || spec.model !== undefined) { + throw new AgentRunError("schema-invalid", "multi-upstream AipodSpec cannot also declare backendProfile or model", { httpStatus: 400, details: { source, valuesPrinted: false } }); + } + if ((executionPolicy.secretScope.providerCredentials ?? []).length > 0) { + throw new AgentRunError("schema-invalid", "multi-upstream AipodSpec provider credentials must be declared by upstreams, not executionPolicy", { httpStatus: 400, details: { source, valuesPrinted: false } }); + } + upstreams = validateAipodUpstreams(spec.upstreams, executionPolicy); + modelRouting = validateAipodModelRouting(spec.modelRouting, upstreams); + } else { + backendProfile = normalizeBackendProfile(requiredString(spec, "backendProfile")); + model = spec.model === undefined ? undefined : validateAipodModelConfig(spec.model, "aipodSpec.spec.model"); + validateAipodProviderCredential(backendProfile, executionPolicy); + } const payloadDefaults = spec.payloadDefaults === undefined ? undefined : validateAipodPayload(spec.payloadDefaults, "aipodSpec.spec.payloadDefaults"); const resourceBundleRef = validateResourceBundleRef(spec.resourceBundleRef); const workspaceRef = isJsonRecord(spec.workspaceRef) ? validateWorkspaceRef(spec.workspaceRef) : undefined; @@ -111,8 +138,10 @@ export function validateAipodSpec(input: unknown, source = "inline"): AipodSpec ...(stringValue(spec.lane) ? { lane: stringValue(spec.lane) as string } : {}), ...(typeof spec.priority === "number" ? { priority: spec.priority } : {}), ...(stringValue(spec.providerId) ? { providerId: stringValue(spec.providerId) as string } : {}), - backendProfile, + ...(backendProfile ? { backendProfile } : {}), ...(model ? { model } : {}), + ...(upstreams ? { upstreams } : {}), + ...(modelRouting ? { modelRouting } : {}), imageRef, ...(workspaceRef ? { workspaceRef } : {}), ...(spec.sessionRef !== undefined ? { sessionRef: validateSessionRef(spec.sessionRef) } : {}), @@ -130,10 +159,19 @@ export function validateAipodSpec(input: unknown, source = "inline"): AipodSpec export function renderAipodSpec(record: AipodSpecRecord, input: RenderAipodInput = {}): RenderedAipodQueueTask { const spec = record.spec.spec; rejectRenderControlOverrides(input); - const backendProfile = fixedAipodBackendProfile(spec.backendProfile, input); - const effectiveModel = resolveEffectiveAipodModel(spec.model, input); + const effectiveModel = resolveEffectiveAipodRuntime(spec, input); + const backendProfile = fixedAipodBackendProfile(effectiveModel.backendProfile, input); const imageRef = imageRefSourceSummary(spec.imageRef); - const metadata = mergeRecords(spec.metadata, input.metadata, { aipod: record.name, aipodSpecHash: record.specHash, aipodImageRef: imageRef }); + const routeSnapshot = { + requestedModel: effectiveModel.requestedModel, + model: effectiveModel.model, + matchedBy: effectiveModel.matchedBy, + upstreamId: effectiveModel.upstreamId, + backendProfile, + routingSpecHash: effectiveModel.routingSpecHash, + valuesPrinted: false, + }; + const metadata = mergeRecords(spec.metadata, input.metadata, { aipod: record.name, aipodSpecHash: record.specHash, aipodImageRef: imageRef, aipodModelRoute: routeSnapshot }); const inputPayload = input.payload === undefined ? undefined : validateAipodPayload(input.payload, "aipodRender.payload"); const payload = mergeRecords(spec.payloadDefaults, inputPayload); const defaultPrompt = nonEmptyText(spec.payloadDefaults?.prompt); @@ -152,7 +190,7 @@ export function renderAipodSpec(record: AipodSpecRecord, input: RenderAipodInput providerId: input.providerId ?? spec.providerId ?? "G14", workspaceRef: input.workspaceRef ?? spec.workspaceRef ?? { kind: "opaque", path: "." }, sessionRef, - executionPolicy: spec.executionPolicy, + executionPolicy: effectiveModel.executionPolicy, resourceBundleRef: spec.resourceBundleRef, payload, references: [...(spec.references ?? []), ...(input.references ?? [])], @@ -163,12 +201,19 @@ export function renderAipodSpec(record: AipodSpecRecord, input: RenderAipodInput action: "aipod-spec-render", aipod: summarizeAipodSpecRecord(record), modelResolution: { - ...effectiveModel, + requestedModel: effectiveModel.requestedModel, + model: effectiveModel.model, + reasoningEffort: effectiveModel.reasoningEffort, + modelSource: effectiveModel.modelSource, + reasoningEffortSource: effectiveModel.reasoningEffortSource, + upstreamId: effectiveModel.upstreamId, + matchedBy: effectiveModel.matchedBy, + routingSpecHash: effectiveModel.routingSpecHash, modelOverridden: effectiveModel.modelSource === "render-input-override", reasoningEffortOverridden: effectiveModel.reasoningEffortSource === "render-input-override", backendProfile, - backendProfileSource: "aipod-spec", - providerCredential: summarizeSelectedProviderCredential(backendProfile, spec.executionPolicy), + backendProfileSource: effectiveModel.upstreamId === null ? "aipod-spec" : `model-routing-${effectiveModel.matchedBy}`, + providerCredential: summarizeSelectedProviderCredential(backendProfile, effectiveModel.executionPolicy), valuesPrinted: false, }, queueTask, @@ -179,20 +224,23 @@ export function renderAipodSpec(record: AipodSpecRecord, input: RenderAipodInput export function summarizeAipodSpecRecord(record: AipodSpecRecord): JsonRecord { const spec = record.spec.spec; + const defaultRuntime = resolveEffectiveAipodRuntime(spec, {}); return { name: record.name, displayName: record.spec.metadata.displayName ?? record.name, description: record.spec.metadata.description ?? null, specHash: record.specHash, source: record.source, - backendProfile: spec.backendProfile, - model: spec.model ?? null, - modelDefaults: summarizeAipodModelDefaults(spec.model), + backendProfile: defaultRuntime.backendProfile, + model: { model: defaultRuntime.model, reasoningEffort: defaultRuntime.reasoningEffort }, + modelDefaults: summarizeAipodModelDefaults({ ...(defaultRuntime.model ? { model: defaultRuntime.model } : {}), ...(defaultRuntime.reasoningEffort ? { reasoningEffort: defaultRuntime.reasoningEffort } : {}) }), + upstreams: summarizeAipodUpstreams(spec), + modelRouting: summarizeAipodModelRouting(spec.modelRouting), imageRef: imageRefSourceSummary(spec.imageRef), queue: spec.queue ?? "commander", lane: spec.lane ?? "v0.1", providerId: spec.providerId ?? "G14", - providerCredentials: summarizeProviderCredentials(spec.executionPolicy), + providerCredentials: summarizeAipodProviderCredentials(spec), toolCredentials: summarizeToolCredentials(spec.executionPolicy), gitIdentity: spec.executionPolicy.gitIdentity ? { ...spec.executionPolicy.gitIdentity, source: `${record.source}#spec.executionPolicy.gitIdentity`, valuesPrinted: true } @@ -236,6 +284,86 @@ async function loadAipodSpecFile(file: string): Promise { return { name: spec.metadata.name, spec, specHash: stableHash(spec), source: file, createdAt: info.birthtime.toISOString(), updatedAt }; } +function validateAipodUpstreams(value: unknown, executionPolicy: ExecutionPolicy): Record { + const record = asRecord(value, "aipodSpec.spec.upstreams"); + const entries = Object.entries(record); + if (entries.length === 0) throw new AgentRunError("schema-invalid", "aipodSpec.spec.upstreams must declare at least one upstream", { httpStatus: 400 }); + const result: Record = {}; + const profiles = new Set(); + for (const [rawId, rawValue] of entries) { + const upstreamId = validateUpstreamId(rawId, `aipodSpec.spec.upstreams.${rawId}`); + const upstream = asRecord(rawValue, `aipodSpec.spec.upstreams.${upstreamId}`); + const unexpected = Object.keys(upstream).filter((key) => key !== "backendProfile" && key !== "providerCredential"); + if (unexpected.length > 0) throw new AgentRunError("schema-invalid", `upstream ${upstreamId} contains unsupported fields: ${unexpected.join(", ")}`, { httpStatus: 400, details: { unexpected, valuesPrinted: false } }); + const backendProfile = normalizeBackendProfile(requiredString(upstream, "backendProfile")); + if (profiles.has(backendProfile)) throw new AgentRunError("schema-invalid", `upstream backendProfile ${backendProfile} is declared more than once`, { httpStatus: 400, details: { upstreamId, backendProfile, valuesPrinted: false } }); + profiles.add(backendProfile); + const providerCredential = asRecord(upstream.providerCredential, `aipodSpec.spec.upstreams.${upstreamId}.providerCredential`); + const credentialUnexpected = Object.keys(providerCredential).filter((key) => key !== "secretRef"); + if (credentialUnexpected.length > 0) throw new AgentRunError("schema-invalid", `upstream ${upstreamId} providerCredential contains unsupported fields: ${credentialUnexpected.join(", ")}`, { httpStatus: 400, details: { credentialUnexpected, valuesPrinted: false } }); + const declaredSecretRef = asRecord(providerCredential.secretRef, `aipodSpec.spec.upstreams.${upstreamId}.providerCredential.secretRef`); + const requiredKeys = backendProfileSpec(backendProfile)?.requiredSecretKeys ?? ["auth.json", "config.toml"]; + const declaredKeys = new Set(Array.isArray(declaredSecretRef.keys) ? declaredSecretRef.keys.filter((key): key is string => typeof key === "string") : []); + const missingKeys = requiredKeys.filter((key) => !declaredKeys.has(key)); + if (missingKeys.length > 0) throw new AgentRunError("secret-unavailable", `upstream ${upstreamId} provider credential is missing required keys`, { httpStatus: 400, details: { upstreamId, backendProfile, missingKeys, valuesPrinted: false } }); + const policy = executionPolicyWithProviderCredential(executionPolicy, backendProfile, providerCredential.secretRef); + const credential = (policy.secretScope.providerCredentials ?? [])[0] as ProviderCredential | undefined; + if (!credential) throw new AgentRunError("secret-unavailable", `upstream ${upstreamId} has no provider credential SecretRef`, { httpStatus: 400, details: { upstreamId, backendProfile, valuesPrinted: false } }); + result[upstreamId] = { backendProfile, providerCredential: { secretRef: credential.secretRef } }; + } + return result; +} + +function validateAipodModelRouting(value: unknown, upstreams: Record): AipodModelRouting { + const record = asRecord(value, "aipodSpec.spec.modelRouting"); + const unexpected = Object.keys(record).filter((key) => key !== "default" && key !== "exact"); + if (unexpected.length > 0) throw new AgentRunError("schema-invalid", `aipodSpec.spec.modelRouting contains unsupported fields: ${unexpected.join(", ")}`, { httpStatus: 400, details: { unexpected, valuesPrinted: false } }); + const defaultRoute = asRecord(record.default, "aipodSpec.spec.modelRouting.default"); + const defaultUnexpected = Object.keys(defaultRoute).filter((key) => key !== "upstream" && key !== "model" && key !== "reasoningEffort"); + if (defaultUnexpected.length > 0) throw new AgentRunError("schema-invalid", `modelRouting.default contains unsupported fields: ${defaultUnexpected.join(", ")}`, { httpStatus: 400, details: { defaultUnexpected, valuesPrinted: false } }); + const defaultUpstream = validateRouteUpstream(requiredString(defaultRoute, "upstream"), upstreams, "modelRouting.default.upstream"); + const defaultModel = validateModelId(defaultRoute.model, "aipodSpec.spec.modelRouting.default.model"); + const defaultReasoningEffort = defaultRoute.reasoningEffort === undefined ? undefined : validateReasoningEffort(defaultRoute.reasoningEffort, "aipodSpec.spec.modelRouting.default.reasoningEffort"); + const exactInput = asRecord(record.exact ?? {}, "aipodSpec.spec.modelRouting.exact"); + const exact: Record = {}; + for (const [rawModel, rawRoute] of Object.entries(exactInput)) { + const model = validateModelId(rawModel, `aipodSpec.spec.modelRouting.exact.${rawModel}`); + const route = asRecord(rawRoute, `aipodSpec.spec.modelRouting.exact.${model}`); + const routeUnexpected = Object.keys(route).filter((key) => key !== "upstream" && key !== "reasoningEffort"); + if (routeUnexpected.length > 0) throw new AgentRunError("schema-invalid", `model route ${model} contains unsupported fields: ${routeUnexpected.join(", ")}`, { httpStatus: 400, details: { routeUnexpected, valuesPrinted: false } }); + const upstream = validateRouteUpstream(requiredString(route, "upstream"), upstreams, `modelRouting.exact.${model}.upstream`); + const reasoningEffort = route.reasoningEffort === undefined ? undefined : validateReasoningEffort(route.reasoningEffort, `aipodSpec.spec.modelRouting.exact.${model}.reasoningEffort`); + exact[model] = { upstream, ...(reasoningEffort ? { reasoningEffort } : {}) }; + } + return { + default: { upstream: defaultUpstream, model: defaultModel, ...(defaultReasoningEffort ? { reasoningEffort: defaultReasoningEffort } : {}) }, + exact, + }; +} + +function validateRouteUpstream(value: string, upstreams: Record, label: string): string { + const upstream = validateUpstreamId(value, label); + if (!upstreams[upstream]) throw new AgentRunError("schema-invalid", `${label} references unknown upstream ${upstream}`, { httpStatus: 400, details: { upstream, available: Object.keys(upstreams).sort(), valuesPrinted: false } }); + return upstream; +} + +function validateUpstreamId(value: string, label: string): string { + const upstream = value.trim().toLowerCase(); + if (!/^[a-z0-9][a-z0-9-]{0,63}$/u.test(upstream)) throw new AgentRunError("schema-invalid", `${label} must be a lowercase slug`, { httpStatus: 400 }); + return upstream; +} + +function executionPolicyWithProviderCredential(executionPolicy: ExecutionPolicy, profile: BackendProfile, secretRef: unknown): ExecutionPolicy { + const validatedSecretRefInput = asRecord(secretRef, `providerCredential.${profile}.secretRef`); + return validateExecutionPolicy({ + ...executionPolicy, + secretScope: { + ...executionPolicy.secretScope, + providerCredentials: [{ profile, secretRef: validatedSecretRefInput }], + }, + }); +} + function validateAipodProviderCredential(profile: BackendProfile, policy: ExecutionPolicy): void { const matching = (policy.secretScope.providerCredentials ?? []).filter((item) => item.profile === profile); if (matching.length !== 1) { @@ -264,6 +392,48 @@ function summarizeProviderCredentials(policy: ExecutionPolicy): JsonRecord { return { count: items.length, profiles: items.map((item) => item.profile), items, valuesPrinted: false }; } +function summarizeAipodProviderCredentials(spec: AipodSpec["spec"]): JsonRecord { + if (!spec.upstreams) return summarizeProviderCredentials(spec.executionPolicy); + const items = Object.entries(spec.upstreams).map(([upstreamId, upstream]) => ({ + upstreamId, + profile: upstream.backendProfile, + name: upstream.providerCredential.secretRef.name, + namespace: upstream.providerCredential.secretRef.namespace ?? null, + keys: upstream.providerCredential.secretRef.keys ?? [], + valuesPrinted: false, + })); + return { count: items.length, profiles: items.map((item) => item.profile), items, valuesPrinted: false }; +} + +function summarizeAipodUpstreams(spec: AipodSpec["spec"]): JsonRecord | null { + if (!spec.upstreams) return null; + const items = Object.entries(spec.upstreams).map(([upstreamId, upstream]) => ({ + upstreamId, + backendProfile: upstream.backendProfile, + providerCredential: { + secretRef: { + name: upstream.providerCredential.secretRef.name, + namespace: upstream.providerCredential.secretRef.namespace ?? null, + keys: upstream.providerCredential.secretRef.keys ?? [], + valuesPrinted: false, + }, + valuesPrinted: false, + }, + valuesPrinted: false, + })); + return { count: items.length, ids: items.map((item) => item.upstreamId), items, valuesPrinted: false }; +} + +function summarizeAipodModelRouting(modelRouting: AipodModelRouting | undefined): JsonRecord | null { + if (!modelRouting) return null; + return { + default: { ...modelRouting.default, valuesPrinted: false }, + exact: Object.entries(modelRouting.exact).map(([model, route]) => ({ model, upstream: route.upstream, reasoningEffort: route.reasoningEffort ?? null, valuesPrinted: false })), + routingSpecHash: stableHash(modelRouting), + valuesPrinted: false, + }; +} + function summarizeToolCredentials(policy: ExecutionPolicy): JsonRecord { const items = (policy.secretScope.toolCredentials ?? []).map((item) => ({ tool: item.tool, purpose: item.purpose ?? null, name: item.secretRef.name, namespace: item.secretRef.namespace ?? null, keys: item.secretRef.keys ?? [], projection: item.projection, valuesPrinted: false })); return { count: items.length, tools: items.map((item) => item.tool), items, valuesPrinted: false }; @@ -343,7 +513,48 @@ function fixedAipodBackendProfile(profile: BackendProfile, input: RenderAipodInp return profile; } -function resolveEffectiveAipodModel(defaults: AipodModelConfig | undefined, input: RenderAipodInput): EffectiveAipodModel { +function resolveEffectiveAipodRuntime(spec: AipodSpec["spec"], input: RenderAipodInput): EffectiveAipodModel { + if (spec.upstreams && spec.modelRouting) { + const modelOverride = renderModelOverride(input, "model", validateModelId); + const reasoningOverride = renderModelOverride(input, "reasoningEffort", validateReasoningEffort); + const requestedModel = modelOverride.provided ? modelOverride.value : spec.modelRouting.default.model; + if (!requestedModel) throw new AgentRunError("schema-invalid", "multi-upstream AipodSpec did not resolve a model", { httpStatus: 400 }); + const exactRoute = modelOverride.provided ? spec.modelRouting.exact[requestedModel] : undefined; + const matchedBy = exactRoute ? "exact" : "default"; + const upstreamId = exactRoute?.upstream ?? spec.modelRouting.default.upstream; + const upstream = spec.upstreams[upstreamId]; + if (!upstream) throw new AgentRunError("schema-invalid", `model route resolved unknown upstream ${upstreamId}`, { httpStatus: 400, details: { upstreamId, valuesPrinted: false } }); + const routeReasoningEffort = exactRoute?.reasoningEffort ?? spec.modelRouting.default.reasoningEffort ?? null; + const reasoningEffort = reasoningOverride.provided ? reasoningOverride.value : routeReasoningEffort; + const executionPolicy = executionPolicyWithProviderCredential(spec.executionPolicy, upstream.backendProfile, upstream.providerCredential.secretRef); + return { + requestedModel, + model: requestedModel, + reasoningEffort, + modelSource: modelOverride.provided ? "render-input-override" : "aipod-spec-default", + reasoningEffortSource: reasoningOverride.provided ? "render-input-override" : reasoningEffort ? "aipod-spec-default" : null, + upstreamId, + matchedBy, + routingSpecHash: stableHash(spec.modelRouting), + backendProfile: upstream.backendProfile, + executionPolicy, + }; + } + const backendProfile = spec.backendProfile; + if (!backendProfile) throw new AgentRunError("schema-invalid", "single-upstream AipodSpec requires backendProfile", { httpStatus: 400 }); + const fixed = resolveFixedAipodModel(spec.model, input); + return { + ...fixed, + requestedModel: fixed.model, + upstreamId: null, + matchedBy: "fixed", + routingSpecHash: null, + backendProfile, + executionPolicy: spec.executionPolicy, + }; +} + +function resolveFixedAipodModel(defaults: AipodModelConfig | undefined, input: RenderAipodInput): Pick { const modelOverride = renderModelOverride(input, "model", validateModelId); const reasoningOverride = renderModelOverride(input, "reasoningEffort", validateReasoningEffort); const defaultModel = defaults?.model ?? null; @@ -387,6 +598,10 @@ function applyEffectiveModelPayload(payload: JsonRecord, effective: EffectiveAip reasoningEffort: effective.reasoningEffort, modelSource: effective.modelSource, reasoningEffortSource: effective.reasoningEffortSource, + requestedModel: effective.requestedModel, + upstreamId: effective.upstreamId, + matchedBy: effective.matchedBy, + routingSpecHash: effective.routingSpecHash, valuesPrinted: false, }; } diff --git a/src/common/types.ts b/src/common/types.ts index 3042385..731be30 100644 --- a/src/common/types.ts +++ b/src/common/types.ts @@ -118,8 +118,10 @@ export interface AipodSpec extends JsonRecord { lane?: string; priority?: number; providerId?: string; - backendProfile: BackendProfile; + backendProfile?: BackendProfile; model?: AipodModelConfig; + upstreams?: Record; + modelRouting?: AipodModelRouting; imageRef: AipodImageRef; workspaceRef?: WorkspaceRef; sessionRef?: SessionRef | null; @@ -137,6 +139,27 @@ export interface AipodModelConfig extends JsonRecord { reasoningEffort?: string; } +export interface AipodUpstreamConfig extends JsonRecord { + backendProfile: BackendProfile; + providerCredential: { + secretRef: SecretRef; + }; +} + +export interface AipodModelRoute extends JsonRecord { + upstream: string; + reasoningEffort?: string; +} + +export interface AipodModelRouting extends JsonRecord { + default: { + upstream: string; + model: string; + reasoningEffort?: string; + }; + exact: Record; +} + export interface AipodSpecRecord extends JsonRecord { name: string; spec: AipodSpec; diff --git a/src/selftest/cases/76-aipod-spec.ts b/src/selftest/cases/76-aipod-spec.ts index 3fad0fa..51ae72f 100644 --- a/src/selftest/cases/76-aipod-spec.ts +++ b/src/selftest/cases/76-aipod-spec.ts @@ -68,9 +68,16 @@ const selfTest: SelfTestCase = async (context) => { assert.equal(shownModelDefaults.modelSource, "aipod-spec-default"); assert.equal(shownModelDefaults.reasoningEffortSource, "aipod-spec-default"); const shownProviderCredentials = shownItem.providerCredentials as JsonRecord; + assert.equal(shownProviderCredentials.count, 2); const shownProviderCredential = (shownProviderCredentials.items as JsonRecord[]).find((item) => item.profile === "gpt-pika"); assert.equal(shownProviderCredential?.name, "agentrun-v02-provider-gpt-pika"); assert.equal(shownProviderCredential?.namespace, "agentrun-v02"); + const shownGrokCredential = (shownProviderCredentials.items as JsonRecord[]).find((item) => item.profile === "grok"); + assert.equal(shownGrokCredential?.name, "agentrun-v02-provider-grok"); + assert.equal(shownGrokCredential?.namespace, "agentrun-v02"); + const shownRouting = shownItem.modelRouting as JsonRecord; + assert.deepEqual(shownRouting.default, { upstream: "pika", model: "gpt-5.6-sol", reasoningEffort: "medium", valuesPrinted: false }); + assert.equal((shownRouting.exact as JsonRecord[]).some((item) => item.model === "grok" && item.upstream === "grok" && item.reasoningEffort === "xhigh"), true); const shownToolCredentials = shownItem.toolCredentials as JsonRecord; assert.equal(shownToolCredentials.count, 4); assert.equal(shownToolCredentials.valuesPrinted, false); @@ -121,7 +128,10 @@ const selfTest: SelfTestCase = async (context) => { assert.equal(modelResolution.modelSource, "aipod-spec-default"); assert.equal(modelResolution.reasoningEffortSource, "aipod-spec-default"); assert.equal(modelResolution.backendProfile, "gpt-pika"); - assert.equal(modelResolution.backendProfileSource, "aipod-spec"); + assert.equal(modelResolution.upstreamId, "pika"); + assert.equal(modelResolution.matchedBy, "default"); + assert.equal(modelResolution.backendProfileSource, "model-routing-default"); + assert.match(String(modelResolution.routingSpecHash), /^[a-f0-9]{64}$/u); assert.equal(modelResolution.valuesPrinted, false); const policy = task.executionPolicy as JsonRecord; assert.equal(policy.timeoutMs, 7_200_000, "Artificer timeout budget must come from the owning AipodSpec YAML"); @@ -211,9 +221,28 @@ const selfTest: SelfTestCase = async (context) => { assert.equal(modelOverridePayload.reasoningEffort, "high"); assert.equal(modelOverrideResolution.modelSource, "render-input-override"); assert.equal(modelOverrideResolution.reasoningEffortSource, "render-input-override"); + assert.equal(modelOverrideResolution.upstreamId, "pika"); + assert.equal(modelOverrideResolution.matchedBy, "default"); assert.deepEqual((modelOverrideResolution.providerCredential as JsonRecord).secretRef, disclosedProviderCredential.secretRef); assert.deepEqual(((modelOverrideTask.executionPolicy as JsonRecord).secretScope as JsonRecord).providerCredentials, providerCredentials); + const grokOverride = await client.post("/api/v1/aipod-specs/Artificer/render", { prompt: "route to grok", model: "grok" }) as JsonRecord; + const grokTask = grokOverride.queueTask as JsonRecord; + const grokPayload = grokTask.payload as JsonRecord; + const grokResolution = grokOverride.modelResolution as JsonRecord; + const grokCredentials = (((grokTask.executionPolicy as JsonRecord).secretScope as JsonRecord).providerCredentials as JsonRecord[]); + assert.equal(grokTask.backendProfile, "grok"); + assert.equal(grokPayload.model, "grok"); + assert.equal(grokPayload.reasoningEffort, "xhigh"); + assert.equal(grokResolution.upstreamId, "grok"); + assert.equal(grokResolution.matchedBy, "exact"); + assert.equal(grokResolution.backendProfileSource, "model-routing-exact"); + assert.equal(grokCredentials.length, 1); + assert.equal(grokCredentials[0]?.profile, "grok"); + assert.equal(((grokCredentials[0]?.secretRef as JsonRecord).name), "agentrun-v02-provider-grok"); + assert.equal(grokCredentials.some((item) => item.profile === "gpt-pika"), false); + assertNoSecretLeak(grokOverride); + const reasoningOverride = await client.post("/api/v1/aipod-specs/Artificer/render", { prompt: "override reasoning", reasoningEffort: "high" }) as JsonRecord; const reasoningOverridePayload = (reasoningOverride.queueTask as JsonRecord).payload as JsonRecord; const reasoningOverrideResolution = reasoningOverride.modelResolution as JsonRecord; @@ -235,9 +264,35 @@ const selfTest: SelfTestCase = async (context) => { const originalSpecDocument = parseYaml(await readFile(artificerSpecFile, "utf8")) as JsonRecord; const missingCredentialDocument = structuredClone(originalSpecDocument); - const missingCredentialPolicy = ((missingCredentialDocument.spec as JsonRecord).executionPolicy as JsonRecord); - ((missingCredentialPolicy.secretScope as JsonRecord).providerCredentials) = []; - assert.throws(() => validateAipodSpec(missingCredentialDocument, "selftest-missing-gpt-pika-credential"), /requires exactly one matching provider credential/u); + delete (((missingCredentialDocument.spec as JsonRecord).upstreams as JsonRecord).grok as JsonRecord).providerCredential; + assert.throws(() => validateAipodSpec(missingCredentialDocument, "selftest-missing-grok-credential"), /providerCredential/u); + const missingKeyDocument = structuredClone(originalSpecDocument); + ((((missingKeyDocument.spec as JsonRecord).upstreams as JsonRecord).grok as JsonRecord).providerCredential as JsonRecord).secretRef = { + namespace: "agentrun-v02", + name: "agentrun-v02-provider-grok", + keys: ["auth.json"], + }; + assert.throws(() => validateAipodSpec(missingKeyDocument, "selftest-missing-grok-key"), /missing required keys/u); + const unknownRouteDocument = structuredClone(originalSpecDocument); + ((((unknownRouteDocument.spec as JsonRecord).modelRouting as JsonRecord).exact as JsonRecord).grok as JsonRecord).upstream = "missing"; + assert.throws(() => validateAipodSpec(unknownRouteDocument, "selftest-unknown-upstream-route"), /references unknown upstream missing/u); + const duplicateProfileDocument = structuredClone(originalSpecDocument); + ((((duplicateProfileDocument.spec as JsonRecord).upstreams as JsonRecord).grok as JsonRecord).backendProfile) = "gpt-pika"; + assert.throws(() => validateAipodSpec(duplicateProfileDocument, "selftest-duplicate-upstream-profile"), /declared more than once/u); + const mixedModeDocument = structuredClone(originalSpecDocument); + ((mixedModeDocument.spec as JsonRecord).backendProfile) = "gpt-pika"; + assert.throws(() => validateAipodSpec(mixedModeDocument, "selftest-mixed-upstream-mode"), /cannot also declare backendProfile or model/u); + const legacySingleUpstreamDocument = structuredClone(originalSpecDocument); + const legacySpec = legacySingleUpstreamDocument.spec as JsonRecord; + delete legacySpec.upstreams; + delete legacySpec.modelRouting; + legacySpec.backendProfile = "gpt-pika"; + legacySpec.model = { model: "gpt-5.6-sol", reasoningEffort: "medium" }; + (((legacySpec.executionPolicy as JsonRecord).secretScope as JsonRecord).providerCredentials) = [{ + profile: "gpt-pika", + secretRef: { namespace: "agentrun-v02", name: "agentrun-v02-provider-gpt-pika", keys: ["auth.json", "config.toml"] }, + }]; + assert.equal(validateAipodSpec(legacySingleUpstreamDocument, "selftest-legacy-single-upstream").spec.backendProfile, "gpt-pika"); const payloadDefaultsBypassDocument = structuredClone(originalSpecDocument); ((payloadDefaultsBypassDocument.spec as JsonRecord).payloadDefaults as JsonRecord).modelConfig = { model: "bypass" }; assert.throws(() => validateAipodSpec(payloadDefaultsBypassDocument, "selftest-payload-defaults-bypass"), /cannot set Aipod control fields: modelConfig/u); @@ -288,6 +343,13 @@ const selfTest: SelfTestCase = async (context) => { assert.equal((sendPlanData.modelResolution as JsonRecord).reasoningEffort, "medium"); assert.equal((((sendPlanData.modelResolution as JsonRecord).providerCredential as JsonRecord).secretRef as JsonRecord).name, "agentrun-v02-provider-gpt-pika"); + const grokSendPlan = await runCliJson(context, server.baseUrl, ["sessions", "send", "--aipod", "Artificer", "--prompt", "dry-run grok session", "--model", "grok", "--no-runner-job", "--dry-run"]); + const grokSendPlanData = grokSendPlan.data as JsonRecord; + assert.equal(grokSendPlanData.profile, "grok"); + assert.equal((grokSendPlanData.modelResolution as JsonRecord).upstreamId, "grok"); + assert.equal((grokSendPlanData.modelResolution as JsonRecord).matchedBy, "exact"); + assert.equal((((grokSendPlanData.modelResolution as JsonRecord).providerCredential as JsonRecord).secretRef as JsonRecord).name, "agentrun-v02-provider-grok"); + const cancelledSessionId = "sess_artificer_cancelled_cli_selftest"; const cancelledConversationId = "conv_artificer_cancelled_cli_selftest"; const cancelledThreadId = "thread_artificer_cancelled_cli_selftest"; @@ -344,8 +406,8 @@ const selfTest: SelfTestCase = async (context) => { const durableTask = await client.post("/api/v1/queue/tasks", rendered.queueTask) as JsonRecord; const changedSpecDocument = structuredClone(originalSpecDocument); - ((changedSpecDocument.spec as JsonRecord).model as JsonRecord).model = "gpt-5.7-terra"; - ((changedSpecDocument.spec as JsonRecord).model as JsonRecord).reasoningEffort = "low"; + (((changedSpecDocument.spec as JsonRecord).modelRouting as JsonRecord).default as JsonRecord).model = "gpt-5.7-terra"; + (((changedSpecDocument.spec as JsonRecord).modelRouting as JsonRecord).default as JsonRecord).reasoningEffort = "low"; await writeFile(artificerSpecFile, stringifyYaml(changedSpecDocument), "utf8"); const changedRender = await client.post("/api/v1/aipod-specs/Artificer/render", { prompt: "new yaml default" }) as JsonRecord; assert.equal(((changedRender.queueTask as JsonRecord).payload as JsonRecord).model, "gpt-5.7-terra"); @@ -353,7 +415,7 @@ const selfTest: SelfTestCase = async (context) => { assert.equal((storedTask.payload as JsonRecord).model, "gpt-5.6-sol"); assert.equal((storedTask.payload as JsonRecord).reasoningEffort, "medium"); assert.deepEqual((storedTask.payload as JsonRecord).modelConfig, taskPayload.modelConfig); - return { name: "aipod-spec", tests: ["aipod-spec-yaml-parser-runtime-compatible", "aipod-spec-artificer-image-ref-render", "aipod-spec-artificer-v02-runtime-authority", "aipod-spec-gpt-pika-secret-namespace", "aipod-spec-model-defaults", "aipod-spec-model-reasoning-overrides", "aipod-spec-model-override-keeps-provider", "aipod-spec-control-override-fail-closed", "aipod-spec-payload-control-bypass-fail-closed", "aipod-spec-missing-gpt-pika-credential", "aipod-spec-durable-task-model-snapshot", "aipod-spec-primary-workspace-contract", "aipod-spec-artificer-github-url-render", "aipod-spec-artificer-github-ssh-required-keys", "aipod-spec-artificer-private-repository-override", "aipod-spec-artificer-private-repository-override-runner-projection", "aipod-spec-git-mirror-url", "git-fetch-stderr-classification", "resource-bundle-source-authority-validation", "resource-bundle-target-conflict-validation", "queue-submit-aipod-dry-run", "session-send-aipod-model-dry-run", "session-send-aipod-cancelled-durable-identity", "session-send-aipod-boundary-conflict-fail-closed", "aipod-cli-help"] }; + return { name: "aipod-spec", tests: ["aipod-spec-yaml-parser-runtime-compatible", "aipod-spec-artificer-image-ref-render", "aipod-spec-artificer-v02-runtime-authority", "aipod-spec-multi-upstream-secret-summary", "aipod-spec-legacy-single-upstream-compatible", "aipod-spec-model-defaults", "aipod-spec-model-reasoning-overrides", "aipod-spec-model-default-fallback", "aipod-spec-grok-exact-route", "aipod-spec-selected-provider-only", "aipod-spec-invalid-route-fail-closed", "aipod-spec-control-override-fail-closed", "aipod-spec-payload-control-bypass-fail-closed", "aipod-spec-durable-task-model-snapshot", "aipod-spec-primary-workspace-contract", "aipod-spec-artificer-github-url-render", "aipod-spec-artificer-github-ssh-required-keys", "aipod-spec-artificer-private-repository-override", "aipod-spec-artificer-private-repository-override-runner-projection", "aipod-spec-git-mirror-url", "git-fetch-stderr-classification", "resource-bundle-source-authority-validation", "resource-bundle-target-conflict-validation", "queue-submit-aipod-dry-run", "session-send-aipod-model-dry-run", "session-send-aipod-grok-dry-run", "session-send-aipod-cancelled-durable-identity", "session-send-aipod-boundary-conflict-fail-closed", "aipod-cli-help"] }; } finally { await new Promise((resolve) => server.server.close(() => resolve())); } diff --git a/src/selftest/cases/80-artificer-model-selection.ts b/src/selftest/cases/80-artificer-model-selection.ts index 2c1d0e8..4c46041 100644 --- a/src/selftest/cases/80-artificer-model-selection.ts +++ b/src/selftest/cases/80-artificer-model-selection.ts @@ -35,7 +35,7 @@ const selfTest: SelfTestCase = async (context) => { assert.equal(task.providerId, "NC01"); assert.equal(modelResolution.model, "gpt-5.6-sol"); assert.equal(modelResolution.reasoningEffort, "medium"); - await assertDurableRetryModel(context.tmp, task as unknown as CreateQueueTaskInput); + await assertDurableRetryModel(context.tmp, task as unknown as CreateQueueTaskInput, { profile: "gpt-pika", model: "gpt-5.6-sol", reasoningEffort: "medium", matchedBy: "default" }); const runtimeExecutionPolicy = structuredClone(task.executionPolicy as JsonRecord); ((runtimeExecutionPolicy.secretScope as JsonRecord).toolCredentials) = []; @@ -83,14 +83,64 @@ const selfTest: SelfTestCase = async (context) => { assert.equal(turnStart?.payload.model, "gpt-5.6-sol"); assert.equal(turnStart?.payload.reasoningEffort, "medium"); assert.equal(turnStart?.payload.reasoningProtocolField, "effort"); - assertNoSecretLeak({ rendered, result, events }); - return { name: "artificer-model-selection", tests: ["artificer-gpt-pika-default-render", "artificer-durable-retry-model", "artificer-gpt-pika-model-runtime", "artificer-reasoning-effort-turn-start", "artificer-model-selection-visible"] }; + + const grokRendered = await client.post("/api/v1/aipod-specs/Artificer/render", { prompt: "verify grok model selection", model: "grok" }) as JsonRecord; + const grokTask = grokRendered.queueTask as JsonRecord; + const grokResolution = grokRendered.modelResolution as JsonRecord; + assert.equal(grokTask.backendProfile, "grok"); + assert.equal(grokResolution.model, "grok"); + assert.equal(grokResolution.reasoningEffort, "xhigh"); + assert.equal(grokResolution.upstreamId, "grok"); + assert.equal(grokResolution.matchedBy, "exact"); + await assertDurableRetryModel(context.tmp, grokTask as unknown as CreateQueueTaskInput, { profile: "grok", model: "grok", reasoningEffort: "xhigh", matchedBy: "exact" }); + const grokRuntimePolicy = structuredClone(grokTask.executionPolicy as JsonRecord); + ((grokRuntimePolicy.secretScope as JsonRecord).toolCredentials) = []; + const grokRun = await client.post("/api/v1/runs", { + tenantId: grokTask.tenantId, + projectId: grokTask.projectId, + providerId: grokTask.providerId, + backendProfile: grokTask.backendProfile, + workspaceRef: { kind: "host-path", path: context.workspace }, + sessionRef: null, + executionPolicy: grokRuntimePolicy, + resourceBundleRef: null, + traceSink: { kind: "self-test", source: "artificer-grok-model-selection" }, + }) as JsonRecord; + const grokCommand = await client.post(`/api/v1/runs/${String(grokRun.id)}/commands`, { + type: "turn", + payload: grokTask.payload, + idempotencyKey: "selftest-artificer-grok-model-selection", + }) as JsonRecord; + const grokResult = await runOnce({ + managerUrl: server.baseUrl, + runId: String(grokRun.id), + commandId: String(grokCommand.id), + backendProfile: "grok", + codexCommand: context.fakeCodexCommand, + codexArgs: context.fakeCodexArgs, + codexHome: context.codexHome, + env: { + CODEX_HOME: context.codexHome, + AGENTRUN_FAKE_CODEX_MODE: "require-explicit-model-effort", + AGENTRUN_FAKE_CODEX_EXPECTED_MODEL: "grok", + AGENTRUN_FAKE_CODEX_EXPECTED_REASONING_EFFORT: "xhigh", + }, + oneShot: true, + }); + assert.equal(grokResult.terminalStatus, "completed"); + const grokEvents = await client.get(`/api/v1/runs/${String(grokRun.id)}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: JsonRecord }> }; + const grokStarting = grokEvents.items?.find((event) => event.type === "backend_status" && event.payload.phase === "codex-app-server-starting"); + assert.equal(grokStarting?.payload.backendProfile, "grok"); + assert.equal((grokStarting?.payload.modelSelection as JsonRecord).model, "grok"); + assert.equal((grokStarting?.payload.modelSelection as JsonRecord).reasoningEffort, "xhigh"); + assertNoSecretLeak({ rendered, result, events, grokRendered, grokResult, grokEvents }); + return { name: "artificer-model-selection", tests: ["artificer-gpt-pika-default-render", "artificer-grok-exact-render", "artificer-durable-retry-model", "artificer-durable-retry-upstream", "artificer-gpt-pika-model-runtime", "artificer-grok-model-runtime", "artificer-reasoning-effort-turn-start", "artificer-model-selection-visible"] }; } finally { await new Promise((resolve) => server.server.close(() => resolve())); } }; -async function assertDurableRetryModel(tmp: string, input: CreateQueueTaskInput): Promise { +async function assertDurableRetryModel(tmp: string, input: CreateQueueTaskInput, expected: { profile: string; model: string; reasoningEffort: string; matchedBy: string }): Promise { const store = new MemoryAgentRunStore(); const retryInput = { ...structuredClone(input), sessionRef: null }; assert.ok(retryInput.executionPolicy); @@ -139,9 +189,11 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin }); const command = retried.command as JsonRecord; assert.deepEqual(command.payload, task.payload); - assert.equal((command.payload as JsonRecord).model, "gpt-5.6-sol"); - assert.equal((command.payload as JsonRecord).reasoningEffort, "medium"); - assert.equal(((command.payload as JsonRecord).modelConfig as JsonRecord).modelSource, "aipod-spec-default"); + assert.ok(retried.run); + assert.equal(retried.run.backendProfile, expected.profile); + assert.equal((command.payload as JsonRecord).model, expected.model); + assert.equal((command.payload as JsonRecord).reasoningEffort, expected.reasoningEffort); + assert.equal(((command.payload as JsonRecord).modelConfig as JsonRecord).matchedBy, expected.matchedBy); } export default selfTest;