Compare commits

...

7 Commits

Author SHA1 Message Date
joshavant
ed402b6944 Gateway: add manual secrets reload command 2026-02-21 17:17:52 -08:00
joshavant
6eb674d6d0 Auth profiles: never persist plaintext when refs are present 2026-02-21 17:17:52 -08:00
joshavant
16b7d4bbed Agents: inject pi auth storage from runtime profiles 2026-02-21 17:17:52 -08:00
joshavant
209287d5e0 Secrets runtime: include sourceConfig in prepared snapshot type 2026-02-21 17:17:50 -08:00
joshavant
bb052deec6 Secrets: preserve runtime snapshot source refs on write 2026-02-21 17:09:04 -08:00
joshavant
06e51505bc Gateway: avoid double secrets activation at startup 2026-02-21 16:04:15 -08:00
joshavant
2b7206470a Gateway: add eager secrets runtime snapshot activation 2026-02-21 16:04:15 -08:00
35 changed files with 1573 additions and 63 deletions

View File

@@ -0,0 +1,71 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
activateSecretsRuntimeSnapshot,
clearSecretsRuntimeSnapshot,
prepareSecretsRuntimeSnapshot,
} from "../secrets/runtime.js";
import { ensureAuthProfileStore, markAuthProfileUsed } from "./auth-profiles.js";
describe("auth profile runtime snapshot persistence", () => {
it("does not write resolved plaintext keys during usage updates", async () => {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-runtime-save-"));
const agentDir = path.join(stateDir, "agents", "main", "agent");
const authPath = path.join(agentDir, "auth-profiles.json");
try {
await fs.mkdir(agentDir, { recursive: true });
await fs.writeFile(
authPath,
`${JSON.stringify(
{
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
keyRef: { source: "env", id: "OPENAI_API_KEY" },
},
},
},
null,
2,
)}\n`,
"utf8",
);
const snapshot = await prepareSecretsRuntimeSnapshot({
config: {},
env: { OPENAI_API_KEY: "sk-runtime-openai" },
agentDirs: [agentDir],
});
activateSecretsRuntimeSnapshot(snapshot);
const runtimeStore = ensureAuthProfileStore(agentDir);
expect(runtimeStore.profiles["openai:default"]).toMatchObject({
type: "api_key",
key: "sk-runtime-openai",
keyRef: { source: "env", id: "OPENAI_API_KEY" },
});
await markAuthProfileUsed({
store: runtimeStore,
profileId: "openai:default",
agentDir,
});
const persisted = JSON.parse(await fs.readFile(authPath, "utf8")) as {
profiles: Record<string, { key?: string; keyRef?: unknown }>;
};
expect(persisted.profiles["openai:default"]?.key).toBeUndefined();
expect(persisted.profiles["openai:default"]?.keyRef).toEqual({
source: "env",
id: "OPENAI_API_KEY",
});
} finally {
clearSecretsRuntimeSnapshot();
await fs.rm(stateDir, { recursive: true, force: true });
}
});
});

View File

@@ -0,0 +1,62 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { resolveAuthStorePath } from "./auth-profiles/paths.js";
import { saveAuthProfileStore } from "./auth-profiles/store.js";
import type { AuthProfileStore } from "./auth-profiles/types.js";
describe("saveAuthProfileStore", () => {
it("strips plaintext when keyRef/tokenRef are present", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-"));
try {
const store: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "sk-runtime-value",
keyRef: { source: "env", id: "OPENAI_API_KEY" },
},
"github-copilot:default": {
type: "token",
provider: "github-copilot",
token: "gh-runtime-token",
tokenRef: { source: "env", id: "GITHUB_TOKEN" },
},
"anthropic:default": {
type: "api_key",
provider: "anthropic",
key: "sk-anthropic-plain",
},
},
};
saveAuthProfileStore(store, agentDir);
const parsed = JSON.parse(await fs.readFile(resolveAuthStorePath(agentDir), "utf8")) as {
profiles: Record<
string,
{ key?: string; keyRef?: unknown; token?: string; tokenRef?: unknown }
>;
};
expect(parsed.profiles["openai:default"]?.key).toBeUndefined();
expect(parsed.profiles["openai:default"]?.keyRef).toEqual({
source: "env",
id: "OPENAI_API_KEY",
});
expect(parsed.profiles["github-copilot:default"]?.token).toBeUndefined();
expect(parsed.profiles["github-copilot:default"]?.tokenRef).toEqual({
source: "env",
id: "GITHUB_TOKEN",
});
expect(parsed.profiles["anthropic:default"]?.key).toBe("sk-anthropic-plain");
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
});

View File

@@ -17,7 +17,10 @@ export {
suggestOAuthProfileIdForLegacyDefault,
} from "./auth-profiles/repair.js";
export {
clearRuntimeAuthProfileStoreSnapshots,
ensureAuthProfileStore,
loadAuthProfileStoreForRuntime,
replaceRuntimeAuthProfileStoreSnapshots,
loadAuthProfileStore,
saveAuthProfileStore,
} from "./auth-profiles/store.js";

View File

@@ -18,6 +18,65 @@ function _syncAuthProfileStore(target: AuthProfileStore, source: AuthProfileStor
target.usageStats = source.usageStats;
}
const runtimeAuthStoreSnapshots = new Map<string, AuthProfileStore>();
function resolveRuntimeStoreKey(agentDir?: string): string {
return resolveAuthStorePath(agentDir);
}
function cloneAuthProfileStore(store: AuthProfileStore): AuthProfileStore {
return structuredClone(store);
}
function resolveRuntimeAuthProfileStore(agentDir?: string): AuthProfileStore | null {
if (runtimeAuthStoreSnapshots.size === 0) {
return null;
}
const mainKey = resolveRuntimeStoreKey(undefined);
const requestedKey = resolveRuntimeStoreKey(agentDir);
const mainStore = runtimeAuthStoreSnapshots.get(mainKey);
const requestedStore = runtimeAuthStoreSnapshots.get(requestedKey);
if (!agentDir || requestedKey === mainKey) {
if (!mainStore) {
return null;
}
return cloneAuthProfileStore(mainStore);
}
if (mainStore && requestedStore) {
return mergeAuthProfileStores(
cloneAuthProfileStore(mainStore),
cloneAuthProfileStore(requestedStore),
);
}
if (requestedStore) {
return cloneAuthProfileStore(requestedStore);
}
if (mainStore) {
return cloneAuthProfileStore(mainStore);
}
return null;
}
export function replaceRuntimeAuthProfileStoreSnapshots(
entries: Array<{ agentDir?: string; store: AuthProfileStore }>,
): void {
runtimeAuthStoreSnapshots.clear();
for (const entry of entries) {
runtimeAuthStoreSnapshots.set(
resolveRuntimeStoreKey(entry.agentDir),
cloneAuthProfileStore(entry.store),
);
}
}
export function clearRuntimeAuthProfileStoreSnapshots(): void {
runtimeAuthStoreSnapshots.clear();
}
export async function updateAuthProfileStoreWithLock(params: {
agentDir?: string;
updater: (store: AuthProfileStore) => boolean;
@@ -316,10 +375,30 @@ function loadAuthProfileStoreForAgent(
return store;
}
export function loadAuthProfileStoreForRuntime(
agentDir?: string,
options?: { allowKeychainPrompt?: boolean },
): AuthProfileStore {
const store = loadAuthProfileStoreForAgent(agentDir, options);
const authPath = resolveAuthStorePath(agentDir);
const mainAuthPath = resolveAuthStorePath();
if (!agentDir || authPath === mainAuthPath) {
return store;
}
const mainStore = loadAuthProfileStoreForAgent(undefined, options);
return mergeAuthProfileStores(mainStore, store);
}
export function ensureAuthProfileStore(
agentDir?: string,
options?: { allowKeychainPrompt?: boolean },
): AuthProfileStore {
const runtimeStore = resolveRuntimeAuthProfileStore(agentDir);
if (runtimeStore) {
return runtimeStore;
}
const store = loadAuthProfileStoreForAgent(agentDir, options);
const authPath = resolveAuthStorePath(agentDir);
const mainAuthPath = resolveAuthStorePath();
@@ -335,9 +414,24 @@ export function ensureAuthProfileStore(
export function saveAuthProfileStore(store: AuthProfileStore, agentDir?: string): void {
const authPath = resolveAuthStorePath(agentDir);
const profiles = Object.fromEntries(
Object.entries(store.profiles).map(([profileId, credential]) => {
if (credential.type === "api_key" && credential.keyRef && credential.key !== undefined) {
const sanitized = { ...credential } as Record<string, unknown>;
delete sanitized.key;
return [profileId, sanitized];
}
if (credential.type === "token" && credential.tokenRef && credential.token !== undefined) {
const sanitized = { ...credential } as Record<string, unknown>;
delete sanitized.token;
return [profileId, sanitized];
}
return [profileId, credential];
}),
) as AuthProfileStore["profiles"];
const payload = {
version: AUTH_STORE_VERSION,
profiles: store.profiles,
profiles,
order: store.order ?? undefined,
lastGood: store.lastGood ?? undefined,
usageStats: store.usageStats ?? undefined,

View File

@@ -1,10 +1,12 @@
import type { OAuthCredentials } from "@mariozechner/pi-ai";
import type { OpenClawConfig } from "../../config/config.js";
import type { SecretRef } from "../../config/types.secrets.js";
export type ApiKeyCredential = {
type: "api_key";
provider: string;
key?: string;
keyRef?: SecretRef;
email?: string;
/** Optional provider-specific metadata (e.g., account IDs, gateway IDs). */
metadata?: Record<string, string>;
@@ -18,6 +20,7 @@ export type TokenCredential = {
type: "token";
provider: string;
token: string;
tokenRef?: SecretRef;
/** Optional expiry timestamp (ms since epoch). */
expires?: number;
email?: string;

View File

@@ -31,6 +31,7 @@ export function mockCatalogImportFailThenRecover() {
throw new Error("boom");
}
return {
discoverAuthStorage: () => ({}),
AuthStorage: class {},
ModelRegistry: class {
getAll() {

View File

@@ -38,6 +38,7 @@ describe("loadModelCatalog", () => {
__setModelCatalogImportForTest(
async () =>
({
discoverAuthStorage: () => ({}),
AuthStorage: class {},
ModelRegistry: class {
getAll() {
@@ -69,6 +70,7 @@ describe("loadModelCatalog", () => {
__setModelCatalogImportForTest(
async () =>
({
discoverAuthStorage: () => ({}),
AuthStorage: class {},
ModelRegistry: class {
getAll() {

View File

@@ -70,14 +70,6 @@ export function __setModelCatalogImportForTest(loader?: () => Promise<PiSdkModul
importPiSdk = loader ?? defaultImportPiSdk;
}
function createAuthStorage(AuthStorageLike: unknown, path: string) {
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
if (typeof withFactory.create === "function") {
return withFactory.create(path);
}
return new (AuthStorageLike as { new (path: string): unknown })(path);
}
export async function loadModelCatalog(params?: {
config?: OpenClawConfig;
useCache?: boolean;
@@ -102,9 +94,6 @@ export async function loadModelCatalog(params?: {
try {
const cfg = params?.config ?? loadConfig();
await ensureOpenClawModelsJson(cfg);
await (
await import("./pi-auth-json.js")
).ensurePiAuthJsonFromAuthProfiles(resolveOpenClawAgentDir());
// IMPORTANT: keep the dynamic import *inside* the try/catch.
// If this fails once (e.g. during a pnpm install that temporarily swaps node_modules),
// we must not poison the cache with a rejected promise (otherwise all channel handlers
@@ -112,7 +101,7 @@ export async function loadModelCatalog(params?: {
const piSdk = await importPiSdk();
const agentDir = resolveOpenClawAgentDir();
const { join } = await import("node:path");
const authStorage = createAuthStorage(piSdk.AuthStorage, join(agentDir, "auth.json"));
const authStorage = piSdk.discoverAuthStorage(agentDir);
const registry = new (piSdk.ModelRegistry as unknown as {
new (
authStorage: unknown,

View File

@@ -5,6 +5,7 @@ import {
DEFAULT_COPILOT_API_BASE_URL,
resolveCopilotApiToken,
} from "../providers/github-copilot-token.js";
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
import { ensureAuthProfileStore, listProfilesForProvider } from "./auth-profiles.js";
import { discoverBedrockModels } from "./bedrock-discovery.js";
import {
@@ -387,16 +388,17 @@ export function normalizeProviders(params: {
for (const [key, provider] of Object.entries(providers)) {
const normalizedKey = key.trim();
let normalizedProvider = provider;
const configuredApiKey = normalizedProvider.apiKey;
// Fix common misconfig: apiKey set to "${ENV_VAR}" instead of "ENV_VAR".
if (
normalizedProvider.apiKey &&
normalizeApiKeyConfig(normalizedProvider.apiKey) !== normalizedProvider.apiKey
typeof configuredApiKey === "string" &&
normalizeApiKeyConfig(configuredApiKey) !== configuredApiKey
) {
mutated = true;
normalizedProvider = {
...normalizedProvider,
apiKey: normalizeApiKeyConfig(normalizedProvider.apiKey),
apiKey: normalizeApiKeyConfig(configuredApiKey),
};
}
@@ -404,7 +406,9 @@ export function normalizeProviders(params: {
// Fill it from the environment or auth profiles when possible.
const hasModels =
Array.isArray(normalizedProvider.models) && normalizedProvider.models.length > 0;
if (hasModels && !normalizedProvider.apiKey?.trim()) {
const normalizedApiKey = normalizeOptionalSecretInput(normalizedProvider.apiKey);
const hasConfiguredApiKey = Boolean(normalizedApiKey || normalizedProvider.apiKey);
if (hasModels && !hasConfiguredApiKey) {
const authMode =
normalizedProvider.auth ?? (normalizedKey === "amazon-bedrock" ? "aws-sdk" : undefined);
if (authMode === "aws-sdk") {
@@ -957,17 +961,8 @@ export async function resolveImplicitCopilotProvider(params: {
}
}
// pi-coding-agent's ModelRegistry marks a model "available" only if its
// `AuthStorage` has auth configured for that provider (via auth.json/env/etc).
// Our Copilot auth lives in OpenClaw's auth-profiles store instead, so we also
// write a runtime-only auth.json entry for pi-coding-agent to pick up.
//
// This is safe because it's (1) within OpenClaw's agent dir, (2) contains the
// GitHub token (not the exchanged Copilot token), and (3) matches existing
// patterns for OAuth-like providers in pi-coding-agent.
// Note: we deliberately do not write pi-coding-agent's `auth.json` here.
// OpenClaw uses its own auth store and exchanges tokens at runtime.
// `models list` uses OpenClaw's auth heuristics for availability.
// We deliberately do not write pi-coding-agent auth.json here.
// OpenClaw keeps auth in auth-profiles and resolves runtime availability from that store.
// We intentionally do NOT define custom models for Copilot in models.json.
// pi-coding-agent treats providers with models as replacements requiring apiKey.

View File

@@ -4,6 +4,10 @@ import { ensureAuthProfileStore } from "./auth-profiles.js";
import type { AuthProfileCredential } from "./auth-profiles/types.js";
import { normalizeProviderId } from "./model-selection.js";
/**
* @deprecated Legacy bridge for older flows that still expect `agentDir/auth.json`.
* Runtime auth resolution uses auth-profiles directly and should not depend on this module.
*/
type AuthJsonCredential =
| {
type: "api_key";
@@ -110,6 +114,8 @@ function credentialsEqual(a: AuthJsonCredential | undefined, b: AuthJsonCredenti
* registry/catalog output.
*
* Syncs all credential types: api_key, token (as api_key), and oauth.
*
* @deprecated Runtime auth now comes from OpenClaw auth-profiles snapshots.
*/
export async function ensurePiAuthJsonFromAuthProfiles(agentDir: string): Promise<{
wrote: boolean;

View File

@@ -0,0 +1,68 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { saveAuthProfileStore } from "./auth-profiles.js";
import { discoverAuthStorage } from "./pi-model-discovery.js";
async function createAgentDir(): Promise<string> {
return await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-pi-auth-storage-"));
}
async function pathExists(pathname: string): Promise<boolean> {
try {
await fs.stat(pathname);
return true;
} catch {
return false;
}
}
describe("discoverAuthStorage", () => {
it("loads runtime credentials from auth-profiles without writing auth.json", async () => {
const agentDir = await createAgentDir();
try {
saveAuthProfileStore(
{
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
"anthropic:default": {
type: "token",
provider: "anthropic",
token: "sk-ant-runtime",
},
"openai-codex:default": {
type: "oauth",
provider: "openai-codex",
access: "oauth-access",
refresh: "oauth-refresh",
expires: Date.now() + 60_000,
},
},
},
agentDir,
);
const authStorage = discoverAuthStorage(agentDir);
expect(authStorage.hasAuth("openrouter")).toBe(true);
expect(authStorage.hasAuth("anthropic")).toBe(true);
expect(authStorage.hasAuth("openai-codex")).toBe(true);
await expect(authStorage.getApiKey("openrouter")).resolves.toBe("sk-or-v1-runtime");
await expect(authStorage.getApiKey("anthropic")).resolves.toBe("sk-ant-runtime");
expect(authStorage.get("openai-codex")).toMatchObject({
type: "oauth",
access: "oauth-access",
});
expect(await pathExists(path.join(agentDir, "auth.json"))).toBe(false);
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
});

View File

@@ -1,19 +1,125 @@
import path from "node:path";
import { AuthStorage, ModelRegistry } from "@mariozechner/pi-coding-agent";
import {
AuthStorage,
InMemoryAuthStorageBackend,
ModelRegistry,
} from "@mariozechner/pi-coding-agent";
import { ensureAuthProfileStore } from "./auth-profiles.js";
import type { AuthProfileCredential } from "./auth-profiles.js";
import { normalizeProviderId } from "./model-selection.js";
export { AuthStorage, ModelRegistry } from "@mariozechner/pi-coding-agent";
function createAuthStorage(AuthStorageLike: unknown, path: string) {
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
if (typeof withFactory.create === "function") {
return withFactory.create(path) as AuthStorage;
type PiApiKeyCredential = { type: "api_key"; key: string };
type PiOAuthCredential = {
type: "oauth";
access: string;
refresh: string;
expires: number;
};
type PiCredential = PiApiKeyCredential | PiOAuthCredential;
type PiCredentialMap = Record<string, PiCredential>;
function createAuthStorage(AuthStorageLike: unknown, path: string, creds: PiCredentialMap) {
const withInMemory = AuthStorageLike as { inMemory?: (data?: unknown) => unknown };
if (typeof withInMemory.inMemory === "function") {
return withInMemory.inMemory(creds) as AuthStorage;
}
return new (AuthStorageLike as { new (path: string): unknown })(path) as AuthStorage;
const withFromStorage = AuthStorageLike as {
fromStorage?: (storage: unknown) => unknown;
};
if (typeof withFromStorage.fromStorage === "function") {
const backend = new InMemoryAuthStorageBackend();
backend.withLock(() => ({
result: undefined,
next: JSON.stringify(creds, null, 2),
}));
return withFromStorage.fromStorage(backend) as AuthStorage;
}
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
const withRuntimeOverride = (
typeof withFactory.create === "function"
? withFactory.create(path)
: new (AuthStorageLike as { new (path: string): unknown })(path)
) as AuthStorage & {
setRuntimeApiKey?: (provider: string, apiKey: string) => void;
};
if (typeof withRuntimeOverride.setRuntimeApiKey === "function") {
for (const [provider, credential] of Object.entries(creds)) {
if (credential.type === "api_key") {
withRuntimeOverride.setRuntimeApiKey(provider, credential.key);
continue;
}
withRuntimeOverride.setRuntimeApiKey(provider, credential.access);
}
}
return withRuntimeOverride;
}
function convertAuthProfileCredential(cred: AuthProfileCredential): PiCredential | null {
if (cred.type === "api_key") {
const key = typeof cred.key === "string" ? cred.key.trim() : "";
if (!key) {
return null;
}
return { type: "api_key", key };
}
if (cred.type === "token") {
const token = typeof cred.token === "string" ? cred.token.trim() : "";
if (!token) {
return null;
}
if (
typeof cred.expires === "number" &&
Number.isFinite(cred.expires) &&
Date.now() >= cred.expires
) {
return null;
}
return { type: "api_key", key: token };
}
if (cred.type === "oauth") {
const access = typeof cred.access === "string" ? cred.access.trim() : "";
const refresh = typeof cred.refresh === "string" ? cred.refresh.trim() : "";
if (!access || !refresh || !Number.isFinite(cred.expires) || cred.expires <= 0) {
return null;
}
return {
type: "oauth",
access,
refresh,
expires: cred.expires,
};
}
return null;
}
function resolvePiCredentials(agentDir: string): PiCredentialMap {
const store = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false });
const credentials: PiCredentialMap = {};
for (const credential of Object.values(store.profiles)) {
const provider = normalizeProviderId(String(credential.provider ?? "")).trim();
if (!provider || credentials[provider]) {
continue;
}
const converted = convertAuthProfileCredential(credential);
if (converted) {
credentials[provider] = converted;
}
}
return credentials;
}
// Compatibility helpers for pi-coding-agent 0.50+ (discover* helpers removed).
export function discoverAuthStorage(agentDir: string): AuthStorage {
return createAuthStorage(AuthStorage, path.join(agentDir, "auth.json"));
const credentials = resolvePiCredentials(agentDir);
return createAuthStorage(AuthStorage, path.join(agentDir, "auth.json"), credentials);
}
export function discoverModels(authStorage: AuthStorage, agentDir: string): ModelRegistry {

View File

@@ -260,6 +260,15 @@ const entries: SubCliEntry[] = [
mod.registerSecurityCli(program);
},
},
{
name: "secrets",
description: "Secrets runtime reload controls",
hasSubcommands: true,
register: async (program) => {
const mod = await import("../secrets-cli.js");
mod.registerSecretsCli(program);
},
},
{
name: "skills",
description: "List and inspect available skills",

View File

@@ -0,0 +1,53 @@
import { Command } from "commander";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { createCliRuntimeCapture } from "./test-runtime-capture.js";
const callGatewayFromCli = vi.fn();
const { defaultRuntime, runtimeLogs, runtimeErrors, resetRuntimeCapture } =
createCliRuntimeCapture();
vi.mock("./gateway-rpc.js", () => ({
addGatewayClientOptions: (cmd: Command) => cmd,
callGatewayFromCli: (method: string, opts: unknown, params?: unknown, extra?: unknown) =>
callGatewayFromCli(method, opts, params, extra),
}));
vi.mock("../runtime.js", () => ({
defaultRuntime,
}));
const { registerSecretsCli } = await import("./secrets-cli.js");
describe("secrets CLI", () => {
const createProgram = () => {
const program = new Command();
program.exitOverride();
registerSecretsCli(program);
return program;
};
beforeEach(() => {
resetRuntimeCapture();
callGatewayFromCli.mockReset();
});
it("calls secrets.reload and prints human output", async () => {
callGatewayFromCli.mockResolvedValue({ ok: true, warningCount: 1 });
await createProgram().parseAsync(["secrets", "reload"], { from: "user" });
expect(callGatewayFromCli).toHaveBeenCalledWith(
"secrets.reload",
expect.anything(),
undefined,
expect.objectContaining({ expectFinal: false }),
);
expect(runtimeLogs.at(-1)).toBe("Secrets reloaded with 1 warning(s).");
expect(runtimeErrors).toHaveLength(0);
});
it("prints JSON when requested", async () => {
callGatewayFromCli.mockResolvedValue({ ok: true, warningCount: 0 });
await createProgram().parseAsync(["secrets", "reload", "--json"], { from: "user" });
expect(runtimeLogs.at(-1)).toContain('"ok": true');
});
});

47
src/cli/secrets-cli.ts Normal file
View File

@@ -0,0 +1,47 @@
import type { Command } from "commander";
import { danger } from "../globals.js";
import { defaultRuntime } from "../runtime.js";
import { formatDocsLink } from "../terminal/links.js";
import { theme } from "../terminal/theme.js";
import { addGatewayClientOptions, callGatewayFromCli, type GatewayRpcOpts } from "./gateway-rpc.js";
type SecretsReloadOptions = GatewayRpcOpts & { json?: boolean };
export function registerSecretsCli(program: Command) {
const secrets = program
.command("secrets")
.description("Secrets runtime controls")
.addHelpText(
"after",
() =>
`\n${theme.muted("Docs:")} ${formatDocsLink("/gateway/security", "docs.openclaw.ai/gateway/security")}\n`,
);
addGatewayClientOptions(
secrets
.command("reload")
.description("Re-resolve secret references and atomically swap runtime snapshot")
.option("--json", "Output JSON", false),
).action(async (opts: SecretsReloadOptions) => {
try {
const result = await callGatewayFromCli("secrets.reload", opts, undefined, {
expectFinal: false,
});
if (opts.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
const warningCount = Number(
(result as { warningCount?: unknown } | undefined)?.warningCount ?? 0,
);
if (Number.isFinite(warningCount) && warningCount > 0) {
defaultRuntime.log(`Secrets reloaded with ${warningCount} warning(s).`);
return;
}
defaultRuntime.log("Secrets reloaded.");
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
}

View File

@@ -95,7 +95,7 @@ describe("models list auth-profile sync", () => {
const openrouter = getProviderRow(String(runtime.log.mock.calls[0]?.[0]), "openrouter/");
expect(openrouter).toBeDefined();
expect(openrouter?.available).toBe(true);
expect(await pathExists(authPath)).toBe(true);
expect(await pathExists(authPath)).toBe(false);
});
});

View File

@@ -6,9 +6,6 @@ let toModelRow: typeof import("./models/list.registry.js").toModelRow;
const loadConfig = vi.fn();
const ensureOpenClawModelsJson = vi.fn().mockResolvedValue(undefined);
const ensurePiAuthJsonFromAuthProfiles = vi
.fn()
.mockResolvedValue({ wrote: false, authPath: "/tmp/openclaw-agent/auth.json" });
const resolveOpenClawAgentDir = vi.fn().mockReturnValue("/tmp/openclaw-agent");
const ensureAuthProfileStore = vi.fn().mockReturnValue({ version: 1, profiles: {} });
const listProfilesForProvider = vi.fn().mockReturnValue([]);
@@ -38,10 +35,6 @@ vi.mock("../agents/models-config.js", () => ({
ensureOpenClawModelsJson,
}));
vi.mock("../agents/pi-auth-json.js", () => ({
ensurePiAuthJsonFromAuthProfiles,
}));
vi.mock("../agents/agent-paths.js", () => ({
resolveOpenClawAgentDir,
}));
@@ -110,7 +103,6 @@ beforeEach(() => {
modelRegistryState.getAllError = undefined;
modelRegistryState.getAvailableError = undefined;
listProfilesForProvider.mockReturnValue([]);
ensurePiAuthJsonFromAuthProfiles.mockClear();
});
afterEach(() => {
@@ -279,13 +271,12 @@ describe("models list/status", () => {
({ loadModelRegistry, toModelRow } = await import("./models/list.registry.js"));
});
it("models list syncs auth-profiles into auth.json before availability checks", async () => {
it("models list runs model discovery without auth.json sync", async () => {
setDefaultZaiRegistry();
const runtime = makeRuntime();
await modelsListCommand({ all: true, json: true }, runtime);
expect(ensurePiAuthJsonFromAuthProfiles).toHaveBeenCalledWith("/tmp/openclaw-agent");
expect(runtime.error).not.toHaveBeenCalled();
});
it("models list outputs canonical zai key for configured z.ai model", async () => {

View File

@@ -12,7 +12,6 @@ import {
resolveForwardCompatModel,
} from "../../agents/model-forward-compat.js";
import { ensureOpenClawModelsJson } from "../../agents/models-config.js";
import { ensurePiAuthJsonFromAuthProfiles } from "../../agents/pi-auth-json.js";
import type { ModelRegistry } from "../../agents/pi-model-discovery.js";
import { discoverAuthStorage, discoverModels } from "../../agents/pi-model-discovery.js";
import type { OpenClawConfig } from "../../config/config.js";
@@ -102,7 +101,6 @@ function loadAvailableModels(registry: ModelRegistry): Model<Api>[] {
export async function loadModelRegistry(cfg: OpenClawConfig) {
await ensureOpenClawModelsJson(cfg);
const agentDir = resolveOpenClawAgentDir();
await ensurePiAuthJsonFromAuthProfiles(agentDir);
const authStorage = discoverAuthStorage(agentDir);
const registry = discoverModels(authStorage, agentDir);
const appended = appendAntigravityForwardCompatModels(registry.getAll(), registry);

View File

@@ -4,6 +4,7 @@ import type { OpenClawConfig } from "../config/config.js";
import type { ModelProviderConfig } from "../config/types.models.js";
import type { RuntimeEnv } from "../runtime.js";
import { fetchWithTimeout } from "../utils/fetch-timeout.js";
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
import type { WizardPrompter } from "../wizard/prompts.js";
import { applyPrimaryModel } from "./model-picker.js";
import { normalizeAlias } from "./models/shared.js";
@@ -521,7 +522,7 @@ export function applyCustomApiConfig(params: ApplyCustomApiConfigParams): Custom
const mergedModels = hasModel ? existingModels : [...existingModels, nextModel];
const { apiKey: existingApiKey, ...existingProviderRest } = existingProvider ?? {};
const normalizedApiKey =
params.apiKey?.trim() || (existingApiKey ? existingApiKey.trim() : undefined);
normalizeOptionalSecretInput(params.apiKey) ?? normalizeOptionalSecretInput(existingApiKey);
let config: OpenClawConfig = {
...params.config,

View File

@@ -1,11 +1,14 @@
export {
clearConfigCache,
clearRuntimeConfigSnapshot,
createConfigIO,
getRuntimeConfigSnapshot,
loadConfig,
parseConfigJson5,
readConfigFileSnapshot,
readConfigFileSnapshotForWrite,
resolveConfigSnapshotHash,
setRuntimeConfigSnapshot,
writeConfigFile,
} from "./io.js";
export { migrateLegacyConfig } from "./legacy-migrate.js";

View File

@@ -0,0 +1,63 @@
import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { withTempHome } from "./home-env.test-harness.js";
import {
clearConfigCache,
clearRuntimeConfigSnapshot,
loadConfig,
setRuntimeConfigSnapshot,
writeConfigFile,
} from "./io.js";
import type { OpenClawConfig } from "./types.js";
describe("runtime config snapshot writes", () => {
it("preserves source secret refs when writeConfigFile receives runtime-resolved config", async () => {
await withTempHome("openclaw-config-runtime-write-", async (home) => {
const configPath = path.join(home, ".openclaw", "openclaw.json");
const sourceConfig: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: { source: "env", id: "OPENAI_API_KEY" },
models: [],
},
},
},
};
const runtimeConfig: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: "sk-runtime-resolved",
models: [],
},
},
},
};
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.writeFile(configPath, `${JSON.stringify(sourceConfig, null, 2)}\n`, "utf8");
try {
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-runtime-resolved");
await writeConfigFile(loadConfig());
const persisted = JSON.parse(await fs.readFile(configPath, "utf8")) as {
models?: { providers?: { openai?: { apiKey?: unknown } } };
};
expect(persisted.models?.providers?.openai?.apiKey).toEqual({
source: "env",
id: "OPENAI_API_KEY",
});
} finally {
clearRuntimeConfigSnapshot();
clearConfigCache();
}
});
});
});

View File

@@ -1061,6 +1061,8 @@ let configCache: {
expiresAt: number;
config: OpenClawConfig;
} | null = null;
let runtimeConfigSnapshot: OpenClawConfig | null = null;
let runtimeConfigSourceSnapshot: OpenClawConfig | null = null;
function resolveConfigCacheMs(env: NodeJS.ProcessEnv): number {
const raw = env.OPENCLAW_CONFIG_CACHE_MS?.trim();
@@ -1088,7 +1090,29 @@ export function clearConfigCache(): void {
configCache = null;
}
export function setRuntimeConfigSnapshot(
config: OpenClawConfig,
sourceConfig?: OpenClawConfig,
): void {
runtimeConfigSnapshot = config;
runtimeConfigSourceSnapshot = sourceConfig ?? null;
clearConfigCache();
}
export function clearRuntimeConfigSnapshot(): void {
runtimeConfigSnapshot = null;
runtimeConfigSourceSnapshot = null;
clearConfigCache();
}
export function getRuntimeConfigSnapshot(): OpenClawConfig | null {
return runtimeConfigSnapshot;
}
export function loadConfig(): OpenClawConfig {
if (runtimeConfigSnapshot) {
return runtimeConfigSnapshot;
}
const io = createConfigIO();
const configPath = io.configPath;
const now = Date.now();
@@ -1125,9 +1149,14 @@ export async function writeConfigFile(
options: ConfigWriteOptions = {},
): Promise<void> {
const io = createConfigIO();
let nextCfg = cfg;
if (runtimeConfigSnapshot && runtimeConfigSourceSnapshot) {
const runtimePatch = createMergePatch(runtimeConfigSnapshot, cfg);
nextCfg = coerceConfig(applyMergePatch(runtimeConfigSourceSnapshot, runtimePatch));
}
const sameConfigPath =
options.expectedConfigPath === undefined || options.expectedConfigPath === io.configPath;
await io.writeConfigFile(cfg, {
await io.writeConfigFile(nextCfg, {
envSnapshotForRestore: sameConfigPath ? options.envSnapshotForRestore : undefined,
});
}

View File

@@ -1,3 +1,5 @@
import type { SecretInput } from "./types.secrets.js";
export type ModelApi =
| "openai-completions"
| "openai-responses"
@@ -43,7 +45,7 @@ export type ModelDefinitionConfig = {
export type ModelProviderConfig = {
baseUrl: string;
apiKey?: string;
apiKey?: SecretInput;
auth?: ModelProviderAuthMode;
api?: ModelApi;
headers?: Record<string, string>;

View File

@@ -13,6 +13,24 @@ export type SecretRef = {
export type SecretInput = string | SecretRef;
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
export function isSecretRef(value: unknown): value is SecretRef {
if (!isRecord(value)) {
return false;
}
if (Object.keys(value).length !== 2) {
return false;
}
return (
(value.source === "env" || value.source === "file") &&
typeof value.id === "string" &&
value.id.trim().length > 0
);
}
export type EnvSecretSourceConfig = {
type?: "env";
};

View File

@@ -250,7 +250,7 @@ export function startGatewayConfigReloader(opts: {
initialConfig: OpenClawConfig;
readSnapshot: () => Promise<ConfigFileSnapshot>;
onHotReload: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void;
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
log: {
info: (msg: string) => void;
warn: (msg: string) => void;
@@ -301,42 +301,53 @@ export function startGatewayConfigReloader(opts: {
}
const nextConfig = snapshot.config;
const changedPaths = diffConfigPaths(currentConfig, nextConfig);
currentConfig = nextConfig;
settings = resolveGatewayReloadSettings(nextConfig);
const nextSettings = resolveGatewayReloadSettings(nextConfig);
if (changedPaths.length === 0) {
currentConfig = nextConfig;
settings = nextSettings;
return;
}
opts.log.info(`config change detected; evaluating reload (${changedPaths.join(", ")})`);
const plan = buildGatewayReloadPlan(changedPaths);
if (settings.mode === "off") {
if (nextSettings.mode === "off") {
opts.log.info("config reload disabled (gateway.reload.mode=off)");
currentConfig = nextConfig;
settings = nextSettings;
return;
}
if (settings.mode === "restart") {
if (nextSettings.mode === "restart") {
if (!restartQueued) {
restartQueued = true;
opts.onRestart(plan, nextConfig);
await opts.onRestart(plan, nextConfig);
}
currentConfig = nextConfig;
settings = nextSettings;
return;
}
if (plan.restartGateway) {
if (settings.mode === "hot") {
if (nextSettings.mode === "hot") {
opts.log.warn(
`config reload requires gateway restart; hot mode ignoring (${plan.restartReasons.join(
", ",
)})`,
);
currentConfig = nextConfig;
settings = nextSettings;
return;
}
if (!restartQueued) {
restartQueued = true;
opts.onRestart(plan, nextConfig);
await opts.onRestart(plan, nextConfig);
}
currentConfig = nextConfig;
settings = nextSettings;
return;
}
await opts.onHotReload(plan, nextConfig);
currentConfig = nextConfig;
settings = nextSettings;
} catch (err) {
opts.log.error(`config reload failed: ${String(err)}`);
} finally {

View File

@@ -97,6 +97,7 @@ const METHOD_SCOPE_GROUPS: Record<OperatorScope, readonly string[]> = {
"agents.delete",
"skills.install",
"skills.update",
"secrets.reload",
"cron.add",
"cron.update",
"cron.remove",

View File

@@ -48,6 +48,7 @@ const BASE_METHODS = [
"update.run",
"voicewake.get",
"voicewake.set",
"secrets.reload",
"sessions.list",
"sessions.preview",
"sessions.patch",

View File

@@ -0,0 +1,43 @@
import { describe, expect, it, vi } from "vitest";
import { createSecretsHandlers } from "./secrets.js";
describe("secrets handlers", () => {
it("responds with warning count on successful reload", async () => {
const handlers = createSecretsHandlers({
reloadSecrets: vi.fn().mockResolvedValue({ warningCount: 2 }),
});
const respond = vi.fn();
await handlers["secrets.reload"]({
req: { type: "req", id: "1", method: "secrets.reload" },
params: {},
client: null,
isWebchatConnect: () => false,
respond,
context: {} as never,
});
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 2 });
});
it("returns unavailable when reload fails", async () => {
const handlers = createSecretsHandlers({
reloadSecrets: vi.fn().mockRejectedValue(new Error("reload failed")),
});
const respond = vi.fn();
await handlers["secrets.reload"]({
req: { type: "req", id: "1", method: "secrets.reload" },
params: {},
client: null,
isWebchatConnect: () => false,
respond,
context: {} as never,
});
expect(respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({
code: "UNAVAILABLE",
message: "Error: reload failed",
}),
);
});
});

View File

@@ -0,0 +1,17 @@
import { ErrorCodes, errorShape } from "../protocol/index.js";
import type { GatewayRequestHandlers } from "./types.js";
export function createSecretsHandlers(params: {
reloadSecrets: () => Promise<{ warningCount: number }>;
}): GatewayRequestHandlers {
return {
"secrets.reload": async ({ respond }) => {
try {
const result = await params.reloadSecrets();
respond(true, { ok: true, warningCount: result.warningCount });
} catch (err) {
respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, String(err)));
}
},
};
}

View File

@@ -11,6 +11,7 @@ import { createDefaultDeps } from "../cli/deps.js";
import { isRestartEnabled } from "../config/commands.js";
import {
CONFIG_PATH,
type OpenClawConfig,
isNixMode,
loadConfig,
migrateLegacyConfig,
@@ -45,6 +46,12 @@ import { createEmptyPluginRegistry } from "../plugins/registry.js";
import type { PluginServicesHandle } from "../plugins/services.js";
import { getTotalQueueSize } from "../process/command-queue.js";
import type { RuntimeEnv } from "../runtime.js";
import {
activateSecretsRuntimeSnapshot,
clearSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
prepareSecretsRuntimeSnapshot,
} from "../secrets/runtime.js";
import { runOnboardingWizard } from "../wizard/onboarding.js";
import { createAuthRateLimiter, type AuthRateLimiter } from "./auth-rate-limit.js";
import { startChannelHealthMonitor } from "./channel-health-monitor.js";
@@ -68,6 +75,7 @@ import { GATEWAY_EVENTS, listGatewayMethods } from "./server-methods-list.js";
import { coreGatewayHandlers } from "./server-methods.js";
import { createExecApprovalHandlers } from "./server-methods/exec-approval.js";
import { safeParseJson } from "./server-methods/nodes.helpers.js";
import { createSecretsHandlers } from "./server-methods/secrets.js";
import { hasConnectedMobileNode } from "./server-mobile-nodes.js";
import { loadGatewayModelCatalog } from "./server-model-catalog.js";
import { createNodeSubscriptionManager } from "./server-node-subscriptions.js";
@@ -107,6 +115,7 @@ const logReload = log.child("reload");
const logHooks = log.child("hooks");
const logPlugins = log.child("plugins");
const logWsControl = log.child("ws");
const logSecrets = log.child("secrets");
const gatewayRuntime = runtimeForLogger(log);
const canvasRuntime = runtimeForLogger(logCanvas);
@@ -233,7 +242,63 @@ export async function startGatewayServer(
}
}
let cfgAtStart = loadConfig();
let secretsDegraded = false;
const activateRuntimeSecrets = async (
config: OpenClawConfig,
params: { reason: "startup" | "reload" | "restart-check"; activate: boolean },
) => {
try {
const prepared = await prepareSecretsRuntimeSnapshot({ config });
if (params.activate) {
activateSecretsRuntimeSnapshot(prepared);
}
for (const warning of prepared.warnings) {
logSecrets.warn(`[${warning.code}] ${warning.message}`);
}
if (secretsDegraded) {
logSecrets.info(
"[SECRETS_RELOADER_RECOVERED] Secret resolution recovered; runtime remained on last-known-good during the outage.",
);
}
secretsDegraded = false;
return prepared;
} catch (err) {
const details = String(err);
if (!secretsDegraded) {
logSecrets.error(`[SECRETS_RELOADER_DEGRADED] ${details}`);
} else {
logSecrets.warn(`[SECRETS_RELOADER_DEGRADED] ${details}`);
}
secretsDegraded = true;
if (params.reason === "startup") {
throw new Error(`Startup failed: required secrets are unavailable. ${details}`, {
cause: err,
});
}
throw err;
}
};
// Fail fast before startup if required refs are unresolved.
let cfgAtStart: OpenClawConfig;
{
const freshSnapshot = await readConfigFileSnapshot();
if (!freshSnapshot.valid) {
const issues =
freshSnapshot.issues.length > 0
? freshSnapshot.issues
.map((issue) => `${issue.path || "<root>"}: ${issue.message}`)
.join("\n")
: "Unknown validation issue.";
throw new Error(`Invalid config at ${freshSnapshot.path}.\n${issues}`);
}
await activateRuntimeSecrets(freshSnapshot.config, {
reason: "startup",
activate: false,
});
}
cfgAtStart = loadConfig();
const authBootstrap = await ensureGatewayStartupAuth({
cfg: cfgAtStart,
env: process.env,
@@ -253,6 +318,12 @@ export async function startGatewayServer(
);
}
}
cfgAtStart = (
await activateRuntimeSecrets(cfgAtStart, {
reason: "startup",
activate: true,
})
).config;
const diagnosticsEnabled = isDiagnosticsEnabled(cfgAtStart);
if (diagnosticsEnabled) {
startDiagnosticHeartbeat();
@@ -560,6 +631,19 @@ export async function startGatewayServer(
const execApprovalHandlers = createExecApprovalHandlers(execApprovalManager, {
forwarder: execApprovalForwarder,
});
const secretsHandlers = createSecretsHandlers({
reloadSecrets: async () => {
const active = getActiveSecretsRuntimeSnapshot();
if (!active) {
throw new Error("Secrets runtime snapshot is not active.");
}
const prepared = await activateRuntimeSecrets(active.sourceConfig, {
reason: "reload",
activate: true,
});
return { warningCount: prepared.warnings.length };
},
});
const canvasHostServerPort = (canvasHostServer as CanvasHostServer | null)?.port;
@@ -580,6 +664,7 @@ export async function startGatewayServer(
extraHandlers: {
...pluginRegistry.gatewayHandlers,
...execApprovalHandlers,
...secretsHandlers,
},
broadcast,
context: {
@@ -710,8 +795,27 @@ export async function startGatewayServer(
return startGatewayConfigReloader({
initialConfig: cfgAtStart,
readSnapshot: readConfigFileSnapshot,
onHotReload: applyHotReload,
onRestart: requestGatewayRestart,
onHotReload: async (plan, nextConfig) => {
const previousSnapshot = getActiveSecretsRuntimeSnapshot();
const prepared = await activateRuntimeSecrets(nextConfig, {
reason: "reload",
activate: true,
});
try {
await applyHotReload(plan, prepared.config);
} catch (err) {
if (previousSnapshot) {
activateSecretsRuntimeSnapshot(previousSnapshot);
} else {
clearSecretsRuntimeSnapshot();
}
throw err;
}
},
onRestart: async (plan, nextConfig) => {
await activateRuntimeSecrets(nextConfig, { reason: "restart-check", activate: false });
requestGatewayRestart(plan, nextConfig);
},
log: {
info: (msg) => logReload.info(msg),
warn: (msg) => logReload.warn(msg),
@@ -764,6 +868,7 @@ export async function startGatewayServer(
skillsChangeUnsub();
authRateLimiter?.dispose();
channelHealthMonitor?.stop();
clearSecretsRuntimeSnapshot();
await close(opts);
},
};

View File

@@ -0,0 +1,94 @@
function failOrUndefined(params: { onMissing: "throw" | "undefined"; message: string }): undefined {
if (params.onMissing === "throw") {
throw new Error(params.message);
}
return undefined;
}
export function decodeJsonPointerToken(token: string): string {
return token.replace(/~1/g, "/").replace(/~0/g, "~");
}
export function encodeJsonPointerToken(token: string): string {
return token.replace(/~/g, "~0").replace(/\//g, "~1");
}
export function readJsonPointer(
root: unknown,
pointer: string,
options: { onMissing?: "throw" | "undefined" } = {},
): unknown {
const onMissing = options.onMissing ?? "throw";
if (!pointer.startsWith("/")) {
return failOrUndefined({
onMissing,
message:
'File-backed secret ids must be absolute JSON pointers (for example: "/providers/openai/apiKey").',
});
}
const tokens = pointer
.slice(1)
.split("/")
.map((token) => decodeJsonPointerToken(token));
let current: unknown = root;
for (const token of tokens) {
if (Array.isArray(current)) {
const index = Number.parseInt(token, 10);
if (!Number.isFinite(index) || index < 0 || index >= current.length) {
return failOrUndefined({
onMissing,
message: `JSON pointer segment "${token}" is out of bounds.`,
});
}
current = current[index];
continue;
}
if (typeof current !== "object" || current === null || Array.isArray(current)) {
return failOrUndefined({
onMissing,
message: `JSON pointer segment "${token}" does not exist.`,
});
}
const record = current as Record<string, unknown>;
if (!Object.hasOwn(record, token)) {
return failOrUndefined({
onMissing,
message: `JSON pointer segment "${token}" does not exist.`,
});
}
current = record[token];
}
return current;
}
export function setJsonPointer(
root: Record<string, unknown>,
pointer: string,
value: unknown,
): void {
if (!pointer.startsWith("/")) {
throw new Error(`Invalid JSON pointer "${pointer}".`);
}
const tokens = pointer
.slice(1)
.split("/")
.map((token) => decodeJsonPointerToken(token));
let current: Record<string, unknown> = root;
for (let index = 0; index < tokens.length; index += 1) {
const token = tokens[index];
const isLast = index === tokens.length - 1;
if (isLast) {
current[token] = value;
return;
}
const child = current[token];
if (typeof child !== "object" || child === null || Array.isArray(child)) {
current[token] = {};
}
current = current[token] as Record<string, unknown>;
}
}

View File

@@ -0,0 +1,28 @@
export const PROVIDER_ENV_VARS: Record<string, readonly string[]> = {
openai: ["OPENAI_API_KEY"],
anthropic: ["ANTHROPIC_API_KEY"],
google: ["GEMINI_API_KEY"],
minimax: ["MINIMAX_API_KEY"],
"minimax-cn": ["MINIMAX_API_KEY"],
moonshot: ["MOONSHOT_API_KEY"],
"kimi-coding": ["KIMI_API_KEY", "KIMICODE_API_KEY"],
synthetic: ["SYNTHETIC_API_KEY"],
venice: ["VENICE_API_KEY"],
zai: ["ZAI_API_KEY", "Z_AI_API_KEY"],
xiaomi: ["XIAOMI_API_KEY"],
openrouter: ["OPENROUTER_API_KEY"],
"cloudflare-ai-gateway": ["CLOUDFLARE_AI_GATEWAY_API_KEY"],
litellm: ["LITELLM_API_KEY"],
"vercel-ai-gateway": ["AI_GATEWAY_API_KEY"],
opencode: ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
together: ["TOGETHER_API_KEY"],
huggingface: ["HUGGINGFACE_HUB_TOKEN", "HF_TOKEN"],
qianfan: ["QIANFAN_API_KEY"],
xai: ["XAI_API_KEY"],
volcengine: ["VOLCANO_ENGINE_API_KEY"],
byteplus: ["BYTEPLUS_API_KEY"],
};
export function listKnownSecretEnvVarNames(): string[] {
return [...new Set(Object.values(PROVIDER_ENV_VARS).flatMap((keys) => keys))];
}

159
src/secrets/runtime.test.ts Normal file
View File

@@ -0,0 +1,159 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { ensureAuthProfileStore } from "../agents/auth-profiles.js";
import { loadConfig, type OpenClawConfig } from "../config/config.js";
import {
activateSecretsRuntimeSnapshot,
clearSecretsRuntimeSnapshot,
prepareSecretsRuntimeSnapshot,
} from "./runtime.js";
const runExecMock = vi.hoisted(() => vi.fn());
vi.mock("../process/exec.js", () => ({
runExec: runExecMock,
}));
describe("secrets runtime snapshot", () => {
afterEach(() => {
runExecMock.mockReset();
clearSecretsRuntimeSnapshot();
});
it("resolves env refs for config and auth profiles", async () => {
const config: OpenClawConfig = {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: { source: "env", id: "OPENAI_API_KEY" },
models: [],
},
},
},
};
const snapshot = await prepareSecretsRuntimeSnapshot({
config,
env: {
OPENAI_API_KEY: "sk-env-openai",
GITHUB_TOKEN: "ghp-env-token",
},
agentDirs: ["/tmp/openclaw-agent-main"],
loadAuthStore: () => ({
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "old-openai",
keyRef: { source: "env", id: "OPENAI_API_KEY" },
},
"github-copilot:default": {
type: "token",
provider: "github-copilot",
token: "old-gh",
tokenRef: { source: "env", id: "GITHUB_TOKEN" },
},
},
}),
});
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-env-openai");
expect(snapshot.warnings).toHaveLength(2);
expect(snapshot.authStores[0]?.store.profiles["openai:default"]).toMatchObject({
type: "api_key",
key: "sk-env-openai",
});
expect(snapshot.authStores[0]?.store.profiles["github-copilot:default"]).toMatchObject({
type: "token",
token: "ghp-env-token",
});
});
it("resolves file refs via sops json payload", async () => {
runExecMock.mockResolvedValueOnce({
stdout: JSON.stringify({
providers: {
openai: {
apiKey: "sk-from-sops",
},
},
}),
stderr: "",
});
const config: OpenClawConfig = {
secrets: {
sources: {
file: {
type: "sops",
path: "~/.openclaw/secrets.enc.json",
timeoutMs: 7000,
},
},
},
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: { source: "file", id: "/providers/openai/apiKey" },
models: [],
},
},
},
};
const snapshot = await prepareSecretsRuntimeSnapshot({
config,
agentDirs: ["/tmp/openclaw-agent-main"],
loadAuthStore: () => ({ version: 1, profiles: {} }),
});
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-from-sops");
expect(runExecMock).toHaveBeenCalledWith(
"sops",
["--decrypt", "--output-type", "json", expect.stringContaining("secrets.enc.json")],
{
timeoutMs: 7000,
maxBuffer: 10 * 1024 * 1024,
},
);
});
it("activates runtime snapshots for loadConfig and ensureAuthProfileStore", async () => {
const prepared = await prepareSecretsRuntimeSnapshot({
config: {
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: { source: "env", id: "OPENAI_API_KEY" },
models: [],
},
},
},
},
env: { OPENAI_API_KEY: "sk-runtime" },
agentDirs: ["/tmp/openclaw-agent-main"],
loadAuthStore: () => ({
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
keyRef: { source: "env", id: "OPENAI_API_KEY" },
},
},
}),
});
activateSecretsRuntimeSnapshot(prepared);
expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-runtime");
const store = ensureAuthProfileStore("/tmp/openclaw-agent-main");
expect(store.profiles["openai:default"]).toMatchObject({
type: "api_key",
key: "sk-runtime",
});
});
});

317
src/secrets/runtime.ts Normal file
View File

@@ -0,0 +1,317 @@
import { resolveOpenClawAgentDir } from "../agents/agent-paths.js";
import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js";
import type { AuthProfileCredential, AuthProfileStore } from "../agents/auth-profiles.js";
import {
clearRuntimeAuthProfileStoreSnapshots,
loadAuthProfileStoreForRuntime,
replaceRuntimeAuthProfileStoreSnapshots,
} from "../agents/auth-profiles.js";
import {
clearRuntimeConfigSnapshot,
setRuntimeConfigSnapshot,
type OpenClawConfig,
} from "../config/config.js";
import { isSecretRef, type SecretRef } from "../config/types.secrets.js";
import { resolveUserPath } from "../utils.js";
import { readJsonPointer } from "./json-pointer.js";
import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
type SecretResolverWarningCode = "SECRETS_REF_OVERRIDES_PLAINTEXT";
export type SecretResolverWarning = {
code: SecretResolverWarningCode;
path: string;
message: string;
};
export type PreparedSecretsRuntimeSnapshot = {
sourceConfig: OpenClawConfig;
config: OpenClawConfig;
authStores: Array<{ agentDir: string; store: AuthProfileStore }>;
warnings: SecretResolverWarning[];
};
type ResolverContext = {
config: OpenClawConfig;
env: NodeJS.ProcessEnv;
fileSecretsPromise: Promise<unknown> | null;
};
type ProviderLike = {
apiKey?: unknown;
};
type GoogleChatAccountLike = {
serviceAccount?: unknown;
serviceAccountRef?: unknown;
accounts?: Record<string, unknown>;
};
type ApiKeyCredentialLike = AuthProfileCredential & {
type: "api_key";
key?: string;
keyRef?: unknown;
};
type TokenCredentialLike = AuthProfileCredential & {
type: "token";
token?: string;
tokenRef?: unknown;
};
let activeSnapshot: PreparedSecretsRuntimeSnapshot | null = null;
function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecretsRuntimeSnapshot {
return {
sourceConfig: structuredClone(snapshot.sourceConfig),
config: structuredClone(snapshot.config),
authStores: snapshot.authStores.map((entry) => ({
agentDir: entry.agentDir,
store: structuredClone(entry.store),
})),
warnings: snapshot.warnings.map((warning) => ({ ...warning })),
};
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function isNonEmptyString(value: unknown): value is string {
return typeof value === "string" && value.trim().length > 0;
}
async function decryptSopsFile(config: OpenClawConfig): Promise<unknown> {
const fileSource = config.secrets?.sources?.file;
if (!fileSource) {
throw new Error(
`Secret reference source "file" is not configured. Configure secrets.sources.file first.`,
);
}
if (fileSource.type !== "sops") {
throw new Error(`Unsupported secrets.sources.file.type "${String(fileSource.type)}".`);
}
const resolvedPath = resolveUserPath(fileSource.path);
const timeoutMs =
typeof fileSource.timeoutMs === "number" && Number.isFinite(fileSource.timeoutMs)
? Math.max(1, Math.floor(fileSource.timeoutMs))
: DEFAULT_SOPS_TIMEOUT_MS;
return await decryptSopsJsonFile({
path: resolvedPath,
timeoutMs,
missingBinaryMessage:
"sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.",
});
}
async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext): Promise<unknown> {
const id = ref.id.trim();
if (!id) {
throw new Error(`Secret reference id is empty.`);
}
if (ref.source === "env") {
const envValue = context.env[id];
if (!isNonEmptyString(envValue)) {
throw new Error(`Environment variable "${id}" is missing or empty.`);
}
return envValue;
}
if (ref.source === "file") {
context.fileSecretsPromise ??= decryptSopsFile(context.config);
const payload = await context.fileSecretsPromise;
return readJsonPointer(payload, id, { onMissing: "throw" });
}
throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`);
}
async function resolveGoogleChatServiceAccount(
target: GoogleChatAccountLike,
path: string,
context: ResolverContext,
warnings: SecretResolverWarning[],
): Promise<void> {
const explicitRef = isSecretRef(target.serviceAccountRef) ? target.serviceAccountRef : null;
const inlineRef = isSecretRef(target.serviceAccount) ? target.serviceAccount : null;
const ref = explicitRef ?? inlineRef;
if (!ref) {
return;
}
if (explicitRef && target.serviceAccount !== undefined && !isSecretRef(target.serviceAccount)) {
warnings.push({
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
path,
message: `${path}: serviceAccountRef is set; runtime will ignore plaintext serviceAccount.`,
});
}
target.serviceAccount = await resolveSecretRefValue(ref, context);
}
async function resolveConfigSecretRefs(params: {
config: OpenClawConfig;
context: ResolverContext;
warnings: SecretResolverWarning[];
}): Promise<OpenClawConfig> {
const resolved = structuredClone(params.config);
const providers = resolved.models?.providers as Record<string, ProviderLike> | undefined;
if (providers) {
for (const [providerId, provider] of Object.entries(providers)) {
if (!isSecretRef(provider.apiKey)) {
continue;
}
const resolvedValue = await resolveSecretRefValue(provider.apiKey, params.context);
if (!isNonEmptyString(resolvedValue)) {
throw new Error(
`models.providers.${providerId}.apiKey resolved to a non-string or empty value.`,
);
}
provider.apiKey = resolvedValue;
}
}
const googleChat = resolved.channels?.googlechat as GoogleChatAccountLike | undefined;
if (googleChat) {
await resolveGoogleChatServiceAccount(
googleChat,
"channels.googlechat",
params.context,
params.warnings,
);
if (isRecord(googleChat.accounts)) {
for (const [accountId, account] of Object.entries(googleChat.accounts)) {
if (!isRecord(account)) {
continue;
}
await resolveGoogleChatServiceAccount(
account as GoogleChatAccountLike,
`channels.googlechat.accounts.${accountId}`,
params.context,
params.warnings,
);
}
}
}
return resolved;
}
async function resolveAuthStoreSecretRefs(params: {
store: AuthProfileStore;
context: ResolverContext;
warnings: SecretResolverWarning[];
agentDir: string;
}): Promise<AuthProfileStore> {
const resolvedStore = structuredClone(params.store);
for (const [profileId, profile] of Object.entries(resolvedStore.profiles)) {
if (profile.type === "api_key") {
const apiProfile = profile as ApiKeyCredentialLike;
const keyRef = isSecretRef(apiProfile.keyRef) ? apiProfile.keyRef : null;
if (keyRef && isNonEmptyString(apiProfile.key)) {
params.warnings.push({
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
path: `${params.agentDir}.auth-profiles.${profileId}.key`,
message: `auth-profiles ${profileId}: keyRef is set; runtime will ignore plaintext key.`,
});
}
if (keyRef) {
const resolvedValue = await resolveSecretRefValue(keyRef, params.context);
if (!isNonEmptyString(resolvedValue)) {
throw new Error(`auth profile "${profileId}" keyRef resolved to an empty value.`);
}
apiProfile.key = resolvedValue;
}
continue;
}
if (profile.type === "token") {
const tokenProfile = profile as TokenCredentialLike;
const tokenRef = isSecretRef(tokenProfile.tokenRef) ? tokenProfile.tokenRef : null;
if (tokenRef && isNonEmptyString(tokenProfile.token)) {
params.warnings.push({
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
path: `${params.agentDir}.auth-profiles.${profileId}.token`,
message: `auth-profiles ${profileId}: tokenRef is set; runtime will ignore plaintext token.`,
});
}
if (tokenRef) {
const resolvedValue = await resolveSecretRefValue(tokenRef, params.context);
if (!isNonEmptyString(resolvedValue)) {
throw new Error(`auth profile "${profileId}" tokenRef resolved to an empty value.`);
}
tokenProfile.token = resolvedValue;
}
}
}
return resolvedStore;
}
function collectCandidateAgentDirs(config: OpenClawConfig): string[] {
const dirs = new Set<string>();
dirs.add(resolveUserPath(resolveOpenClawAgentDir()));
for (const agentId of listAgentIds(config)) {
dirs.add(resolveUserPath(resolveAgentDir(config, agentId)));
}
return [...dirs];
}
export async function prepareSecretsRuntimeSnapshot(params: {
config: OpenClawConfig;
env?: NodeJS.ProcessEnv;
agentDirs?: string[];
loadAuthStore?: (agentDir?: string) => AuthProfileStore;
}): Promise<PreparedSecretsRuntimeSnapshot> {
const warnings: SecretResolverWarning[] = [];
const context: ResolverContext = {
config: params.config,
env: params.env ?? process.env,
fileSecretsPromise: null,
};
const resolvedConfig = await resolveConfigSecretRefs({
config: params.config,
context,
warnings,
});
const loadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreForRuntime;
const candidateDirs = params.agentDirs?.length
? [...new Set(params.agentDirs.map((entry) => resolveUserPath(entry)))]
: collectCandidateAgentDirs(resolvedConfig);
const authStores: Array<{ agentDir: string; store: AuthProfileStore }> = [];
for (const agentDir of candidateDirs) {
const rawStore = loadAuthStore(agentDir);
const resolvedStore = await resolveAuthStoreSecretRefs({
store: rawStore,
context,
warnings,
agentDir,
});
authStores.push({ agentDir, store: resolvedStore });
}
return {
sourceConfig: structuredClone(params.config),
config: resolvedConfig,
authStores,
warnings,
};
}
export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void {
const next = cloneSnapshot(snapshot);
setRuntimeConfigSnapshot(next.config, next.sourceConfig);
replaceRuntimeAuthProfileStoreSnapshots(next.authStores);
activeSnapshot = next;
}
export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapshot | null {
return activeSnapshot ? cloneSnapshot(activeSnapshot) : null;
}
export function clearSecretsRuntimeSnapshot(): void {
activeSnapshot = null;
clearRuntimeConfigSnapshot();
clearRuntimeAuthProfileStoreSnapshots();
}

120
src/secrets/sops.ts Normal file
View File

@@ -0,0 +1,120 @@
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { runExec } from "../process/exec.js";
export const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
function normalizeTimeoutMs(value: number | undefined): number {
if (typeof value === "number" && Number.isFinite(value)) {
return Math.max(1, Math.floor(value));
}
return DEFAULT_SOPS_TIMEOUT_MS;
}
function isTimeoutError(message: string | undefined): boolean {
return typeof message === "string" && message.toLowerCase().includes("timed out");
}
type SopsErrorContext = {
missingBinaryMessage: string;
operationLabel: string;
};
function toSopsError(err: unknown, params: SopsErrorContext): Error {
const error = err as NodeJS.ErrnoException & { message?: string };
if (error.code === "ENOENT") {
return new Error(params.missingBinaryMessage, { cause: err });
}
return new Error(`${params.operationLabel}: ${String(error.message ?? err)}`, {
cause: err,
});
}
function ensureDirForFile(filePath: string): void {
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
}
export async function decryptSopsJsonFile(params: {
path: string;
timeoutMs?: number;
missingBinaryMessage: string;
}): Promise<unknown> {
const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
try {
const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", params.path], {
timeoutMs,
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
});
return JSON.parse(stdout) as unknown;
} catch (err) {
const error = err as NodeJS.ErrnoException & { message?: string };
if (isTimeoutError(error.message)) {
throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${params.path}.`, {
cause: err,
});
}
throw toSopsError(err, {
missingBinaryMessage: params.missingBinaryMessage,
operationLabel: `sops decrypt failed for ${params.path}`,
});
}
}
export async function encryptSopsJsonFile(params: {
path: string;
payload: Record<string, unknown>;
timeoutMs?: number;
missingBinaryMessage: string;
}): Promise<void> {
ensureDirForFile(params.path);
const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
const tmpPlain = path.join(
path.dirname(params.path),
`${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.plain.tmp`,
);
const tmpEncrypted = path.join(
path.dirname(params.path),
`${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.enc.tmp`,
);
fs.writeFileSync(tmpPlain, `${JSON.stringify(params.payload, null, 2)}\n`, "utf8");
fs.chmodSync(tmpPlain, 0o600);
try {
await runExec(
"sops",
[
"--encrypt",
"--input-type",
"json",
"--output-type",
"json",
"--output",
tmpEncrypted,
tmpPlain,
],
{
timeoutMs,
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
},
);
fs.renameSync(tmpEncrypted, params.path);
fs.chmodSync(params.path, 0o600);
} catch (err) {
const error = err as NodeJS.ErrnoException & { message?: string };
if (isTimeoutError(error.message)) {
throw new Error(`sops encrypt timed out after ${timeoutMs}ms for ${params.path}.`, {
cause: err,
});
}
throw toSopsError(err, {
missingBinaryMessage: params.missingBinaryMessage,
operationLabel: `sops encrypt failed for ${params.path}`,
});
} finally {
fs.rmSync(tmpPlain, { force: true });
fs.rmSync(tmpEncrypted, { force: true });
}
}