Compare commits

...

8 Commits

Author SHA1 Message Date
joshavant
cad3a00f8a Onboard: store OpenAI auth in profiles instead of .env 2026-02-21 17:18:18 -08:00
joshavant
add2d123b1 Onboard auth: use shared secret-ref helpers 2026-02-21 17:18:18 -08:00
joshavant
ef852b0ae9 Onboard: persist env-backed API keys as secret refs 2026-02-21 17:18:18 -08:00
joshavant
ca8a0dee4b Secrets migrate: share helpers and narrow env scrub scope 2026-02-21 17:17:52 -08:00
joshavant
43748ea2ef Secrets: add migrate rollback and skill ref support 2026-02-21 17:17:52 -08:00
joshavant
ed402b6944 Gateway: add manual secrets reload command 2026-02-21 17:17:52 -08:00
joshavant
6eb674d6d0 Auth profiles: never persist plaintext when refs are present 2026-02-21 17:17:52 -08:00
joshavant
16b7d4bbed Agents: inject pi auth storage from runtime profiles 2026-02-21 17:17:52 -08:00
35 changed files with 2227 additions and 189 deletions

View File

@@ -0,0 +1,71 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
activateSecretsRuntimeSnapshot,
clearSecretsRuntimeSnapshot,
prepareSecretsRuntimeSnapshot,
} from "../secrets/runtime.js";
import { ensureAuthProfileStore, markAuthProfileUsed } from "./auth-profiles.js";
describe("auth profile runtime snapshot persistence", () => {
it("does not write resolved plaintext keys during usage updates", async () => {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-runtime-save-"));
const agentDir = path.join(stateDir, "agents", "main", "agent");
const authPath = path.join(agentDir, "auth-profiles.json");
try {
await fs.mkdir(agentDir, { recursive: true });
await fs.writeFile(
authPath,
`${JSON.stringify(
{
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
keyRef: { source: "env", id: "OPENAI_API_KEY" },
},
},
},
null,
2,
)}\n`,
"utf8",
);
const snapshot = await prepareSecretsRuntimeSnapshot({
config: {},
env: { OPENAI_API_KEY: "sk-runtime-openai" },
agentDirs: [agentDir],
});
activateSecretsRuntimeSnapshot(snapshot);
const runtimeStore = ensureAuthProfileStore(agentDir);
expect(runtimeStore.profiles["openai:default"]).toMatchObject({
type: "api_key",
key: "sk-runtime-openai",
keyRef: { source: "env", id: "OPENAI_API_KEY" },
});
await markAuthProfileUsed({
store: runtimeStore,
profileId: "openai:default",
agentDir,
});
const persisted = JSON.parse(await fs.readFile(authPath, "utf8")) as {
profiles: Record<string, { key?: string; keyRef?: unknown }>;
};
expect(persisted.profiles["openai:default"]?.key).toBeUndefined();
expect(persisted.profiles["openai:default"]?.keyRef).toEqual({
source: "env",
id: "OPENAI_API_KEY",
});
} finally {
clearSecretsRuntimeSnapshot();
await fs.rm(stateDir, { recursive: true, force: true });
}
});
});

View File

@@ -0,0 +1,62 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { resolveAuthStorePath } from "./auth-profiles/paths.js";
import { saveAuthProfileStore } from "./auth-profiles/store.js";
import type { AuthProfileStore } from "./auth-profiles/types.js";
describe("saveAuthProfileStore", () => {
it("strips plaintext when keyRef/tokenRef are present", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-"));
try {
const store: AuthProfileStore = {
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "sk-runtime-value",
keyRef: { source: "env", id: "OPENAI_API_KEY" },
},
"github-copilot:default": {
type: "token",
provider: "github-copilot",
token: "gh-runtime-token",
tokenRef: { source: "env", id: "GITHUB_TOKEN" },
},
"anthropic:default": {
type: "api_key",
provider: "anthropic",
key: "sk-anthropic-plain",
},
},
};
saveAuthProfileStore(store, agentDir);
const parsed = JSON.parse(await fs.readFile(resolveAuthStorePath(agentDir), "utf8")) as {
profiles: Record<
string,
{ key?: string; keyRef?: unknown; token?: string; tokenRef?: unknown }
>;
};
expect(parsed.profiles["openai:default"]?.key).toBeUndefined();
expect(parsed.profiles["openai:default"]?.keyRef).toEqual({
source: "env",
id: "OPENAI_API_KEY",
});
expect(parsed.profiles["github-copilot:default"]?.token).toBeUndefined();
expect(parsed.profiles["github-copilot:default"]?.tokenRef).toEqual({
source: "env",
id: "GITHUB_TOKEN",
});
expect(parsed.profiles["anthropic:default"]?.key).toBe("sk-anthropic-plain");
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
});

View File

@@ -414,9 +414,24 @@ export function ensureAuthProfileStore(
export function saveAuthProfileStore(store: AuthProfileStore, agentDir?: string): void {
const authPath = resolveAuthStorePath(agentDir);
const profiles = Object.fromEntries(
Object.entries(store.profiles).map(([profileId, credential]) => {
if (credential.type === "api_key" && credential.keyRef && credential.key !== undefined) {
const sanitized = { ...credential } as Record<string, unknown>;
delete sanitized.key;
return [profileId, sanitized];
}
if (credential.type === "token" && credential.tokenRef && credential.token !== undefined) {
const sanitized = { ...credential } as Record<string, unknown>;
delete sanitized.token;
return [profileId, sanitized];
}
return [profileId, credential];
}),
) as AuthProfileStore["profiles"];
const payload = {
version: AUTH_STORE_VERSION,
profiles: store.profiles,
profiles,
order: store.order ?? undefined,
lastGood: store.lastGood ?? undefined,
usageStats: store.usageStats ?? undefined,

View File

@@ -31,6 +31,7 @@ export function mockCatalogImportFailThenRecover() {
throw new Error("boom");
}
return {
discoverAuthStorage: () => ({}),
AuthStorage: class {},
ModelRegistry: class {
getAll() {

View File

@@ -38,6 +38,7 @@ describe("loadModelCatalog", () => {
__setModelCatalogImportForTest(
async () =>
({
discoverAuthStorage: () => ({}),
AuthStorage: class {},
ModelRegistry: class {
getAll() {
@@ -69,6 +70,7 @@ describe("loadModelCatalog", () => {
__setModelCatalogImportForTest(
async () =>
({
discoverAuthStorage: () => ({}),
AuthStorage: class {},
ModelRegistry: class {
getAll() {

View File

@@ -70,14 +70,6 @@ export function __setModelCatalogImportForTest(loader?: () => Promise<PiSdkModul
importPiSdk = loader ?? defaultImportPiSdk;
}
function createAuthStorage(AuthStorageLike: unknown, path: string) {
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
if (typeof withFactory.create === "function") {
return withFactory.create(path);
}
return new (AuthStorageLike as { new (path: string): unknown })(path);
}
export async function loadModelCatalog(params?: {
config?: OpenClawConfig;
useCache?: boolean;
@@ -102,9 +94,6 @@ export async function loadModelCatalog(params?: {
try {
const cfg = params?.config ?? loadConfig();
await ensureOpenClawModelsJson(cfg);
await (
await import("./pi-auth-json.js")
).ensurePiAuthJsonFromAuthProfiles(resolveOpenClawAgentDir());
// IMPORTANT: keep the dynamic import *inside* the try/catch.
// If this fails once (e.g. during a pnpm install that temporarily swaps node_modules),
// we must not poison the cache with a rejected promise (otherwise all channel handlers
@@ -112,7 +101,7 @@ export async function loadModelCatalog(params?: {
const piSdk = await importPiSdk();
const agentDir = resolveOpenClawAgentDir();
const { join } = await import("node:path");
const authStorage = createAuthStorage(piSdk.AuthStorage, join(agentDir, "auth.json"));
const authStorage = piSdk.discoverAuthStorage(agentDir);
const registry = new (piSdk.ModelRegistry as unknown as {
new (
authStorage: unknown,

View File

@@ -961,17 +961,8 @@ export async function resolveImplicitCopilotProvider(params: {
}
}
// pi-coding-agent's ModelRegistry marks a model "available" only if its
// `AuthStorage` has auth configured for that provider (via auth.json/env/etc).
// Our Copilot auth lives in OpenClaw's auth-profiles store instead, so we also
// write a runtime-only auth.json entry for pi-coding-agent to pick up.
//
// This is safe because it's (1) within OpenClaw's agent dir, (2) contains the
// GitHub token (not the exchanged Copilot token), and (3) matches existing
// patterns for OAuth-like providers in pi-coding-agent.
// Note: we deliberately do not write pi-coding-agent's `auth.json` here.
// OpenClaw uses its own auth store and exchanges tokens at runtime.
// `models list` uses OpenClaw's auth heuristics for availability.
// We deliberately do not write pi-coding-agent auth.json here.
// OpenClaw keeps auth in auth-profiles and resolves runtime availability from that store.
// We intentionally do NOT define custom models for Copilot in models.json.
// pi-coding-agent treats providers with models as replacements requiring apiKey.

View File

@@ -4,6 +4,10 @@ import { ensureAuthProfileStore } from "./auth-profiles.js";
import type { AuthProfileCredential } from "./auth-profiles/types.js";
import { normalizeProviderId } from "./model-selection.js";
/**
* @deprecated Legacy bridge for older flows that still expect `agentDir/auth.json`.
* Runtime auth resolution uses auth-profiles directly and should not depend on this module.
*/
type AuthJsonCredential =
| {
type: "api_key";
@@ -110,6 +114,8 @@ function credentialsEqual(a: AuthJsonCredential | undefined, b: AuthJsonCredenti
* registry/catalog output.
*
* Syncs all credential types: api_key, token (as api_key), and oauth.
*
* @deprecated Runtime auth now comes from OpenClaw auth-profiles snapshots.
*/
export async function ensurePiAuthJsonFromAuthProfiles(agentDir: string): Promise<{
wrote: boolean;

View File

@@ -0,0 +1,116 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { saveAuthProfileStore } from "./auth-profiles.js";
import { discoverAuthStorage } from "./pi-model-discovery.js";
async function createAgentDir(): Promise<string> {
return await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-pi-auth-storage-"));
}
async function pathExists(pathname: string): Promise<boolean> {
try {
await fs.stat(pathname);
return true;
} catch {
return false;
}
}
describe("discoverAuthStorage", () => {
it("loads runtime credentials from auth-profiles without writing auth.json", async () => {
const agentDir = await createAgentDir();
try {
saveAuthProfileStore(
{
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
"anthropic:default": {
type: "token",
provider: "anthropic",
token: "sk-ant-runtime",
},
"openai-codex:default": {
type: "oauth",
provider: "openai-codex",
access: "oauth-access",
refresh: "oauth-refresh",
expires: Date.now() + 60_000,
},
},
},
agentDir,
);
const authStorage = discoverAuthStorage(agentDir);
expect(authStorage.hasAuth("openrouter")).toBe(true);
expect(authStorage.hasAuth("anthropic")).toBe(true);
expect(authStorage.hasAuth("openai-codex")).toBe(true);
await expect(authStorage.getApiKey("openrouter")).resolves.toBe("sk-or-v1-runtime");
await expect(authStorage.getApiKey("anthropic")).resolves.toBe("sk-ant-runtime");
expect(authStorage.get("openai-codex")).toMatchObject({
type: "oauth",
access: "oauth-access",
});
expect(await pathExists(path.join(agentDir, "auth.json"))).toBe(false);
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
it("scrubs static api_key entries from legacy auth.json and keeps oauth entries", async () => {
const agentDir = await createAgentDir();
try {
saveAuthProfileStore(
{
version: 1,
profiles: {
"openrouter:default": {
type: "api_key",
provider: "openrouter",
key: "sk-or-v1-runtime",
},
},
},
agentDir,
);
await fs.writeFile(
path.join(agentDir, "auth.json"),
JSON.stringify(
{
openrouter: { type: "api_key", key: "legacy-static-key" },
"openai-codex": {
type: "oauth",
access: "oauth-access",
refresh: "oauth-refresh",
expires: Date.now() + 60_000,
},
},
null,
2,
),
);
discoverAuthStorage(agentDir);
const parsed = JSON.parse(await fs.readFile(path.join(agentDir, "auth.json"), "utf8")) as {
[key: string]: unknown;
};
expect(parsed.openrouter).toBeUndefined();
expect(parsed["openai-codex"]).toMatchObject({
type: "oauth",
access: "oauth-access",
});
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
});
});

View File

@@ -1,19 +1,172 @@
import fs from "node:fs";
import path from "node:path";
import { AuthStorage, ModelRegistry } from "@mariozechner/pi-coding-agent";
import {
AuthStorage,
InMemoryAuthStorageBackend,
ModelRegistry,
} from "@mariozechner/pi-coding-agent";
import { ensureAuthProfileStore } from "./auth-profiles.js";
import type { AuthProfileCredential } from "./auth-profiles.js";
import { normalizeProviderId } from "./model-selection.js";
export { AuthStorage, ModelRegistry } from "@mariozechner/pi-coding-agent";
function createAuthStorage(AuthStorageLike: unknown, path: string) {
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
if (typeof withFactory.create === "function") {
return withFactory.create(path) as AuthStorage;
type PiApiKeyCredential = { type: "api_key"; key: string };
type PiOAuthCredential = {
type: "oauth";
access: string;
refresh: string;
expires: number;
};
type PiCredential = PiApiKeyCredential | PiOAuthCredential;
type PiCredentialMap = Record<string, PiCredential>;
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function scrubLegacyStaticAuthJsonEntries(pathname: string): void {
if (!fs.existsSync(pathname)) {
return;
}
return new (AuthStorageLike as { new (path: string): unknown })(path) as AuthStorage;
let parsed: unknown;
try {
parsed = JSON.parse(fs.readFileSync(pathname, "utf8")) as unknown;
} catch {
return;
}
if (!isRecord(parsed)) {
return;
}
let changed = false;
for (const [provider, value] of Object.entries(parsed)) {
if (!isRecord(value)) {
continue;
}
if (value.type !== "api_key") {
continue;
}
delete parsed[provider];
changed = true;
}
if (!changed) {
return;
}
if (Object.keys(parsed).length === 0) {
fs.rmSync(pathname, { force: true });
return;
}
fs.writeFileSync(pathname, `${JSON.stringify(parsed, null, 2)}\n`, "utf8");
fs.chmodSync(pathname, 0o600);
}
function createAuthStorage(AuthStorageLike: unknown, path: string, creds: PiCredentialMap) {
const withInMemory = AuthStorageLike as { inMemory?: (data?: unknown) => unknown };
if (typeof withInMemory.inMemory === "function") {
return withInMemory.inMemory(creds) as AuthStorage;
}
const withFromStorage = AuthStorageLike as {
fromStorage?: (storage: unknown) => unknown;
};
if (typeof withFromStorage.fromStorage === "function") {
const backend = new InMemoryAuthStorageBackend();
backend.withLock(() => ({
result: undefined,
next: JSON.stringify(creds, null, 2),
}));
return withFromStorage.fromStorage(backend) as AuthStorage;
}
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
const withRuntimeOverride = (
typeof withFactory.create === "function"
? withFactory.create(path)
: new (AuthStorageLike as { new (path: string): unknown })(path)
) as AuthStorage & {
setRuntimeApiKey?: (provider: string, apiKey: string) => void;
};
if (typeof withRuntimeOverride.setRuntimeApiKey === "function") {
for (const [provider, credential] of Object.entries(creds)) {
if (credential.type === "api_key") {
withRuntimeOverride.setRuntimeApiKey(provider, credential.key);
continue;
}
withRuntimeOverride.setRuntimeApiKey(provider, credential.access);
}
}
return withRuntimeOverride;
}
function convertAuthProfileCredential(cred: AuthProfileCredential): PiCredential | null {
if (cred.type === "api_key") {
const key = typeof cred.key === "string" ? cred.key.trim() : "";
if (!key) {
return null;
}
return { type: "api_key", key };
}
if (cred.type === "token") {
const token = typeof cred.token === "string" ? cred.token.trim() : "";
if (!token) {
return null;
}
if (
typeof cred.expires === "number" &&
Number.isFinite(cred.expires) &&
Date.now() >= cred.expires
) {
return null;
}
return { type: "api_key", key: token };
}
if (cred.type === "oauth") {
const access = typeof cred.access === "string" ? cred.access.trim() : "";
const refresh = typeof cred.refresh === "string" ? cred.refresh.trim() : "";
if (!access || !refresh || !Number.isFinite(cred.expires) || cred.expires <= 0) {
return null;
}
return {
type: "oauth",
access,
refresh,
expires: cred.expires,
};
}
return null;
}
function resolvePiCredentials(agentDir: string): PiCredentialMap {
const store = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false });
const credentials: PiCredentialMap = {};
for (const credential of Object.values(store.profiles)) {
const provider = normalizeProviderId(String(credential.provider ?? "")).trim();
if (!provider || credentials[provider]) {
continue;
}
const converted = convertAuthProfileCredential(credential);
if (converted) {
credentials[provider] = converted;
}
}
return credentials;
}
// Compatibility helpers for pi-coding-agent 0.50+ (discover* helpers removed).
export function discoverAuthStorage(agentDir: string): AuthStorage {
return createAuthStorage(AuthStorage, path.join(agentDir, "auth.json"));
const credentials = resolvePiCredentials(agentDir);
const authPath = path.join(agentDir, "auth.json");
scrubLegacyStaticAuthJsonEntries(authPath);
return createAuthStorage(AuthStorage, authPath, credentials);
}
export function discoverModels(authStorage: AuthStorage, agentDir: string): ModelRegistry {

View File

@@ -105,9 +105,10 @@ function applySkillConfigEnvOverrides(params: {
}
}
if (normalizedPrimaryEnv && skillConfig.apiKey && !process.env[normalizedPrimaryEnv]) {
const resolvedApiKey = typeof skillConfig.apiKey === "string" ? skillConfig.apiKey.trim() : "";
if (normalizedPrimaryEnv && resolvedApiKey && !process.env[normalizedPrimaryEnv]) {
if (!pendingOverrides[normalizedPrimaryEnv]) {
pendingOverrides[normalizedPrimaryEnv] = skillConfig.apiKey;
pendingOverrides[normalizedPrimaryEnv] = resolvedApiKey;
}
}

View File

@@ -260,6 +260,15 @@ const entries: SubCliEntry[] = [
mod.registerSecurityCli(program);
},
},
{
name: "secrets",
description: "Secrets runtime reload controls",
hasSubcommands: true,
register: async (program) => {
const mod = await import("../secrets-cli.js");
mod.registerSecretsCli(program);
},
},
{
name: "skills",
description: "List and inspect available skills",

View File

@@ -0,0 +1,96 @@
import { Command } from "commander";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { createCliRuntimeCapture } from "./test-runtime-capture.js";
const callGatewayFromCli = vi.fn();
const runSecretsMigration = vi.fn();
const rollbackSecretsMigration = vi.fn();
const { defaultRuntime, runtimeLogs, runtimeErrors, resetRuntimeCapture } =
createCliRuntimeCapture();
vi.mock("./gateway-rpc.js", () => ({
addGatewayClientOptions: (cmd: Command) => cmd,
callGatewayFromCli: (method: string, opts: unknown, params?: unknown, extra?: unknown) =>
callGatewayFromCli(method, opts, params, extra),
}));
vi.mock("../runtime.js", () => ({
defaultRuntime,
}));
vi.mock("../secrets/migrate.js", () => ({
runSecretsMigration: (options: unknown) => runSecretsMigration(options),
rollbackSecretsMigration: (options: unknown) => rollbackSecretsMigration(options),
}));
const { registerSecretsCli } = await import("./secrets-cli.js");
describe("secrets CLI", () => {
const createProgram = () => {
const program = new Command();
program.exitOverride();
registerSecretsCli(program);
return program;
};
beforeEach(() => {
resetRuntimeCapture();
callGatewayFromCli.mockReset();
runSecretsMigration.mockReset();
rollbackSecretsMigration.mockReset();
});
it("calls secrets.reload and prints human output", async () => {
callGatewayFromCli.mockResolvedValue({ ok: true, warningCount: 1 });
await createProgram().parseAsync(["secrets", "reload"], { from: "user" });
expect(callGatewayFromCli).toHaveBeenCalledWith(
"secrets.reload",
expect.anything(),
undefined,
expect.objectContaining({ expectFinal: false }),
);
expect(runtimeLogs.at(-1)).toBe("Secrets reloaded with 1 warning(s).");
expect(runtimeErrors).toHaveLength(0);
});
it("prints JSON when requested", async () => {
callGatewayFromCli.mockResolvedValue({ ok: true, warningCount: 0 });
await createProgram().parseAsync(["secrets", "reload", "--json"], { from: "user" });
expect(runtimeLogs.at(-1)).toContain('"ok": true');
});
it("runs secrets migrate as dry-run by default", async () => {
runSecretsMigration.mockResolvedValue({
mode: "dry-run",
changed: true,
secretsFilePath: "/tmp/secrets.enc.json",
counters: { secretsWritten: 3 },
changedFiles: ["/tmp/openclaw.json"],
});
await createProgram().parseAsync(["secrets", "migrate"], { from: "user" });
expect(runSecretsMigration).toHaveBeenCalledWith(
expect.objectContaining({ write: false, scrubEnv: true }),
);
expect(runtimeLogs.at(-1)).toContain("dry run");
});
it("runs rollback when --rollback is provided", async () => {
rollbackSecretsMigration.mockResolvedValue({
backupId: "20260221T010203Z",
restoredFiles: 2,
deletedFiles: 1,
});
await createProgram().parseAsync(["secrets", "migrate", "--rollback", "20260221T010203Z"], {
from: "user",
});
expect(rollbackSecretsMigration).toHaveBeenCalledWith({
backupId: "20260221T010203Z",
});
expect(runtimeLogs.at(-1)).toContain("rollback complete");
});
});

121
src/cli/secrets-cli.ts Normal file
View File

@@ -0,0 +1,121 @@
import type { Command } from "commander";
import { danger } from "../globals.js";
import { defaultRuntime } from "../runtime.js";
import {
rollbackSecretsMigration,
runSecretsMigration,
type SecretsMigrationRollbackResult,
type SecretsMigrationRunResult,
} from "../secrets/migrate.js";
import { formatDocsLink } from "../terminal/links.js";
import { theme } from "../terminal/theme.js";
import { addGatewayClientOptions, callGatewayFromCli, type GatewayRpcOpts } from "./gateway-rpc.js";
type SecretsReloadOptions = GatewayRpcOpts & { json?: boolean };
type SecretsMigrateOptions = {
write?: boolean;
rollback?: string;
scrubEnv?: boolean;
json?: boolean;
};
function printMigrationResult(
result: SecretsMigrationRunResult | SecretsMigrationRollbackResult,
json: boolean,
): void {
if (json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
if ("restoredFiles" in result) {
defaultRuntime.log(
`Secrets rollback complete for backup ${result.backupId}. Restored ${result.restoredFiles} file(s), deleted ${result.deletedFiles} file(s).`,
);
return;
}
if (result.mode === "dry-run") {
if (!result.changed) {
defaultRuntime.log("Secrets migrate dry run: no changes needed.");
return;
}
defaultRuntime.log(
`Secrets migrate dry run: ${result.changedFiles.length} file(s) would change, ${result.counters.secretsWritten} secret value(s) would move to ${result.secretsFilePath}.`,
);
return;
}
if (!result.changed) {
defaultRuntime.log("Secrets migrate: no changes applied.");
return;
}
defaultRuntime.log(
`Secrets migrated. Backup: ${result.backupId}. Moved ${result.counters.secretsWritten} secret value(s) into ${result.secretsFilePath}.`,
);
}
export function registerSecretsCli(program: Command) {
const secrets = program
.command("secrets")
.description("Secrets runtime controls")
.addHelpText(
"after",
() =>
`\n${theme.muted("Docs:")} ${formatDocsLink("/gateway/security", "docs.openclaw.ai/gateway/security")}\n`,
);
addGatewayClientOptions(
secrets
.command("reload")
.description("Re-resolve secret references and atomically swap runtime snapshot")
.option("--json", "Output JSON", false),
).action(async (opts: SecretsReloadOptions) => {
try {
const result = await callGatewayFromCli("secrets.reload", opts, undefined, {
expectFinal: false,
});
if (opts.json) {
defaultRuntime.log(JSON.stringify(result, null, 2));
return;
}
const warningCount = Number(
(result as { warningCount?: unknown } | undefined)?.warningCount ?? 0,
);
if (Number.isFinite(warningCount) && warningCount > 0) {
defaultRuntime.log(`Secrets reloaded with ${warningCount} warning(s).`);
return;
}
defaultRuntime.log("Secrets reloaded.");
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
secrets
.command("migrate")
.description("Migrate plaintext secrets to file-backed SecretRefs (sops)")
.option("--write", "Apply migration changes (default is dry-run)", false)
.option("--rollback <backup-id>", "Rollback a previous migration backup id")
.option("--no-scrub-env", "Keep matching plaintext values in ~/.openclaw/.env")
.option("--json", "Output JSON", false)
.action(async (opts: SecretsMigrateOptions) => {
try {
if (typeof opts.rollback === "string" && opts.rollback.trim()) {
const result = await rollbackSecretsMigration({ backupId: opts.rollback.trim() });
printMigrationResult(result, Boolean(opts.json));
return;
}
const result = await runSecretsMigration({
write: Boolean(opts.write),
scrubEnv: opts.scrubEnv ?? true,
});
printMigrationResult(result, Boolean(opts.json));
} catch (err) {
defaultRuntime.error(danger(String(err)));
defaultRuntime.exit(1);
}
});
}

View File

@@ -0,0 +1,89 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { applyAuthChoiceOpenAI } from "./auth-choice.apply.openai.js";
import {
createAuthTestLifecycle,
createExitThrowingRuntime,
createWizardPrompter,
readAuthProfilesForAgent,
setupAuthTestEnv,
} from "./test-wizard-helpers.js";
describe("applyAuthChoiceOpenAI", () => {
const lifecycle = createAuthTestLifecycle([
"OPENCLAW_STATE_DIR",
"OPENCLAW_AGENT_DIR",
"PI_CODING_AGENT_DIR",
"OPENAI_API_KEY",
]);
async function setupTempState() {
const env = await setupAuthTestEnv("openclaw-openai-");
lifecycle.setStateDir(env.stateDir);
return env.agentDir;
}
afterEach(async () => {
await lifecycle.cleanup();
});
it("writes env-backed OpenAI key as keyRef in auth profiles", async () => {
const agentDir = await setupTempState();
process.env.OPENAI_API_KEY = "sk-openai-env";
const confirm = vi.fn(async () => true);
const text = vi.fn(async () => "unused");
const prompter = createWizardPrompter({ confirm, text }, { defaultSelect: "" });
const runtime = createExitThrowingRuntime();
const result = await applyAuthChoiceOpenAI({
authChoice: "openai-api-key",
config: {},
prompter,
runtime,
setDefaultModel: true,
});
expect(result).not.toBeNull();
expect(result?.config.auth?.profiles?.["openai:default"]).toMatchObject({
provider: "openai",
mode: "api_key",
});
expect(result?.config.agents?.defaults?.model?.primary).toBe("openai/gpt-5.1-codex");
expect(text).not.toHaveBeenCalled();
const parsed = await readAuthProfilesForAgent<{
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(agentDir);
expect(parsed.profiles?.["openai:default"]).toMatchObject({
keyRef: { source: "env", id: "OPENAI_API_KEY" },
});
expect(parsed.profiles?.["openai:default"]?.key).toBeUndefined();
});
it("writes explicit token input into openai auth profile", async () => {
const agentDir = await setupTempState();
const prompter = createWizardPrompter({}, { defaultSelect: "" });
const runtime = createExitThrowingRuntime();
const result = await applyAuthChoiceOpenAI({
authChoice: "apiKey",
config: {},
prompter,
runtime,
setDefaultModel: true,
opts: {
tokenProvider: "openai",
token: "sk-openai-token",
},
});
expect(result).not.toBeNull();
const parsed = await readAuthProfilesForAgent<{
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(agentDir);
expect(parsed.profiles?.["openai:default"]?.key).toBe("sk-openai-token");
expect(parsed.profiles?.["openai:default"]?.keyRef).toBeUndefined();
});
});

View File

@@ -1,5 +1,4 @@
import { resolveEnvApiKey } from "../agents/model-auth.js";
import { upsertSharedEnvVar } from "../infra/env-file.js";
import {
formatApiKeyPreview,
normalizeApiKeyInput,
@@ -9,7 +8,7 @@ import { createAuthChoiceAgentModelNoter } from "./auth-choice.apply-helpers.js"
import type { ApplyAuthChoiceParams, ApplyAuthChoiceResult } from "./auth-choice.apply.js";
import { applyDefaultModelChoice } from "./auth-choice.default-model.js";
import { isRemoteEnvironment } from "./oauth-env.js";
import { applyAuthProfileConfig, writeOAuthCredentials } from "./onboard-auth.js";
import { applyAuthProfileConfig, setOpenaiApiKey, writeOAuthCredentials } from "./onboard-auth.js";
import { openUrl } from "./onboard-helpers.js";
import {
applyOpenAICodexModelDefault,
@@ -58,17 +57,12 @@ export async function applyAuthChoiceOpenAI(
initialValue: true,
});
if (useExisting) {
const result = upsertSharedEnvVar({
key: "OPENAI_API_KEY",
value: envKey.apiKey,
await setOpenaiApiKey(envKey.apiKey, params.agentDir);
nextConfig = applyAuthProfileConfig(nextConfig, {
profileId: "openai:default",
provider: "openai",
mode: "api_key",
});
if (!process.env.OPENAI_API_KEY) {
process.env.OPENAI_API_KEY = envKey.apiKey;
}
await params.prompter.note(
`Copied OPENAI_API_KEY to ${result.path} for launchd compatibility.`,
"OpenAI API key",
);
return await applyOpenAiDefaultModelChoice();
}
}
@@ -84,15 +78,12 @@ export async function applyAuthChoiceOpenAI(
}
const trimmed = normalizeApiKeyInput(String(key));
const result = upsertSharedEnvVar({
key: "OPENAI_API_KEY",
value: trimmed,
await setOpenaiApiKey(trimmed, params.agentDir);
nextConfig = applyAuthProfileConfig(nextConfig, {
profileId: "openai:default",
provider: "openai",
mode: "api_key",
});
process.env.OPENAI_API_KEY = trimmed;
await params.prompter.note(
`Saved OPENAI_API_KEY to ${result.path} for launchd compatibility.`,
"OpenAI API key",
);
return await applyOpenAiDefaultModelChoice();
}

View File

@@ -95,7 +95,7 @@ describe("models list auth-profile sync", () => {
const openrouter = getProviderRow(String(runtime.log.mock.calls[0]?.[0]), "openrouter/");
expect(openrouter).toBeDefined();
expect(openrouter?.available).toBe(true);
expect(await pathExists(authPath)).toBe(true);
expect(await pathExists(authPath)).toBe(false);
});
});

View File

@@ -6,9 +6,6 @@ let toModelRow: typeof import("./models/list.registry.js").toModelRow;
const loadConfig = vi.fn();
const ensureOpenClawModelsJson = vi.fn().mockResolvedValue(undefined);
const ensurePiAuthJsonFromAuthProfiles = vi
.fn()
.mockResolvedValue({ wrote: false, authPath: "/tmp/openclaw-agent/auth.json" });
const resolveOpenClawAgentDir = vi.fn().mockReturnValue("/tmp/openclaw-agent");
const ensureAuthProfileStore = vi.fn().mockReturnValue({ version: 1, profiles: {} });
const listProfilesForProvider = vi.fn().mockReturnValue([]);
@@ -38,10 +35,6 @@ vi.mock("../agents/models-config.js", () => ({
ensureOpenClawModelsJson,
}));
vi.mock("../agents/pi-auth-json.js", () => ({
ensurePiAuthJsonFromAuthProfiles,
}));
vi.mock("../agents/agent-paths.js", () => ({
resolveOpenClawAgentDir,
}));
@@ -110,7 +103,6 @@ beforeEach(() => {
modelRegistryState.getAllError = undefined;
modelRegistryState.getAvailableError = undefined;
listProfilesForProvider.mockReturnValue([]);
ensurePiAuthJsonFromAuthProfiles.mockClear();
});
afterEach(() => {
@@ -279,13 +271,12 @@ describe("models list/status", () => {
({ loadModelRegistry, toModelRow } = await import("./models/list.registry.js"));
});
it("models list syncs auth-profiles into auth.json before availability checks", async () => {
it("models list runs model discovery without auth.json sync", async () => {
setDefaultZaiRegistry();
const runtime = makeRuntime();
await modelsListCommand({ all: true, json: true }, runtime);
expect(ensurePiAuthJsonFromAuthProfiles).toHaveBeenCalledWith("/tmp/openclaw-agent");
expect(runtime.error).not.toHaveBeenCalled();
});
it("models list outputs canonical zai key for configured z.ai model", async () => {

View File

@@ -12,7 +12,6 @@ import {
resolveForwardCompatModel,
} from "../../agents/model-forward-compat.js";
import { ensureOpenClawModelsJson } from "../../agents/models-config.js";
import { ensurePiAuthJsonFromAuthProfiles } from "../../agents/pi-auth-json.js";
import type { ModelRegistry } from "../../agents/pi-model-discovery.js";
import { discoverAuthStorage, discoverModels } from "../../agents/pi-model-discovery.js";
import type { OpenClawConfig } from "../../config/config.js";
@@ -102,7 +101,6 @@ function loadAvailableModels(registry: ModelRegistry): Model<Api>[] {
export async function loadModelRegistry(cfg: OpenClawConfig) {
await ensureOpenClawModelsJson(cfg);
const agentDir = resolveOpenClawAgentDir();
await ensurePiAuthJsonFromAuthProfiles(agentDir);
const authStorage = discoverAuthStorage(agentDir);
const registry = discoverModels(authStorage, agentDir);
const appended = appendAntigravityForwardCompatModels(registry.getAll(), registry);

View File

@@ -0,0 +1,106 @@
import { afterEach, describe, expect, it } from "vitest";
import {
setCloudflareAiGatewayConfig,
setMoonshotApiKey,
setOpenaiApiKey,
} from "./onboard-auth.js";
import {
createAuthTestLifecycle,
readAuthProfilesForAgent,
setupAuthTestEnv,
} from "./test-wizard-helpers.js";
describe("onboard auth credentials secret refs", () => {
const lifecycle = createAuthTestLifecycle([
"OPENCLAW_STATE_DIR",
"OPENCLAW_AGENT_DIR",
"PI_CODING_AGENT_DIR",
"MOONSHOT_API_KEY",
"OPENAI_API_KEY",
"CLOUDFLARE_AI_GATEWAY_API_KEY",
]);
afterEach(async () => {
await lifecycle.cleanup();
});
it("stores env-backed moonshot key as keyRef", async () => {
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-");
lifecycle.setStateDir(env.stateDir);
process.env.MOONSHOT_API_KEY = "sk-moonshot-env";
await setMoonshotApiKey("sk-moonshot-env");
const parsed = await readAuthProfilesForAgent<{
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["moonshot:default"]).toMatchObject({
keyRef: { source: "env", id: "MOONSHOT_API_KEY" },
});
expect(parsed.profiles?.["moonshot:default"]?.key).toBeUndefined();
});
it("stores ${ENV} moonshot input as keyRef even when env value is unset", async () => {
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-inline-ref-");
lifecycle.setStateDir(env.stateDir);
await setMoonshotApiKey("${MOONSHOT_API_KEY}");
const parsed = await readAuthProfilesForAgent<{
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["moonshot:default"]).toMatchObject({
keyRef: { source: "env", id: "MOONSHOT_API_KEY" },
});
expect(parsed.profiles?.["moonshot:default"]?.key).toBeUndefined();
});
it("keeps plaintext moonshot key when no env ref applies", async () => {
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-plaintext-");
lifecycle.setStateDir(env.stateDir);
process.env.MOONSHOT_API_KEY = "sk-moonshot-other";
await setMoonshotApiKey("sk-moonshot-plaintext");
const parsed = await readAuthProfilesForAgent<{
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["moonshot:default"]).toMatchObject({
key: "sk-moonshot-plaintext",
});
expect(parsed.profiles?.["moonshot:default"]?.keyRef).toBeUndefined();
});
it("preserves cloudflare metadata when storing keyRef", async () => {
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-cloudflare-");
lifecycle.setStateDir(env.stateDir);
process.env.CLOUDFLARE_AI_GATEWAY_API_KEY = "cf-secret";
await setCloudflareAiGatewayConfig("account-1", "gateway-1", "cf-secret");
const parsed = await readAuthProfilesForAgent<{
profiles?: Record<string, { key?: string; keyRef?: unknown; metadata?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["cloudflare-ai-gateway:default"]).toMatchObject({
keyRef: { source: "env", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
metadata: { accountId: "account-1", gatewayId: "gateway-1" },
});
expect(parsed.profiles?.["cloudflare-ai-gateway:default"]?.key).toBeUndefined();
});
it("stores env-backed openai key as keyRef", async () => {
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-openai-");
lifecycle.setStateDir(env.stateDir);
process.env.OPENAI_API_KEY = "sk-openai-env";
await setOpenaiApiKey("sk-openai-env");
const parsed = await readAuthProfilesForAgent<{
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
}>(env.agentDir);
expect(parsed.profiles?.["openai:default"]).toMatchObject({
keyRef: { source: "env", id: "OPENAI_API_KEY" },
});
expect(parsed.profiles?.["openai:default"]?.key).toBeUndefined();
});
});

View File

@@ -4,11 +4,85 @@ import type { OAuthCredentials } from "@mariozechner/pi-ai";
import { resolveOpenClawAgentDir } from "../agents/agent-paths.js";
import { upsertAuthProfile } from "../agents/auth-profiles.js";
import { resolveStateDir } from "../config/paths.js";
import { isSecretRef, type SecretInput, type SecretRef } from "../config/types.secrets.js";
import { PROVIDER_ENV_VARS } from "../secrets/provider-env-vars.js";
import { normalizeSecretInput } from "../utils/normalize-secret-input.js";
export { CLOUDFLARE_AI_GATEWAY_DEFAULT_MODEL_REF } from "../agents/cloudflare-ai-gateway.js";
export { XAI_DEFAULT_MODEL_REF } from "./onboard-auth.models.js";
const resolveAuthAgentDir = (agentDir?: string) => agentDir ?? resolveOpenClawAgentDir();
const ENV_REF_PATTERN = /^\$\{([A-Z][A-Z0-9_]*)\}$/;
function buildEnvSecretRef(id: string): SecretRef {
return { source: "env", id };
}
function parseEnvSecretRef(value: string): SecretRef | null {
const match = ENV_REF_PATTERN.exec(value);
if (!match) {
return null;
}
return buildEnvSecretRef(match[1]);
}
function inferProviderEnvSecretRef(provider: string, value: string): SecretRef | null {
const envVars = PROVIDER_ENV_VARS[provider];
if (!envVars || value.length === 0) {
return null;
}
for (const envVar of envVars) {
const envValue = normalizeSecretInput(process.env[envVar] ?? "");
if (envValue && envValue === value) {
return buildEnvSecretRef(envVar);
}
}
return null;
}
function resolveApiKeySecretInput(provider: string, input: SecretInput): SecretInput {
if (isSecretRef(input)) {
return input;
}
const normalized = normalizeSecretInput(input);
const inlineEnvRef = parseEnvSecretRef(normalized);
if (inlineEnvRef) {
return inlineEnvRef;
}
const inferredEnvRef = inferProviderEnvSecretRef(provider, normalized);
if (inferredEnvRef) {
return inferredEnvRef;
}
return normalized;
}
function buildApiKeyCredential(
provider: string,
input: SecretInput,
metadata?: Record<string, string>,
): {
type: "api_key";
provider: string;
key?: string;
keyRef?: SecretRef;
metadata?: Record<string, string>;
} {
const secretInput = resolveApiKeySecretInput(provider, input);
if (typeof secretInput === "string") {
return {
type: "api_key",
provider,
key: secretInput,
...(metadata ? { metadata } : {}),
};
}
return {
type: "api_key",
provider,
keyRef: secretInput,
...(metadata ? { metadata } : {}),
};
}
export type WriteOAuthCredentialsOptions = {
syncSiblingAgents?: boolean;
};
@@ -110,34 +184,34 @@ export async function writeOAuthCredentials(
return profileId;
}
export async function setAnthropicApiKey(key: string, agentDir?: string) {
export async function setAnthropicApiKey(key: SecretInput, agentDir?: string) {
// Write to resolved agent dir so gateway finds credentials on startup.
upsertAuthProfile({
profileId: "anthropic:default",
credential: {
type: "api_key",
provider: "anthropic",
key,
},
credential: buildApiKeyCredential("anthropic", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setGeminiApiKey(key: string, agentDir?: string) {
export async function setOpenaiApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "openai:default",
credential: buildApiKeyCredential("openai", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setGeminiApiKey(key: SecretInput, agentDir?: string) {
// Write to resolved agent dir so gateway finds credentials on startup.
upsertAuthProfile({
profileId: "google:default",
credential: {
type: "api_key",
provider: "google",
key,
},
credential: buildApiKeyCredential("google", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setMinimaxApiKey(
key: string,
key: SecretInput,
agentDir?: string,
profileId: string = "minimax:default",
) {
@@ -145,63 +219,43 @@ export async function setMinimaxApiKey(
// Write to resolved agent dir so gateway finds credentials on startup.
upsertAuthProfile({
profileId,
credential: {
type: "api_key",
provider,
key,
},
credential: buildApiKeyCredential(provider, key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setMoonshotApiKey(key: string, agentDir?: string) {
export async function setMoonshotApiKey(key: SecretInput, agentDir?: string) {
// Write to resolved agent dir so gateway finds credentials on startup.
upsertAuthProfile({
profileId: "moonshot:default",
credential: {
type: "api_key",
provider: "moonshot",
key,
},
credential: buildApiKeyCredential("moonshot", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setKimiCodingApiKey(key: string, agentDir?: string) {
export async function setKimiCodingApiKey(key: SecretInput, agentDir?: string) {
// Write to resolved agent dir so gateway finds credentials on startup.
upsertAuthProfile({
profileId: "kimi-coding:default",
credential: {
type: "api_key",
provider: "kimi-coding",
key,
},
credential: buildApiKeyCredential("kimi-coding", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setSyntheticApiKey(key: string, agentDir?: string) {
export async function setSyntheticApiKey(key: SecretInput, agentDir?: string) {
// Write to resolved agent dir so gateway finds credentials on startup.
upsertAuthProfile({
profileId: "synthetic:default",
credential: {
type: "api_key",
provider: "synthetic",
key,
},
credential: buildApiKeyCredential("synthetic", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setVeniceApiKey(key: string, agentDir?: string) {
export async function setVeniceApiKey(key: SecretInput, agentDir?: string) {
// Write to resolved agent dir so gateway finds credentials on startup.
upsertAuthProfile({
profileId: "venice:default",
credential: {
type: "api_key",
provider: "venice",
key,
},
credential: buildApiKeyCredential("venice", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
@@ -214,41 +268,29 @@ export const TOGETHER_DEFAULT_MODEL_REF = "together/moonshotai/Kimi-K2.5";
export const LITELLM_DEFAULT_MODEL_REF = "litellm/claude-opus-4-6";
export const VERCEL_AI_GATEWAY_DEFAULT_MODEL_REF = "vercel-ai-gateway/anthropic/claude-opus-4.6";
export async function setZaiApiKey(key: string, agentDir?: string) {
export async function setZaiApiKey(key: SecretInput, agentDir?: string) {
// Write to resolved agent dir so gateway finds credentials on startup.
upsertAuthProfile({
profileId: "zai:default",
credential: {
type: "api_key",
provider: "zai",
key,
},
credential: buildApiKeyCredential("zai", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setXiaomiApiKey(key: string, agentDir?: string) {
export async function setXiaomiApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "xiaomi:default",
credential: {
type: "api_key",
provider: "xiaomi",
key,
},
credential: buildApiKeyCredential("xiaomi", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setOpenrouterApiKey(key: string, agentDir?: string) {
export async function setOpenrouterApiKey(key: SecretInput, agentDir?: string) {
// Never persist the literal "undefined" (e.g. when prompt returns undefined and caller used String(key)).
const safeKey = key === "undefined" ? "" : key;
const safeKey = typeof key === "string" && key === "undefined" ? "" : key;
upsertAuthProfile({
profileId: "openrouter:default",
credential: {
type: "api_key",
provider: "openrouter",
key: safeKey,
},
credential: buildApiKeyCredential("openrouter", safeKey),
agentDir: resolveAuthAgentDir(agentDir),
});
}
@@ -256,107 +298,73 @@ export async function setOpenrouterApiKey(key: string, agentDir?: string) {
export async function setCloudflareAiGatewayConfig(
accountId: string,
gatewayId: string,
apiKey: string,
apiKey: SecretInput,
agentDir?: string,
) {
const normalizedAccountId = accountId.trim();
const normalizedGatewayId = gatewayId.trim();
const normalizedKey = apiKey.trim();
upsertAuthProfile({
profileId: "cloudflare-ai-gateway:default",
credential: {
type: "api_key",
provider: "cloudflare-ai-gateway",
key: normalizedKey,
metadata: {
accountId: normalizedAccountId,
gatewayId: normalizedGatewayId,
},
},
credential: buildApiKeyCredential("cloudflare-ai-gateway", apiKey, {
accountId: normalizedAccountId,
gatewayId: normalizedGatewayId,
}),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setLitellmApiKey(key: string, agentDir?: string) {
export async function setLitellmApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "litellm:default",
credential: {
type: "api_key",
provider: "litellm",
key,
},
credential: buildApiKeyCredential("litellm", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setVercelAiGatewayApiKey(key: string, agentDir?: string) {
export async function setVercelAiGatewayApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "vercel-ai-gateway:default",
credential: {
type: "api_key",
provider: "vercel-ai-gateway",
key,
},
credential: buildApiKeyCredential("vercel-ai-gateway", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setOpencodeZenApiKey(key: string, agentDir?: string) {
export async function setOpencodeZenApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "opencode:default",
credential: {
type: "api_key",
provider: "opencode",
key,
},
credential: buildApiKeyCredential("opencode", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setTogetherApiKey(key: string, agentDir?: string) {
export async function setTogetherApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "together:default",
credential: {
type: "api_key",
provider: "together",
key,
},
credential: buildApiKeyCredential("together", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export async function setHuggingfaceApiKey(key: string, agentDir?: string) {
export async function setHuggingfaceApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "huggingface:default",
credential: {
type: "api_key",
provider: "huggingface",
key,
},
credential: buildApiKeyCredential("huggingface", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export function setQianfanApiKey(key: string, agentDir?: string) {
export function setQianfanApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "qianfan:default",
credential: {
type: "api_key",
provider: "qianfan",
key,
},
credential: buildApiKeyCredential("qianfan", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}
export function setXaiApiKey(key: string, agentDir?: string) {
export function setXaiApiKey(key: SecretInput, agentDir?: string) {
upsertAuthProfile({
profileId: "xai:default",
credential: {
type: "api_key",
provider: "xai",
key,
},
credential: buildApiKeyCredential("xai", key),
agentDir: resolveAuthAgentDir(agentDir),
});
}

View File

@@ -55,6 +55,7 @@ export {
CLOUDFLARE_AI_GATEWAY_DEFAULT_MODEL_REF,
LITELLM_DEFAULT_MODEL_REF,
OPENROUTER_DEFAULT_MODEL_REF,
setOpenaiApiKey,
setAnthropicApiKey,
setCloudflareAiGatewayConfig,
setQianfanApiKey,

View File

@@ -38,6 +38,7 @@ import {
setLitellmApiKey,
setMinimaxApiKey,
setMoonshotApiKey,
setOpenaiApiKey,
setOpencodeZenApiKey,
setOpenrouterApiKey,
setSyntheticApiKey,
@@ -381,15 +382,16 @@ export async function applyNonInteractiveAuthChoice(params: {
flagName: "--openai-api-key",
envVar: "OPENAI_API_KEY",
runtime,
allowProfile: false,
});
if (!resolved) {
return null;
}
const key = resolved.key;
const result = upsertSharedEnvVar({ key: "OPENAI_API_KEY", value: key });
process.env.OPENAI_API_KEY = key;
runtime.log(`Saved OPENAI_API_KEY to ${shortenHomePath(result.path)}`);
await setOpenaiApiKey(resolved.key);
nextConfig = applyAuthProfileConfig(nextConfig, {
profileId: "openai:default",
provider: "openai",
mode: "api_key",
});
return applyOpenAIConfig(nextConfig);
}

View File

@@ -36,6 +36,21 @@ describe("config secret refs schema", () => {
expect(result.ok).toBe(true);
});
it("accepts skills entry apiKey refs", () => {
const result = validateConfigObjectRaw({
skills: {
entries: {
"review-pr": {
enabled: true,
apiKey: { source: "env", id: "SKILL_REVIEW_PR_API_KEY" },
},
},
},
});
expect(result.ok).toBe(true);
});
it("rejects invalid secret ref id", () => {
const result = validateConfigObjectRaw({
models: {

View File

@@ -1,6 +1,8 @@
import type { SecretInput } from "./types.secrets.js";
export type SkillConfig = {
enabled?: boolean;
apiKey?: string;
apiKey?: SecretInput;
env?: Record<string, string>;
config?: Record<string, unknown>;
};

View File

@@ -2,7 +2,12 @@ import { z } from "zod";
import { ToolsSchema } from "./zod-schema.agent-runtime.js";
import { AgentsSchema, AudioSchema, BindingsSchema, BroadcastSchema } from "./zod-schema.agents.js";
import { ApprovalsSchema } from "./zod-schema.approvals.js";
import { HexColorSchema, ModelsConfigSchema, SecretsConfigSchema } from "./zod-schema.core.js";
import {
HexColorSchema,
ModelsConfigSchema,
SecretInputSchema,
SecretsConfigSchema,
} from "./zod-schema.core.js";
import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-schema.hooks.js";
import { InstallRecordShape } from "./zod-schema.installs.js";
import { ChannelsSchema } from "./zod-schema.providers.js";
@@ -613,7 +618,7 @@ export const OpenClawSchema = z
z
.object({
enabled: z.boolean().optional(),
apiKey: z.string().optional().register(sensitive),
apiKey: SecretInputSchema.optional().register(sensitive),
env: z.record(z.string(), z.string()).optional(),
config: z.record(z.string(), z.unknown()).optional(),
})

View File

@@ -97,6 +97,7 @@ const METHOD_SCOPE_GROUPS: Record<OperatorScope, readonly string[]> = {
"agents.delete",
"skills.install",
"skills.update",
"secrets.reload",
"cron.add",
"cron.update",
"cron.remove",

View File

@@ -48,6 +48,7 @@ const BASE_METHODS = [
"update.run",
"voicewake.get",
"voicewake.set",
"secrets.reload",
"sessions.list",
"sessions.preview",
"sessions.patch",

View File

@@ -0,0 +1,43 @@
import { describe, expect, it, vi } from "vitest";
import { createSecretsHandlers } from "./secrets.js";
describe("secrets handlers", () => {
it("responds with warning count on successful reload", async () => {
const handlers = createSecretsHandlers({
reloadSecrets: vi.fn().mockResolvedValue({ warningCount: 2 }),
});
const respond = vi.fn();
await handlers["secrets.reload"]({
req: { type: "req", id: "1", method: "secrets.reload" },
params: {},
client: null,
isWebchatConnect: () => false,
respond,
context: {} as never,
});
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 2 });
});
it("returns unavailable when reload fails", async () => {
const handlers = createSecretsHandlers({
reloadSecrets: vi.fn().mockRejectedValue(new Error("reload failed")),
});
const respond = vi.fn();
await handlers["secrets.reload"]({
req: { type: "req", id: "1", method: "secrets.reload" },
params: {},
client: null,
isWebchatConnect: () => false,
respond,
context: {} as never,
});
expect(respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({
code: "UNAVAILABLE",
message: "Error: reload failed",
}),
);
});
});

View File

@@ -0,0 +1,17 @@
import { ErrorCodes, errorShape } from "../protocol/index.js";
import type { GatewayRequestHandlers } from "./types.js";
export function createSecretsHandlers(params: {
reloadSecrets: () => Promise<{ warningCount: number }>;
}): GatewayRequestHandlers {
return {
"secrets.reload": async ({ respond }) => {
try {
const result = await params.reloadSecrets();
respond(true, { ok: true, warningCount: result.warningCount });
} catch (err) {
respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, String(err)));
}
},
};
}

View File

@@ -75,6 +75,7 @@ import { GATEWAY_EVENTS, listGatewayMethods } from "./server-methods-list.js";
import { coreGatewayHandlers } from "./server-methods.js";
import { createExecApprovalHandlers } from "./server-methods/exec-approval.js";
import { safeParseJson } from "./server-methods/nodes.helpers.js";
import { createSecretsHandlers } from "./server-methods/secrets.js";
import { hasConnectedMobileNode } from "./server-mobile-nodes.js";
import { loadGatewayModelCatalog } from "./server-model-catalog.js";
import { createNodeSubscriptionManager } from "./server-node-subscriptions.js";
@@ -630,6 +631,19 @@ export async function startGatewayServer(
const execApprovalHandlers = createExecApprovalHandlers(execApprovalManager, {
forwarder: execApprovalForwarder,
});
const secretsHandlers = createSecretsHandlers({
reloadSecrets: async () => {
const active = getActiveSecretsRuntimeSnapshot();
if (!active) {
throw new Error("Secrets runtime snapshot is not active.");
}
const prepared = await activateRuntimeSecrets(active.sourceConfig, {
reason: "reload",
activate: true,
});
return { warningCount: prepared.warnings.length };
},
});
const canvasHostServerPort = (canvasHostServer as CanvasHostServer | null)?.port;
@@ -650,6 +664,7 @@ export async function startGatewayServer(
extraHandlers: {
...pluginRegistry.gatewayHandlers,
...execApprovalHandlers,
...secretsHandlers,
},
broadcast,
context: {

204
src/secrets/migrate.test.ts Normal file
View File

@@ -0,0 +1,204 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const runExecMock = vi.hoisted(() => vi.fn());
vi.mock("../process/exec.js", () => ({
runExec: runExecMock,
}));
const { rollbackSecretsMigration, runSecretsMigration } = await import("./migrate.js");
describe("secrets migrate", () => {
let baseDir = "";
let stateDir = "";
let configPath = "";
let env: NodeJS.ProcessEnv;
let authStorePath = "";
let envPath = "";
beforeEach(async () => {
baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-migrate-"));
stateDir = path.join(baseDir, ".openclaw");
configPath = path.join(stateDir, "openclaw.json");
authStorePath = path.join(stateDir, "agents", "main", "agent", "auth-profiles.json");
envPath = path.join(stateDir, ".env");
env = {
...process.env,
OPENCLAW_STATE_DIR: stateDir,
OPENCLAW_CONFIG_PATH: configPath,
};
await fs.mkdir(path.dirname(configPath), { recursive: true });
await fs.mkdir(path.dirname(authStorePath), { recursive: true });
await fs.writeFile(
configPath,
`${JSON.stringify(
{
models: {
providers: {
openai: {
baseUrl: "https://api.openai.com/v1",
apiKey: "sk-openai-plaintext",
models: [{ id: "gpt-5", name: "gpt-5" }],
},
},
},
skills: {
entries: {
"review-pr": {
enabled: true,
apiKey: "sk-skill-plaintext",
},
},
},
channels: {
googlechat: {
serviceAccount: '{"type":"service_account","client_email":"bot@example.com"}',
},
},
},
null,
2,
)}\n`,
"utf8",
);
await fs.writeFile(
authStorePath,
`${JSON.stringify(
{
version: 1,
profiles: {
"openai:default": {
type: "api_key",
provider: "openai",
key: "sk-profile-plaintext",
},
},
},
null,
2,
)}\n`,
"utf8",
);
await fs.writeFile(
envPath,
"OPENAI_API_KEY=sk-openai-plaintext\nSKILL_KEY=sk-skill-plaintext\nUNRELATED=value\n",
"utf8",
);
runExecMock.mockReset();
runExecMock.mockImplementation(async (_cmd: string, args: string[]) => {
if (args[0] === "--encrypt") {
const outputPath = args[args.indexOf("--output") + 1];
const inputPath = args.at(-1);
if (!outputPath || !inputPath) {
throw new Error("missing sops encrypt paths");
}
await fs.copyFile(inputPath, outputPath);
return { stdout: "", stderr: "" };
}
if (args[0] === "--decrypt") {
const sourcePath = args.at(-1);
if (!sourcePath) {
throw new Error("missing sops decrypt source");
}
const raw = await fs.readFile(sourcePath, "utf8");
return { stdout: raw, stderr: "" };
}
throw new Error(`unexpected sops invocation: ${args.join(" ")}`);
});
});
afterEach(async () => {
await fs.rm(baseDir, { recursive: true, force: true });
});
it("reports a dry-run without mutating files", async () => {
const beforeConfig = await fs.readFile(configPath, "utf8");
const beforeAuthStore = await fs.readFile(authStorePath, "utf8");
const result = await runSecretsMigration({ env });
expect(result.mode).toBe("dry-run");
expect(result.changed).toBe(true);
expect(result.counters.secretsWritten).toBeGreaterThanOrEqual(3);
expect(await fs.readFile(configPath, "utf8")).toBe(beforeConfig);
expect(await fs.readFile(authStorePath, "utf8")).toBe(beforeAuthStore);
});
it("migrates plaintext to file-backed refs and can rollback", async () => {
const applyResult = await runSecretsMigration({ env, write: true });
expect(applyResult.mode).toBe("write");
expect(applyResult.changed).toBe(true);
expect(applyResult.backupId).toBeTruthy();
const migratedConfig = JSON.parse(await fs.readFile(configPath, "utf8")) as {
models: { providers: { openai: { apiKey: unknown } } };
skills: { entries: { "review-pr": { apiKey: unknown } } };
channels: { googlechat: { serviceAccount?: unknown; serviceAccountRef?: unknown } };
secrets: { sources: { file: { type: string; path: string } } };
};
expect(migratedConfig.models.providers.openai.apiKey).toEqual({
source: "file",
id: "/providers/openai/apiKey",
});
expect(migratedConfig.skills.entries["review-pr"].apiKey).toEqual({
source: "file",
id: "/skills/entries/review-pr/apiKey",
});
expect(migratedConfig.channels.googlechat.serviceAccount).toBeUndefined();
expect(migratedConfig.channels.googlechat.serviceAccountRef).toEqual({
source: "file",
id: "/channels/googlechat/serviceAccount",
});
expect(migratedConfig.secrets.sources.file.type).toBe("sops");
const migratedAuth = JSON.parse(await fs.readFile(authStorePath, "utf8")) as {
profiles: { "openai:default": { key?: string; keyRef?: unknown } };
};
expect(migratedAuth.profiles["openai:default"].key).toBeUndefined();
expect(migratedAuth.profiles["openai:default"].keyRef).toEqual({
source: "file",
id: "/auth-profiles/main/openai:default/key",
});
const migratedEnv = await fs.readFile(envPath, "utf8");
expect(migratedEnv).not.toContain("sk-openai-plaintext");
expect(migratedEnv).toContain("SKILL_KEY=sk-skill-plaintext");
expect(migratedEnv).toContain("UNRELATED=value");
const secretsPath = path.join(stateDir, "secrets.enc.json");
const secretsPayload = JSON.parse(await fs.readFile(secretsPath, "utf8")) as {
providers: { openai: { apiKey: string } };
skills: { entries: { "review-pr": { apiKey: string } } };
channels: { googlechat: { serviceAccount: string } };
"auth-profiles": { main: { "openai:default": { key: string } } };
};
expect(secretsPayload.providers.openai.apiKey).toBe("sk-openai-plaintext");
expect(secretsPayload.skills.entries["review-pr"].apiKey).toBe("sk-skill-plaintext");
expect(secretsPayload.channels.googlechat.serviceAccount).toContain("service_account");
expect(secretsPayload["auth-profiles"].main["openai:default"].key).toBe("sk-profile-plaintext");
const rollbackResult = await rollbackSecretsMigration({ env, backupId: applyResult.backupId! });
expect(rollbackResult.restoredFiles).toBeGreaterThan(0);
const rolledBackConfig = await fs.readFile(configPath, "utf8");
expect(rolledBackConfig).toContain("sk-openai-plaintext");
expect(rolledBackConfig).toContain("sk-skill-plaintext");
const rolledBackAuth = await fs.readFile(authStorePath, "utf8");
expect(rolledBackAuth).toContain("sk-profile-plaintext");
await expect(fs.stat(secretsPath)).rejects.toThrow();
const rolledBackEnv = await fs.readFile(envPath, "utf8");
expect(rolledBackEnv).toContain("OPENAI_API_KEY=sk-openai-plaintext");
});
});

886
src/secrets/migrate.ts Normal file
View File

@@ -0,0 +1,886 @@
import crypto from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js";
import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js";
import { createConfigIO, resolveStateDir, type OpenClawConfig } from "../config/config.js";
import { isSecretRef } from "../config/types.secrets.js";
import { resolveConfigDir, resolveUserPath } from "../utils.js";
import {
encodeJsonPointerToken,
readJsonPointer as readJsonPointerRaw,
setJsonPointer,
} from "./json-pointer.js";
import { listKnownSecretEnvVarNames } from "./provider-env-vars.js";
import { decryptSopsJsonFile, encryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
const DEFAULT_SECRETS_FILE_PATH = "~/.openclaw/secrets.enc.json";
const BACKUP_DIRNAME = "secrets-migrate";
const BACKUP_MANIFEST_FILENAME = "manifest.json";
const BACKUP_RETENTION = 20;
type MigrationCounters = {
configRefs: number;
authProfileRefs: number;
plaintextRemoved: number;
secretsWritten: number;
envEntriesRemoved: number;
authStoresChanged: number;
};
type AuthStoreChange = {
path: string;
nextStore: Record<string, unknown>;
};
type EnvChange = {
path: string;
nextRaw: string;
};
type BackupManifestEntry = {
path: string;
existed: boolean;
backupPath?: string;
mode?: number;
};
type BackupManifest = {
version: 1;
backupId: string;
createdAt: string;
entries: BackupManifestEntry[];
};
type MigrationPlan = {
changed: boolean;
counters: MigrationCounters;
stateDir: string;
configChanged: boolean;
nextConfig: OpenClawConfig;
configWriteOptions: Awaited<
ReturnType<ReturnType<typeof createConfigIO>["readConfigFileSnapshotForWrite"]>
>["writeOptions"];
authStoreChanges: AuthStoreChange[];
payloadChanged: boolean;
nextPayload: Record<string, unknown>;
secretsFilePath: string;
secretsFileTimeoutMs: number;
envChange: EnvChange | null;
backupTargets: string[];
};
export type SecretsMigrationRunOptions = {
write?: boolean;
scrubEnv?: boolean;
env?: NodeJS.ProcessEnv;
now?: Date;
};
export type SecretsMigrationRunResult = {
mode: "dry-run" | "write";
changed: boolean;
backupId?: string;
backupDir?: string;
secretsFilePath: string;
counters: MigrationCounters;
changedFiles: string[];
};
export type SecretsMigrationRollbackOptions = {
backupId: string;
env?: NodeJS.ProcessEnv;
};
export type SecretsMigrationRollbackResult = {
backupId: string;
restoredFiles: number;
deletedFiles: number;
};
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function isNonEmptyString(value: unknown): value is string {
return typeof value === "string" && value.trim().length > 0;
}
function normalizeSopsTimeoutMs(value: unknown): number {
if (typeof value === "number" && Number.isFinite(value)) {
return Math.max(1, Math.floor(value));
}
return DEFAULT_SOPS_TIMEOUT_MS;
}
function readJsonPointer(root: unknown, pointer: string): unknown {
return readJsonPointerRaw(root, pointer, { onMissing: "undefined" });
}
function formatBackupId(now: Date): string {
const year = now.getUTCFullYear();
const month = String(now.getUTCMonth() + 1).padStart(2, "0");
const day = String(now.getUTCDate()).padStart(2, "0");
const hour = String(now.getUTCHours()).padStart(2, "0");
const minute = String(now.getUTCMinutes()).padStart(2, "0");
const second = String(now.getUTCSeconds()).padStart(2, "0");
return `${year}${month}${day}T${hour}${minute}${second}Z`;
}
function parseEnvValue(raw: string): string {
const trimmed = raw.trim();
if (
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
(trimmed.startsWith("'") && trimmed.endsWith("'"))
) {
return trimmed.slice(1, -1);
}
return trimmed;
}
function scrubEnvRaw(
raw: string,
migratedValues: Set<string>,
allowedEnvKeys: Set<string>,
): {
nextRaw: string;
removed: number;
} {
if (migratedValues.size === 0 || allowedEnvKeys.size === 0) {
return { nextRaw: raw, removed: 0 };
}
const lines = raw.split(/\r?\n/);
const nextLines: string[] = [];
let removed = 0;
for (const line of lines) {
const match = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
if (!match) {
nextLines.push(line);
continue;
}
const envKey = match[1] ?? "";
if (!allowedEnvKeys.has(envKey)) {
nextLines.push(line);
continue;
}
const parsedValue = parseEnvValue(match[2] ?? "");
if (migratedValues.has(parsedValue)) {
removed += 1;
continue;
}
nextLines.push(line);
}
const hadTrailingNewline = raw.endsWith("\n");
const joined = nextLines.join("\n");
return {
nextRaw:
hadTrailingNewline || joined.length === 0
? `${joined}${joined.endsWith("\n") ? "" : "\n"}`
: joined,
removed,
};
}
function ensureDirForFile(filePath: string): void {
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
}
function saveJsonFile(pathname: string, value: unknown): void {
ensureDirForFile(pathname);
fs.writeFileSync(pathname, `${JSON.stringify(value, null, 2)}\n`, "utf8");
fs.chmodSync(pathname, 0o600);
}
function resolveFileSource(
config: OpenClawConfig,
env: NodeJS.ProcessEnv,
): {
path: string;
timeoutMs: number;
hadConfiguredSource: boolean;
} {
const source = config.secrets?.sources?.file;
if (source && source.type === "sops" && isNonEmptyString(source.path)) {
return {
path: resolveUserPath(source.path),
timeoutMs: normalizeSopsTimeoutMs(source.timeoutMs),
hadConfiguredSource: true,
};
}
return {
path: resolveUserPath(resolveDefaultSecretsConfigPath(env)),
timeoutMs: DEFAULT_SOPS_TIMEOUT_MS,
hadConfiguredSource: false,
};
}
function resolveDefaultSecretsConfigPath(env: NodeJS.ProcessEnv): string {
if (env.OPENCLAW_STATE_DIR?.trim() || env.CLAWDBOT_STATE_DIR?.trim()) {
return path.join(resolveStateDir(env, os.homedir), "secrets.enc.json");
}
return DEFAULT_SECRETS_FILE_PATH;
}
async function decryptSopsJson(
pathname: string,
timeoutMs: number,
): Promise<Record<string, unknown>> {
if (!fs.existsSync(pathname)) {
return {};
}
const parsed = await decryptSopsJsonFile({
path: pathname,
timeoutMs,
missingBinaryMessage:
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
});
if (!isRecord(parsed)) {
throw new Error("sops decrypt failed: decrypted payload is not a JSON object");
}
return parsed;
}
async function encryptSopsJson(params: {
pathname: string;
timeoutMs: number;
payload: Record<string, unknown>;
}): Promise<void> {
await encryptSopsJsonFile({
path: params.pathname,
payload: params.payload,
timeoutMs: params.timeoutMs,
missingBinaryMessage:
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
});
}
function migrateModelProviderSecrets(params: {
config: OpenClawConfig;
payload: Record<string, unknown>;
counters: MigrationCounters;
migratedValues: Set<string>;
}): void {
const providers = params.config.models?.providers as
| Record<string, { apiKey?: unknown }>
| undefined;
if (!providers) {
return;
}
for (const [providerId, provider] of Object.entries(providers)) {
if (isSecretRef(provider.apiKey)) {
continue;
}
if (!isNonEmptyString(provider.apiKey)) {
continue;
}
const value = provider.apiKey.trim();
const id = `/providers/${encodeJsonPointerToken(providerId)}/apiKey`;
const existing = readJsonPointer(params.payload, id);
if (!isDeepStrictEqual(existing, value)) {
setJsonPointer(params.payload, id, value);
params.counters.secretsWritten += 1;
}
provider.apiKey = { source: "file", id };
params.counters.configRefs += 1;
params.migratedValues.add(value);
}
}
function migrateSkillEntrySecrets(params: {
config: OpenClawConfig;
payload: Record<string, unknown>;
counters: MigrationCounters;
migratedValues: Set<string>;
}): void {
const entries = params.config.skills?.entries as Record<string, { apiKey?: unknown }> | undefined;
if (!entries) {
return;
}
for (const [skillKey, entry] of Object.entries(entries)) {
if (!isRecord(entry) || isSecretRef(entry.apiKey)) {
continue;
}
if (!isNonEmptyString(entry.apiKey)) {
continue;
}
const value = entry.apiKey.trim();
const id = `/skills/entries/${encodeJsonPointerToken(skillKey)}/apiKey`;
const existing = readJsonPointer(params.payload, id);
if (!isDeepStrictEqual(existing, value)) {
setJsonPointer(params.payload, id, value);
params.counters.secretsWritten += 1;
}
entry.apiKey = { source: "file", id };
params.counters.configRefs += 1;
params.migratedValues.add(value);
}
}
function migrateGoogleChatServiceAccount(params: {
account: Record<string, unknown>;
pointerId: string;
counters: MigrationCounters;
payload: Record<string, unknown>;
}): void {
const explicitRef = isSecretRef(params.account.serviceAccountRef)
? params.account.serviceAccountRef
: null;
const inlineRef = isSecretRef(params.account.serviceAccount)
? params.account.serviceAccount
: null;
if (explicitRef || inlineRef) {
if (
params.account.serviceAccount !== undefined &&
!isSecretRef(params.account.serviceAccount)
) {
delete params.account.serviceAccount;
params.counters.plaintextRemoved += 1;
}
return;
}
const value = params.account.serviceAccount;
const hasStringValue = isNonEmptyString(value);
const hasObjectValue = isRecord(value) && Object.keys(value).length > 0;
if (!hasStringValue && !hasObjectValue) {
return;
}
const id = `${params.pointerId}/serviceAccount`;
const normalizedValue = hasStringValue ? value.trim() : structuredClone(value);
const existing = readJsonPointer(params.payload, id);
if (!isDeepStrictEqual(existing, normalizedValue)) {
setJsonPointer(params.payload, id, normalizedValue);
params.counters.secretsWritten += 1;
}
params.account.serviceAccountRef = { source: "file", id };
delete params.account.serviceAccount;
params.counters.configRefs += 1;
}
function migrateGoogleChatSecrets(params: {
config: OpenClawConfig;
payload: Record<string, unknown>;
counters: MigrationCounters;
}): void {
const googlechat = params.config.channels?.googlechat;
if (!isRecord(googlechat)) {
return;
}
migrateGoogleChatServiceAccount({
account: googlechat,
pointerId: "/channels/googlechat",
payload: params.payload,
counters: params.counters,
});
if (!isRecord(googlechat.accounts)) {
return;
}
for (const [accountId, accountValue] of Object.entries(googlechat.accounts)) {
if (!isRecord(accountValue)) {
continue;
}
migrateGoogleChatServiceAccount({
account: accountValue,
pointerId: `/channels/googlechat/accounts/${encodeJsonPointerToken(accountId)}`,
payload: params.payload,
counters: params.counters,
});
}
}
function collectAuthStorePaths(config: OpenClawConfig, stateDir: string): string[] {
const paths = new Set<string>();
paths.add(resolveUserPath(resolveAuthStorePath()));
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
if (fs.existsSync(agentsRoot)) {
for (const entry of fs.readdirSync(agentsRoot, { withFileTypes: true })) {
if (!entry.isDirectory()) {
continue;
}
paths.add(path.join(agentsRoot, entry.name, "agent", "auth-profiles.json"));
}
}
for (const agentId of listAgentIds(config)) {
const agentDir = resolveAgentDir(config, agentId);
paths.add(resolveUserPath(resolveAuthStorePath(agentDir)));
}
return [...paths];
}
function deriveAuthStoreScope(authStorePath: string, stateDir: string): string {
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
const relative = path.relative(agentsRoot, authStorePath);
if (!relative.startsWith("..")) {
const segments = relative.split(path.sep);
if (segments.length >= 3 && segments[1] === "agent" && segments[2] === "auth-profiles.json") {
const candidate = segments[0]?.trim();
if (candidate) {
return candidate;
}
}
}
const digest = crypto.createHash("sha1").update(authStorePath).digest("hex").slice(0, 8);
return `path-${digest}`;
}
function migrateAuthStoreSecrets(params: {
store: Record<string, unknown>;
scope: string;
payload: Record<string, unknown>;
counters: MigrationCounters;
migratedValues: Set<string>;
}): boolean {
const profiles = params.store.profiles;
if (!isRecord(profiles)) {
return false;
}
let changed = false;
for (const [profileId, profileValue] of Object.entries(profiles)) {
if (!isRecord(profileValue)) {
continue;
}
if (profileValue.type === "api_key") {
const keyRef = isSecretRef(profileValue.keyRef) ? profileValue.keyRef : null;
const key = isNonEmptyString(profileValue.key) ? profileValue.key.trim() : "";
if (keyRef) {
if (key) {
delete profileValue.key;
params.counters.plaintextRemoved += 1;
changed = true;
}
continue;
}
if (!key) {
continue;
}
const id = `/auth-profiles/${encodeJsonPointerToken(params.scope)}/${encodeJsonPointerToken(profileId)}/key`;
const existing = readJsonPointer(params.payload, id);
if (!isDeepStrictEqual(existing, key)) {
setJsonPointer(params.payload, id, key);
params.counters.secretsWritten += 1;
}
profileValue.keyRef = { source: "file", id };
delete profileValue.key;
params.counters.authProfileRefs += 1;
params.migratedValues.add(key);
changed = true;
continue;
}
if (profileValue.type === "token") {
const tokenRef = isSecretRef(profileValue.tokenRef) ? profileValue.tokenRef : null;
const token = isNonEmptyString(profileValue.token) ? profileValue.token.trim() : "";
if (tokenRef) {
if (token) {
delete profileValue.token;
params.counters.plaintextRemoved += 1;
changed = true;
}
continue;
}
if (!token) {
continue;
}
const id = `/auth-profiles/${encodeJsonPointerToken(params.scope)}/${encodeJsonPointerToken(profileId)}/token`;
const existing = readJsonPointer(params.payload, id);
if (!isDeepStrictEqual(existing, token)) {
setJsonPointer(params.payload, id, token);
params.counters.secretsWritten += 1;
}
profileValue.tokenRef = { source: "file", id };
delete profileValue.token;
params.counters.authProfileRefs += 1;
params.migratedValues.add(token);
changed = true;
}
}
return changed;
}
function resolveBackupRoot(stateDir: string): string {
return path.join(resolveUserPath(stateDir), "backups", BACKUP_DIRNAME);
}
function createBackupManifest(params: {
stateDir: string;
targets: string[];
backupId: string;
now: Date;
}): { backupDir: string; manifestPath: string; manifest: BackupManifest } {
const backupDir = path.join(resolveBackupRoot(params.stateDir), params.backupId);
fs.mkdirSync(backupDir, { recursive: true, mode: 0o700 });
const entries: BackupManifestEntry[] = [];
let index = 0;
for (const target of params.targets) {
const normalized = resolveUserPath(target);
const exists = fs.existsSync(normalized);
if (!exists) {
entries.push({ path: normalized, existed: false });
continue;
}
const backupName = `file-${String(index).padStart(4, "0")}.bak`;
const backupPath = path.join(backupDir, backupName);
fs.copyFileSync(normalized, backupPath);
const stats = fs.statSync(normalized);
entries.push({
path: normalized,
existed: true,
backupPath,
mode: stats.mode & 0o777,
});
index += 1;
}
const manifest: BackupManifest = {
version: 1,
backupId: params.backupId,
createdAt: params.now.toISOString(),
entries,
};
const manifestPath = path.join(backupDir, BACKUP_MANIFEST_FILENAME);
fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
fs.chmodSync(manifestPath, 0o600);
return { backupDir, manifestPath, manifest };
}
function restoreFromManifest(manifest: BackupManifest): {
restoredFiles: number;
deletedFiles: number;
} {
let restoredFiles = 0;
let deletedFiles = 0;
for (const entry of manifest.entries) {
if (!entry.existed) {
if (fs.existsSync(entry.path)) {
fs.rmSync(entry.path, { force: true });
deletedFiles += 1;
}
continue;
}
if (!entry.backupPath || !fs.existsSync(entry.backupPath)) {
throw new Error(`Backup file is missing for ${entry.path}.`);
}
ensureDirForFile(entry.path);
fs.copyFileSync(entry.backupPath, entry.path);
fs.chmodSync(entry.path, entry.mode ?? 0o600);
restoredFiles += 1;
}
return { restoredFiles, deletedFiles };
}
function pruneOldBackups(stateDir: string): void {
const backupRoot = resolveBackupRoot(stateDir);
if (!fs.existsSync(backupRoot)) {
return;
}
const dirs = fs
.readdirSync(backupRoot, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.toSorted();
if (dirs.length <= BACKUP_RETENTION) {
return;
}
const toDelete = dirs.slice(0, Math.max(0, dirs.length - BACKUP_RETENTION));
for (const dir of toDelete) {
fs.rmSync(path.join(backupRoot, dir), { recursive: true, force: true });
}
}
async function buildMigrationPlan(params: {
env: NodeJS.ProcessEnv;
scrubEnv: boolean;
}): Promise<MigrationPlan> {
const io = createConfigIO({ env: params.env });
const { snapshot, writeOptions } = await io.readConfigFileSnapshotForWrite();
if (!snapshot.valid) {
const issues =
snapshot.issues.length > 0
? snapshot.issues.map((issue) => `${issue.path || "<root>"}: ${issue.message}`).join("\n")
: "Unknown validation issue.";
throw new Error(`Cannot migrate secrets because config is invalid:\n${issues}`);
}
const stateDir = resolveStateDir(params.env, os.homedir);
const nextConfig = structuredClone(snapshot.config);
const fileSource = resolveFileSource(nextConfig, params.env);
const previousPayload = await decryptSopsJson(fileSource.path, fileSource.timeoutMs);
const nextPayload = structuredClone(previousPayload);
const counters: MigrationCounters = {
configRefs: 0,
authProfileRefs: 0,
plaintextRemoved: 0,
secretsWritten: 0,
envEntriesRemoved: 0,
authStoresChanged: 0,
};
const migratedValues = new Set<string>();
migrateModelProviderSecrets({
config: nextConfig,
payload: nextPayload,
counters,
migratedValues,
});
migrateSkillEntrySecrets({
config: nextConfig,
payload: nextPayload,
counters,
migratedValues,
});
migrateGoogleChatSecrets({
config: nextConfig,
payload: nextPayload,
counters,
});
const authStoreChanges: AuthStoreChange[] = [];
for (const authStorePath of collectAuthStorePaths(nextConfig, stateDir)) {
if (!fs.existsSync(authStorePath)) {
continue;
}
const raw = fs.readFileSync(authStorePath, "utf8");
let parsed: unknown;
try {
parsed = JSON.parse(raw) as unknown;
} catch {
continue;
}
if (!isRecord(parsed)) {
continue;
}
const nextStore = structuredClone(parsed);
const scope = deriveAuthStoreScope(authStorePath, stateDir);
const changed = migrateAuthStoreSecrets({
store: nextStore,
scope,
payload: nextPayload,
counters,
migratedValues,
});
if (!changed) {
continue;
}
authStoreChanges.push({ path: authStorePath, nextStore });
}
counters.authStoresChanged = authStoreChanges.length;
if (counters.secretsWritten > 0 && !fileSource.hadConfiguredSource) {
const defaultConfigPath = resolveDefaultSecretsConfigPath(params.env);
nextConfig.secrets ??= {};
nextConfig.secrets.sources ??= {};
nextConfig.secrets.sources.file = {
type: "sops",
path: defaultConfigPath,
timeoutMs: DEFAULT_SOPS_TIMEOUT_MS,
};
}
const configChanged = !isDeepStrictEqual(snapshot.config, nextConfig);
const payloadChanged = !isDeepStrictEqual(previousPayload, nextPayload);
let envChange: EnvChange | null = null;
if (params.scrubEnv && migratedValues.size > 0) {
const envPath = path.join(resolveConfigDir(params.env, os.homedir), ".env");
if (fs.existsSync(envPath)) {
const rawEnv = fs.readFileSync(envPath, "utf8");
const scrubbed = scrubEnvRaw(rawEnv, migratedValues, new Set(listKnownSecretEnvVarNames()));
if (scrubbed.removed > 0 && scrubbed.nextRaw !== rawEnv) {
counters.envEntriesRemoved = scrubbed.removed;
envChange = {
path: envPath,
nextRaw: scrubbed.nextRaw,
};
}
}
}
const backupTargets = new Set<string>();
if (configChanged) {
backupTargets.add(io.configPath);
}
if (payloadChanged) {
backupTargets.add(fileSource.path);
}
for (const change of authStoreChanges) {
backupTargets.add(change.path);
}
if (envChange) {
backupTargets.add(envChange.path);
}
return {
changed: configChanged || payloadChanged || authStoreChanges.length > 0 || Boolean(envChange),
counters,
stateDir,
configChanged,
nextConfig,
configWriteOptions: writeOptions,
authStoreChanges,
payloadChanged,
nextPayload,
secretsFilePath: fileSource.path,
secretsFileTimeoutMs: fileSource.timeoutMs,
envChange,
backupTargets: [...backupTargets],
};
}
export async function runSecretsMigration(
options: SecretsMigrationRunOptions = {},
): Promise<SecretsMigrationRunResult> {
const env = options.env ?? process.env;
const scrubEnv = options.scrubEnv ?? true;
const plan = await buildMigrationPlan({ env, scrubEnv });
if (!options.write) {
return {
mode: "dry-run",
changed: plan.changed,
secretsFilePath: plan.secretsFilePath,
counters: plan.counters,
changedFiles: plan.backupTargets,
};
}
if (!plan.changed) {
return {
mode: "write",
changed: false,
secretsFilePath: plan.secretsFilePath,
counters: plan.counters,
changedFiles: [],
};
}
const now = options.now ?? new Date();
const backupId = formatBackupId(now);
const backup = createBackupManifest({
stateDir: plan.stateDir,
targets: plan.backupTargets,
backupId,
now,
});
try {
if (plan.payloadChanged) {
await encryptSopsJson({
pathname: plan.secretsFilePath,
timeoutMs: plan.secretsFileTimeoutMs,
payload: plan.nextPayload,
});
}
if (plan.configChanged) {
const io = createConfigIO({ env });
await io.writeConfigFile(plan.nextConfig, plan.configWriteOptions);
}
for (const change of plan.authStoreChanges) {
saveJsonFile(change.path, change.nextStore);
}
if (plan.envChange) {
ensureDirForFile(plan.envChange.path);
fs.writeFileSync(plan.envChange.path, plan.envChange.nextRaw, "utf8");
fs.chmodSync(plan.envChange.path, 0o600);
}
} catch (err) {
restoreFromManifest(backup.manifest);
throw new Error(
`Secrets migration failed and was rolled back from backup ${backupId}: ${String(err)}`,
{
cause: err,
},
);
}
pruneOldBackups(plan.stateDir);
return {
mode: "write",
changed: true,
backupId,
backupDir: backup.backupDir,
secretsFilePath: plan.secretsFilePath,
counters: plan.counters,
changedFiles: plan.backupTargets,
};
}
export function resolveSecretsMigrationBackupRoot(env: NodeJS.ProcessEnv = process.env): string {
return resolveBackupRoot(resolveStateDir(env, os.homedir));
}
export function listSecretsMigrationBackups(env: NodeJS.ProcessEnv = process.env): string[] {
const root = resolveSecretsMigrationBackupRoot(env);
if (!fs.existsSync(root)) {
return [];
}
return fs
.readdirSync(root, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.toSorted();
}
export async function rollbackSecretsMigration(
options: SecretsMigrationRollbackOptions,
): Promise<SecretsMigrationRollbackResult> {
const env = options.env ?? process.env;
const backupDir = path.join(resolveSecretsMigrationBackupRoot(env), options.backupId);
const manifestPath = path.join(backupDir, BACKUP_MANIFEST_FILENAME);
if (!fs.existsSync(manifestPath)) {
const available = listSecretsMigrationBackups(env);
const suffix =
available.length > 0
? ` Available backups: ${available.slice(-10).join(", ")}`
: " No backups were found.";
throw new Error(`Backup "${options.backupId}" was not found.${suffix}`);
}
let parsed: unknown;
try {
parsed = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as unknown;
} catch (err) {
throw new Error(`Failed to read backup manifest at ${manifestPath}: ${String(err)}`, {
cause: err,
});
}
if (!isRecord(parsed) || !Array.isArray(parsed.entries)) {
throw new Error(`Backup manifest at ${manifestPath} is invalid.`);
}
const manifest = parsed as BackupManifest;
const restored = restoreFromManifest(manifest);
return {
backupId: options.backupId,
restoredFiles: restored.restoredFiles,
deletedFiles: restored.deletedFiles,
};
}

View File

@@ -30,6 +30,14 @@ describe("secrets runtime snapshot", () => {
},
},
},
skills: {
entries: {
"review-pr": {
enabled: true,
apiKey: { source: "env", id: "REVIEW_SKILL_API_KEY" },
},
},
},
};
const snapshot = await prepareSecretsRuntimeSnapshot({
@@ -37,6 +45,7 @@ describe("secrets runtime snapshot", () => {
env: {
OPENAI_API_KEY: "sk-env-openai",
GITHUB_TOKEN: "ghp-env-token",
REVIEW_SKILL_API_KEY: "sk-skill-ref",
},
agentDirs: ["/tmp/openclaw-agent-main"],
loadAuthStore: () => ({
@@ -59,6 +68,7 @@ describe("secrets runtime snapshot", () => {
});
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-env-openai");
expect(snapshot.config.skills?.entries?.["review-pr"]?.apiKey).toBe("sk-skill-ref");
expect(snapshot.warnings).toHaveLength(2);
expect(snapshot.authStores[0]?.store.profiles["openai:default"]).toMatchObject({
type: "api_key",

View File

@@ -41,6 +41,10 @@ type ProviderLike = {
apiKey?: unknown;
};
type SkillEntryLike = {
apiKey?: unknown;
};
type GoogleChatAccountLike = {
serviceAccount?: unknown;
serviceAccountRef?: unknown;
@@ -172,6 +176,22 @@ async function resolveConfigSecretRefs(params: {
}
}
const skillEntries = resolved.skills?.entries as Record<string, SkillEntryLike> | undefined;
if (skillEntries) {
for (const [skillKey, entry] of Object.entries(skillEntries)) {
if (!isSecretRef(entry.apiKey)) {
continue;
}
const resolvedValue = await resolveSecretRefValue(entry.apiKey, params.context);
if (!isNonEmptyString(resolvedValue)) {
throw new Error(
`skills.entries.${skillKey}.apiKey resolved to a non-string or empty value.`,
);
}
entry.apiKey = resolvedValue;
}
}
const googleChat = resolved.channels?.googlechat as GoogleChatAccountLike | undefined;
if (googleChat) {
await resolveGoogleChatServiceAccount(