Compare commits
13 Commits
main
...
secrets/pr
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cad3a00f8a | ||
|
|
add2d123b1 | ||
|
|
ef852b0ae9 | ||
|
|
ca8a0dee4b | ||
|
|
43748ea2ef | ||
|
|
ed402b6944 | ||
|
|
6eb674d6d0 | ||
|
|
16b7d4bbed | ||
|
|
209287d5e0 | ||
|
|
bb052deec6 | ||
|
|
06e51505bc | ||
|
|
2b7206470a | ||
|
|
7103ba3758 |
71
src/agents/auth-profiles.runtime-snapshot-save.test.ts
Normal file
71
src/agents/auth-profiles.runtime-snapshot-save.test.ts
Normal file
@@ -0,0 +1,71 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
activateSecretsRuntimeSnapshot,
|
||||
clearSecretsRuntimeSnapshot,
|
||||
prepareSecretsRuntimeSnapshot,
|
||||
} from "../secrets/runtime.js";
|
||||
import { ensureAuthProfileStore, markAuthProfileUsed } from "./auth-profiles.js";
|
||||
|
||||
describe("auth profile runtime snapshot persistence", () => {
|
||||
it("does not write resolved plaintext keys during usage updates", async () => {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-runtime-save-"));
|
||||
const agentDir = path.join(stateDir, "agents", "main", "agent");
|
||||
const authPath = path.join(agentDir, "auth-profiles.json");
|
||||
try {
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
authPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config: {},
|
||||
env: { OPENAI_API_KEY: "sk-runtime-openai" },
|
||||
agentDirs: [agentDir],
|
||||
});
|
||||
activateSecretsRuntimeSnapshot(snapshot);
|
||||
|
||||
const runtimeStore = ensureAuthProfileStore(agentDir);
|
||||
expect(runtimeStore.profiles["openai:default"]).toMatchObject({
|
||||
type: "api_key",
|
||||
key: "sk-runtime-openai",
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
});
|
||||
|
||||
await markAuthProfileUsed({
|
||||
store: runtimeStore,
|
||||
profileId: "openai:default",
|
||||
agentDir,
|
||||
});
|
||||
|
||||
const persisted = JSON.parse(await fs.readFile(authPath, "utf8")) as {
|
||||
profiles: Record<string, { key?: string; keyRef?: unknown }>;
|
||||
};
|
||||
expect(persisted.profiles["openai:default"]?.key).toBeUndefined();
|
||||
expect(persisted.profiles["openai:default"]?.keyRef).toEqual({
|
||||
source: "env",
|
||||
id: "OPENAI_API_KEY",
|
||||
});
|
||||
} finally {
|
||||
clearSecretsRuntimeSnapshot();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
62
src/agents/auth-profiles.store.save.test.ts
Normal file
62
src/agents/auth-profiles.store.save.test.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveAuthStorePath } from "./auth-profiles/paths.js";
|
||||
import { saveAuthProfileStore } from "./auth-profiles/store.js";
|
||||
import type { AuthProfileStore } from "./auth-profiles/types.js";
|
||||
|
||||
describe("saveAuthProfileStore", () => {
|
||||
it("strips plaintext when keyRef/tokenRef are present", async () => {
|
||||
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-auth-save-"));
|
||||
try {
|
||||
const store: AuthProfileStore = {
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
key: "sk-runtime-value",
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
},
|
||||
"github-copilot:default": {
|
||||
type: "token",
|
||||
provider: "github-copilot",
|
||||
token: "gh-runtime-token",
|
||||
tokenRef: { source: "env", id: "GITHUB_TOKEN" },
|
||||
},
|
||||
"anthropic:default": {
|
||||
type: "api_key",
|
||||
provider: "anthropic",
|
||||
key: "sk-anthropic-plain",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
saveAuthProfileStore(store, agentDir);
|
||||
|
||||
const parsed = JSON.parse(await fs.readFile(resolveAuthStorePath(agentDir), "utf8")) as {
|
||||
profiles: Record<
|
||||
string,
|
||||
{ key?: string; keyRef?: unknown; token?: string; tokenRef?: unknown }
|
||||
>;
|
||||
};
|
||||
|
||||
expect(parsed.profiles["openai:default"]?.key).toBeUndefined();
|
||||
expect(parsed.profiles["openai:default"]?.keyRef).toEqual({
|
||||
source: "env",
|
||||
id: "OPENAI_API_KEY",
|
||||
});
|
||||
|
||||
expect(parsed.profiles["github-copilot:default"]?.token).toBeUndefined();
|
||||
expect(parsed.profiles["github-copilot:default"]?.tokenRef).toEqual({
|
||||
source: "env",
|
||||
id: "GITHUB_TOKEN",
|
||||
});
|
||||
|
||||
expect(parsed.profiles["anthropic:default"]?.key).toBe("sk-anthropic-plain");
|
||||
} finally {
|
||||
await fs.rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -17,7 +17,10 @@ export {
|
||||
suggestOAuthProfileIdForLegacyDefault,
|
||||
} from "./auth-profiles/repair.js";
|
||||
export {
|
||||
clearRuntimeAuthProfileStoreSnapshots,
|
||||
ensureAuthProfileStore,
|
||||
loadAuthProfileStoreForRuntime,
|
||||
replaceRuntimeAuthProfileStoreSnapshots,
|
||||
loadAuthProfileStore,
|
||||
saveAuthProfileStore,
|
||||
} from "./auth-profiles/store.js";
|
||||
|
||||
@@ -18,6 +18,65 @@ function _syncAuthProfileStore(target: AuthProfileStore, source: AuthProfileStor
|
||||
target.usageStats = source.usageStats;
|
||||
}
|
||||
|
||||
const runtimeAuthStoreSnapshots = new Map<string, AuthProfileStore>();
|
||||
|
||||
function resolveRuntimeStoreKey(agentDir?: string): string {
|
||||
return resolveAuthStorePath(agentDir);
|
||||
}
|
||||
|
||||
function cloneAuthProfileStore(store: AuthProfileStore): AuthProfileStore {
|
||||
return structuredClone(store);
|
||||
}
|
||||
|
||||
function resolveRuntimeAuthProfileStore(agentDir?: string): AuthProfileStore | null {
|
||||
if (runtimeAuthStoreSnapshots.size === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const mainKey = resolveRuntimeStoreKey(undefined);
|
||||
const requestedKey = resolveRuntimeStoreKey(agentDir);
|
||||
const mainStore = runtimeAuthStoreSnapshots.get(mainKey);
|
||||
const requestedStore = runtimeAuthStoreSnapshots.get(requestedKey);
|
||||
|
||||
if (!agentDir || requestedKey === mainKey) {
|
||||
if (!mainStore) {
|
||||
return null;
|
||||
}
|
||||
return cloneAuthProfileStore(mainStore);
|
||||
}
|
||||
|
||||
if (mainStore && requestedStore) {
|
||||
return mergeAuthProfileStores(
|
||||
cloneAuthProfileStore(mainStore),
|
||||
cloneAuthProfileStore(requestedStore),
|
||||
);
|
||||
}
|
||||
if (requestedStore) {
|
||||
return cloneAuthProfileStore(requestedStore);
|
||||
}
|
||||
if (mainStore) {
|
||||
return cloneAuthProfileStore(mainStore);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function replaceRuntimeAuthProfileStoreSnapshots(
|
||||
entries: Array<{ agentDir?: string; store: AuthProfileStore }>,
|
||||
): void {
|
||||
runtimeAuthStoreSnapshots.clear();
|
||||
for (const entry of entries) {
|
||||
runtimeAuthStoreSnapshots.set(
|
||||
resolveRuntimeStoreKey(entry.agentDir),
|
||||
cloneAuthProfileStore(entry.store),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function clearRuntimeAuthProfileStoreSnapshots(): void {
|
||||
runtimeAuthStoreSnapshots.clear();
|
||||
}
|
||||
|
||||
export async function updateAuthProfileStoreWithLock(params: {
|
||||
agentDir?: string;
|
||||
updater: (store: AuthProfileStore) => boolean;
|
||||
@@ -316,10 +375,30 @@ function loadAuthProfileStoreForAgent(
|
||||
return store;
|
||||
}
|
||||
|
||||
export function loadAuthProfileStoreForRuntime(
|
||||
agentDir?: string,
|
||||
options?: { allowKeychainPrompt?: boolean },
|
||||
): AuthProfileStore {
|
||||
const store = loadAuthProfileStoreForAgent(agentDir, options);
|
||||
const authPath = resolveAuthStorePath(agentDir);
|
||||
const mainAuthPath = resolveAuthStorePath();
|
||||
if (!agentDir || authPath === mainAuthPath) {
|
||||
return store;
|
||||
}
|
||||
|
||||
const mainStore = loadAuthProfileStoreForAgent(undefined, options);
|
||||
return mergeAuthProfileStores(mainStore, store);
|
||||
}
|
||||
|
||||
export function ensureAuthProfileStore(
|
||||
agentDir?: string,
|
||||
options?: { allowKeychainPrompt?: boolean },
|
||||
): AuthProfileStore {
|
||||
const runtimeStore = resolveRuntimeAuthProfileStore(agentDir);
|
||||
if (runtimeStore) {
|
||||
return runtimeStore;
|
||||
}
|
||||
|
||||
const store = loadAuthProfileStoreForAgent(agentDir, options);
|
||||
const authPath = resolveAuthStorePath(agentDir);
|
||||
const mainAuthPath = resolveAuthStorePath();
|
||||
@@ -335,9 +414,24 @@ export function ensureAuthProfileStore(
|
||||
|
||||
export function saveAuthProfileStore(store: AuthProfileStore, agentDir?: string): void {
|
||||
const authPath = resolveAuthStorePath(agentDir);
|
||||
const profiles = Object.fromEntries(
|
||||
Object.entries(store.profiles).map(([profileId, credential]) => {
|
||||
if (credential.type === "api_key" && credential.keyRef && credential.key !== undefined) {
|
||||
const sanitized = { ...credential } as Record<string, unknown>;
|
||||
delete sanitized.key;
|
||||
return [profileId, sanitized];
|
||||
}
|
||||
if (credential.type === "token" && credential.tokenRef && credential.token !== undefined) {
|
||||
const sanitized = { ...credential } as Record<string, unknown>;
|
||||
delete sanitized.token;
|
||||
return [profileId, sanitized];
|
||||
}
|
||||
return [profileId, credential];
|
||||
}),
|
||||
) as AuthProfileStore["profiles"];
|
||||
const payload = {
|
||||
version: AUTH_STORE_VERSION,
|
||||
profiles: store.profiles,
|
||||
profiles,
|
||||
order: store.order ?? undefined,
|
||||
lastGood: store.lastGood ?? undefined,
|
||||
usageStats: store.usageStats ?? undefined,
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import type { OAuthCredentials } from "@mariozechner/pi-ai";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import type { SecretRef } from "../../config/types.secrets.js";
|
||||
|
||||
export type ApiKeyCredential = {
|
||||
type: "api_key";
|
||||
provider: string;
|
||||
key?: string;
|
||||
keyRef?: SecretRef;
|
||||
email?: string;
|
||||
/** Optional provider-specific metadata (e.g., account IDs, gateway IDs). */
|
||||
metadata?: Record<string, string>;
|
||||
@@ -18,6 +20,7 @@ export type TokenCredential = {
|
||||
type: "token";
|
||||
provider: string;
|
||||
token: string;
|
||||
tokenRef?: SecretRef;
|
||||
/** Optional expiry timestamp (ms since epoch). */
|
||||
expires?: number;
|
||||
email?: string;
|
||||
|
||||
@@ -31,6 +31,7 @@ export function mockCatalogImportFailThenRecover() {
|
||||
throw new Error("boom");
|
||||
}
|
||||
return {
|
||||
discoverAuthStorage: () => ({}),
|
||||
AuthStorage: class {},
|
||||
ModelRegistry: class {
|
||||
getAll() {
|
||||
|
||||
@@ -38,6 +38,7 @@ describe("loadModelCatalog", () => {
|
||||
__setModelCatalogImportForTest(
|
||||
async () =>
|
||||
({
|
||||
discoverAuthStorage: () => ({}),
|
||||
AuthStorage: class {},
|
||||
ModelRegistry: class {
|
||||
getAll() {
|
||||
@@ -69,6 +70,7 @@ describe("loadModelCatalog", () => {
|
||||
__setModelCatalogImportForTest(
|
||||
async () =>
|
||||
({
|
||||
discoverAuthStorage: () => ({}),
|
||||
AuthStorage: class {},
|
||||
ModelRegistry: class {
|
||||
getAll() {
|
||||
|
||||
@@ -70,14 +70,6 @@ export function __setModelCatalogImportForTest(loader?: () => Promise<PiSdkModul
|
||||
importPiSdk = loader ?? defaultImportPiSdk;
|
||||
}
|
||||
|
||||
function createAuthStorage(AuthStorageLike: unknown, path: string) {
|
||||
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
|
||||
if (typeof withFactory.create === "function") {
|
||||
return withFactory.create(path);
|
||||
}
|
||||
return new (AuthStorageLike as { new (path: string): unknown })(path);
|
||||
}
|
||||
|
||||
export async function loadModelCatalog(params?: {
|
||||
config?: OpenClawConfig;
|
||||
useCache?: boolean;
|
||||
@@ -102,9 +94,6 @@ export async function loadModelCatalog(params?: {
|
||||
try {
|
||||
const cfg = params?.config ?? loadConfig();
|
||||
await ensureOpenClawModelsJson(cfg);
|
||||
await (
|
||||
await import("./pi-auth-json.js")
|
||||
).ensurePiAuthJsonFromAuthProfiles(resolveOpenClawAgentDir());
|
||||
// IMPORTANT: keep the dynamic import *inside* the try/catch.
|
||||
// If this fails once (e.g. during a pnpm install that temporarily swaps node_modules),
|
||||
// we must not poison the cache with a rejected promise (otherwise all channel handlers
|
||||
@@ -112,7 +101,7 @@ export async function loadModelCatalog(params?: {
|
||||
const piSdk = await importPiSdk();
|
||||
const agentDir = resolveOpenClawAgentDir();
|
||||
const { join } = await import("node:path");
|
||||
const authStorage = createAuthStorage(piSdk.AuthStorage, join(agentDir, "auth.json"));
|
||||
const authStorage = piSdk.discoverAuthStorage(agentDir);
|
||||
const registry = new (piSdk.ModelRegistry as unknown as {
|
||||
new (
|
||||
authStorage: unknown,
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
DEFAULT_COPILOT_API_BASE_URL,
|
||||
resolveCopilotApiToken,
|
||||
} from "../providers/github-copilot-token.js";
|
||||
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
|
||||
import { ensureAuthProfileStore, listProfilesForProvider } from "./auth-profiles.js";
|
||||
import { discoverBedrockModels } from "./bedrock-discovery.js";
|
||||
import {
|
||||
@@ -387,16 +388,17 @@ export function normalizeProviders(params: {
|
||||
for (const [key, provider] of Object.entries(providers)) {
|
||||
const normalizedKey = key.trim();
|
||||
let normalizedProvider = provider;
|
||||
const configuredApiKey = normalizedProvider.apiKey;
|
||||
|
||||
// Fix common misconfig: apiKey set to "${ENV_VAR}" instead of "ENV_VAR".
|
||||
if (
|
||||
normalizedProvider.apiKey &&
|
||||
normalizeApiKeyConfig(normalizedProvider.apiKey) !== normalizedProvider.apiKey
|
||||
typeof configuredApiKey === "string" &&
|
||||
normalizeApiKeyConfig(configuredApiKey) !== configuredApiKey
|
||||
) {
|
||||
mutated = true;
|
||||
normalizedProvider = {
|
||||
...normalizedProvider,
|
||||
apiKey: normalizeApiKeyConfig(normalizedProvider.apiKey),
|
||||
apiKey: normalizeApiKeyConfig(configuredApiKey),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -404,7 +406,9 @@ export function normalizeProviders(params: {
|
||||
// Fill it from the environment or auth profiles when possible.
|
||||
const hasModels =
|
||||
Array.isArray(normalizedProvider.models) && normalizedProvider.models.length > 0;
|
||||
if (hasModels && !normalizedProvider.apiKey?.trim()) {
|
||||
const normalizedApiKey = normalizeOptionalSecretInput(normalizedProvider.apiKey);
|
||||
const hasConfiguredApiKey = Boolean(normalizedApiKey || normalizedProvider.apiKey);
|
||||
if (hasModels && !hasConfiguredApiKey) {
|
||||
const authMode =
|
||||
normalizedProvider.auth ?? (normalizedKey === "amazon-bedrock" ? "aws-sdk" : undefined);
|
||||
if (authMode === "aws-sdk") {
|
||||
@@ -957,17 +961,8 @@ export async function resolveImplicitCopilotProvider(params: {
|
||||
}
|
||||
}
|
||||
|
||||
// pi-coding-agent's ModelRegistry marks a model "available" only if its
|
||||
// `AuthStorage` has auth configured for that provider (via auth.json/env/etc).
|
||||
// Our Copilot auth lives in OpenClaw's auth-profiles store instead, so we also
|
||||
// write a runtime-only auth.json entry for pi-coding-agent to pick up.
|
||||
//
|
||||
// This is safe because it's (1) within OpenClaw's agent dir, (2) contains the
|
||||
// GitHub token (not the exchanged Copilot token), and (3) matches existing
|
||||
// patterns for OAuth-like providers in pi-coding-agent.
|
||||
// Note: we deliberately do not write pi-coding-agent's `auth.json` here.
|
||||
// OpenClaw uses its own auth store and exchanges tokens at runtime.
|
||||
// `models list` uses OpenClaw's auth heuristics for availability.
|
||||
// We deliberately do not write pi-coding-agent auth.json here.
|
||||
// OpenClaw keeps auth in auth-profiles and resolves runtime availability from that store.
|
||||
|
||||
// We intentionally do NOT define custom models for Copilot in models.json.
|
||||
// pi-coding-agent treats providers with models as replacements requiring apiKey.
|
||||
|
||||
@@ -4,6 +4,10 @@ import { ensureAuthProfileStore } from "./auth-profiles.js";
|
||||
import type { AuthProfileCredential } from "./auth-profiles/types.js";
|
||||
import { normalizeProviderId } from "./model-selection.js";
|
||||
|
||||
/**
|
||||
* @deprecated Legacy bridge for older flows that still expect `agentDir/auth.json`.
|
||||
* Runtime auth resolution uses auth-profiles directly and should not depend on this module.
|
||||
*/
|
||||
type AuthJsonCredential =
|
||||
| {
|
||||
type: "api_key";
|
||||
@@ -110,6 +114,8 @@ function credentialsEqual(a: AuthJsonCredential | undefined, b: AuthJsonCredenti
|
||||
* registry/catalog output.
|
||||
*
|
||||
* Syncs all credential types: api_key, token (as api_key), and oauth.
|
||||
*
|
||||
* @deprecated Runtime auth now comes from OpenClaw auth-profiles snapshots.
|
||||
*/
|
||||
export async function ensurePiAuthJsonFromAuthProfiles(agentDir: string): Promise<{
|
||||
wrote: boolean;
|
||||
|
||||
116
src/agents/pi-model-discovery.auth.test.ts
Normal file
116
src/agents/pi-model-discovery.auth.test.ts
Normal file
@@ -0,0 +1,116 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { saveAuthProfileStore } from "./auth-profiles.js";
|
||||
import { discoverAuthStorage } from "./pi-model-discovery.js";
|
||||
|
||||
async function createAgentDir(): Promise<string> {
|
||||
return await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-pi-auth-storage-"));
|
||||
}
|
||||
|
||||
async function pathExists(pathname: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.stat(pathname);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
describe("discoverAuthStorage", () => {
|
||||
it("loads runtime credentials from auth-profiles without writing auth.json", async () => {
|
||||
const agentDir = await createAgentDir();
|
||||
try {
|
||||
saveAuthProfileStore(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openrouter:default": {
|
||||
type: "api_key",
|
||||
provider: "openrouter",
|
||||
key: "sk-or-v1-runtime",
|
||||
},
|
||||
"anthropic:default": {
|
||||
type: "token",
|
||||
provider: "anthropic",
|
||||
token: "sk-ant-runtime",
|
||||
},
|
||||
"openai-codex:default": {
|
||||
type: "oauth",
|
||||
provider: "openai-codex",
|
||||
access: "oauth-access",
|
||||
refresh: "oauth-refresh",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
},
|
||||
agentDir,
|
||||
);
|
||||
|
||||
const authStorage = discoverAuthStorage(agentDir);
|
||||
|
||||
expect(authStorage.hasAuth("openrouter")).toBe(true);
|
||||
expect(authStorage.hasAuth("anthropic")).toBe(true);
|
||||
expect(authStorage.hasAuth("openai-codex")).toBe(true);
|
||||
await expect(authStorage.getApiKey("openrouter")).resolves.toBe("sk-or-v1-runtime");
|
||||
await expect(authStorage.getApiKey("anthropic")).resolves.toBe("sk-ant-runtime");
|
||||
expect(authStorage.get("openai-codex")).toMatchObject({
|
||||
type: "oauth",
|
||||
access: "oauth-access",
|
||||
});
|
||||
|
||||
expect(await pathExists(path.join(agentDir, "auth.json"))).toBe(false);
|
||||
} finally {
|
||||
await fs.rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("scrubs static api_key entries from legacy auth.json and keeps oauth entries", async () => {
|
||||
const agentDir = await createAgentDir();
|
||||
try {
|
||||
saveAuthProfileStore(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openrouter:default": {
|
||||
type: "api_key",
|
||||
provider: "openrouter",
|
||||
key: "sk-or-v1-runtime",
|
||||
},
|
||||
},
|
||||
},
|
||||
agentDir,
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(agentDir, "auth.json"),
|
||||
JSON.stringify(
|
||||
{
|
||||
openrouter: { type: "api_key", key: "legacy-static-key" },
|
||||
"openai-codex": {
|
||||
type: "oauth",
|
||||
access: "oauth-access",
|
||||
refresh: "oauth-refresh",
|
||||
expires: Date.now() + 60_000,
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
discoverAuthStorage(agentDir);
|
||||
|
||||
const parsed = JSON.parse(await fs.readFile(path.join(agentDir, "auth.json"), "utf8")) as {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
expect(parsed.openrouter).toBeUndefined();
|
||||
expect(parsed["openai-codex"]).toMatchObject({
|
||||
type: "oauth",
|
||||
access: "oauth-access",
|
||||
});
|
||||
} finally {
|
||||
await fs.rm(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,19 +1,172 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { AuthStorage, ModelRegistry } from "@mariozechner/pi-coding-agent";
|
||||
import {
|
||||
AuthStorage,
|
||||
InMemoryAuthStorageBackend,
|
||||
ModelRegistry,
|
||||
} from "@mariozechner/pi-coding-agent";
|
||||
import { ensureAuthProfileStore } from "./auth-profiles.js";
|
||||
import type { AuthProfileCredential } from "./auth-profiles.js";
|
||||
import { normalizeProviderId } from "./model-selection.js";
|
||||
|
||||
export { AuthStorage, ModelRegistry } from "@mariozechner/pi-coding-agent";
|
||||
|
||||
function createAuthStorage(AuthStorageLike: unknown, path: string) {
|
||||
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
|
||||
if (typeof withFactory.create === "function") {
|
||||
return withFactory.create(path) as AuthStorage;
|
||||
type PiApiKeyCredential = { type: "api_key"; key: string };
|
||||
type PiOAuthCredential = {
|
||||
type: "oauth";
|
||||
access: string;
|
||||
refresh: string;
|
||||
expires: number;
|
||||
};
|
||||
|
||||
type PiCredential = PiApiKeyCredential | PiOAuthCredential;
|
||||
type PiCredentialMap = Record<string, PiCredential>;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function scrubLegacyStaticAuthJsonEntries(pathname: string): void {
|
||||
if (!fs.existsSync(pathname)) {
|
||||
return;
|
||||
}
|
||||
return new (AuthStorageLike as { new (path: string): unknown })(path) as AuthStorage;
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(fs.readFileSync(pathname, "utf8")) as unknown;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!isRecord(parsed)) {
|
||||
return;
|
||||
}
|
||||
|
||||
let changed = false;
|
||||
for (const [provider, value] of Object.entries(parsed)) {
|
||||
if (!isRecord(value)) {
|
||||
continue;
|
||||
}
|
||||
if (value.type !== "api_key") {
|
||||
continue;
|
||||
}
|
||||
delete parsed[provider];
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if (!changed) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (Object.keys(parsed).length === 0) {
|
||||
fs.rmSync(pathname, { force: true });
|
||||
return;
|
||||
}
|
||||
|
||||
fs.writeFileSync(pathname, `${JSON.stringify(parsed, null, 2)}\n`, "utf8");
|
||||
fs.chmodSync(pathname, 0o600);
|
||||
}
|
||||
|
||||
function createAuthStorage(AuthStorageLike: unknown, path: string, creds: PiCredentialMap) {
|
||||
const withInMemory = AuthStorageLike as { inMemory?: (data?: unknown) => unknown };
|
||||
if (typeof withInMemory.inMemory === "function") {
|
||||
return withInMemory.inMemory(creds) as AuthStorage;
|
||||
}
|
||||
|
||||
const withFromStorage = AuthStorageLike as {
|
||||
fromStorage?: (storage: unknown) => unknown;
|
||||
};
|
||||
if (typeof withFromStorage.fromStorage === "function") {
|
||||
const backend = new InMemoryAuthStorageBackend();
|
||||
backend.withLock(() => ({
|
||||
result: undefined,
|
||||
next: JSON.stringify(creds, null, 2),
|
||||
}));
|
||||
return withFromStorage.fromStorage(backend) as AuthStorage;
|
||||
}
|
||||
|
||||
const withFactory = AuthStorageLike as { create?: (path: string) => unknown };
|
||||
const withRuntimeOverride = (
|
||||
typeof withFactory.create === "function"
|
||||
? withFactory.create(path)
|
||||
: new (AuthStorageLike as { new (path: string): unknown })(path)
|
||||
) as AuthStorage & {
|
||||
setRuntimeApiKey?: (provider: string, apiKey: string) => void;
|
||||
};
|
||||
if (typeof withRuntimeOverride.setRuntimeApiKey === "function") {
|
||||
for (const [provider, credential] of Object.entries(creds)) {
|
||||
if (credential.type === "api_key") {
|
||||
withRuntimeOverride.setRuntimeApiKey(provider, credential.key);
|
||||
continue;
|
||||
}
|
||||
withRuntimeOverride.setRuntimeApiKey(provider, credential.access);
|
||||
}
|
||||
}
|
||||
return withRuntimeOverride;
|
||||
}
|
||||
|
||||
function convertAuthProfileCredential(cred: AuthProfileCredential): PiCredential | null {
|
||||
if (cred.type === "api_key") {
|
||||
const key = typeof cred.key === "string" ? cred.key.trim() : "";
|
||||
if (!key) {
|
||||
return null;
|
||||
}
|
||||
return { type: "api_key", key };
|
||||
}
|
||||
|
||||
if (cred.type === "token") {
|
||||
const token = typeof cred.token === "string" ? cred.token.trim() : "";
|
||||
if (!token) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
typeof cred.expires === "number" &&
|
||||
Number.isFinite(cred.expires) &&
|
||||
Date.now() >= cred.expires
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return { type: "api_key", key: token };
|
||||
}
|
||||
|
||||
if (cred.type === "oauth") {
|
||||
const access = typeof cred.access === "string" ? cred.access.trim() : "";
|
||||
const refresh = typeof cred.refresh === "string" ? cred.refresh.trim() : "";
|
||||
if (!access || !refresh || !Number.isFinite(cred.expires) || cred.expires <= 0) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: "oauth",
|
||||
access,
|
||||
refresh,
|
||||
expires: cred.expires,
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolvePiCredentials(agentDir: string): PiCredentialMap {
|
||||
const store = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false });
|
||||
const credentials: PiCredentialMap = {};
|
||||
for (const credential of Object.values(store.profiles)) {
|
||||
const provider = normalizeProviderId(String(credential.provider ?? "")).trim();
|
||||
if (!provider || credentials[provider]) {
|
||||
continue;
|
||||
}
|
||||
const converted = convertAuthProfileCredential(credential);
|
||||
if (converted) {
|
||||
credentials[provider] = converted;
|
||||
}
|
||||
}
|
||||
return credentials;
|
||||
}
|
||||
|
||||
// Compatibility helpers for pi-coding-agent 0.50+ (discover* helpers removed).
|
||||
export function discoverAuthStorage(agentDir: string): AuthStorage {
|
||||
return createAuthStorage(AuthStorage, path.join(agentDir, "auth.json"));
|
||||
const credentials = resolvePiCredentials(agentDir);
|
||||
const authPath = path.join(agentDir, "auth.json");
|
||||
scrubLegacyStaticAuthJsonEntries(authPath);
|
||||
return createAuthStorage(AuthStorage, authPath, credentials);
|
||||
}
|
||||
|
||||
export function discoverModels(authStorage: AuthStorage, agentDir: string): ModelRegistry {
|
||||
|
||||
@@ -105,9 +105,10 @@ function applySkillConfigEnvOverrides(params: {
|
||||
}
|
||||
}
|
||||
|
||||
if (normalizedPrimaryEnv && skillConfig.apiKey && !process.env[normalizedPrimaryEnv]) {
|
||||
const resolvedApiKey = typeof skillConfig.apiKey === "string" ? skillConfig.apiKey.trim() : "";
|
||||
if (normalizedPrimaryEnv && resolvedApiKey && !process.env[normalizedPrimaryEnv]) {
|
||||
if (!pendingOverrides[normalizedPrimaryEnv]) {
|
||||
pendingOverrides[normalizedPrimaryEnv] = skillConfig.apiKey;
|
||||
pendingOverrides[normalizedPrimaryEnv] = resolvedApiKey;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -260,6 +260,15 @@ const entries: SubCliEntry[] = [
|
||||
mod.registerSecurityCli(program);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "secrets",
|
||||
description: "Secrets runtime reload controls",
|
||||
hasSubcommands: true,
|
||||
register: async (program) => {
|
||||
const mod = await import("../secrets-cli.js");
|
||||
mod.registerSecretsCli(program);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "skills",
|
||||
description: "List and inspect available skills",
|
||||
|
||||
96
src/cli/secrets-cli.test.ts
Normal file
96
src/cli/secrets-cli.test.ts
Normal file
@@ -0,0 +1,96 @@
|
||||
import { Command } from "commander";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createCliRuntimeCapture } from "./test-runtime-capture.js";
|
||||
|
||||
const callGatewayFromCli = vi.fn();
|
||||
const runSecretsMigration = vi.fn();
|
||||
const rollbackSecretsMigration = vi.fn();
|
||||
|
||||
const { defaultRuntime, runtimeLogs, runtimeErrors, resetRuntimeCapture } =
|
||||
createCliRuntimeCapture();
|
||||
|
||||
vi.mock("./gateway-rpc.js", () => ({
|
||||
addGatewayClientOptions: (cmd: Command) => cmd,
|
||||
callGatewayFromCli: (method: string, opts: unknown, params?: unknown, extra?: unknown) =>
|
||||
callGatewayFromCli(method, opts, params, extra),
|
||||
}));
|
||||
|
||||
vi.mock("../runtime.js", () => ({
|
||||
defaultRuntime,
|
||||
}));
|
||||
|
||||
vi.mock("../secrets/migrate.js", () => ({
|
||||
runSecretsMigration: (options: unknown) => runSecretsMigration(options),
|
||||
rollbackSecretsMigration: (options: unknown) => rollbackSecretsMigration(options),
|
||||
}));
|
||||
|
||||
const { registerSecretsCli } = await import("./secrets-cli.js");
|
||||
|
||||
describe("secrets CLI", () => {
|
||||
const createProgram = () => {
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
registerSecretsCli(program);
|
||||
return program;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
resetRuntimeCapture();
|
||||
callGatewayFromCli.mockReset();
|
||||
runSecretsMigration.mockReset();
|
||||
rollbackSecretsMigration.mockReset();
|
||||
});
|
||||
|
||||
it("calls secrets.reload and prints human output", async () => {
|
||||
callGatewayFromCli.mockResolvedValue({ ok: true, warningCount: 1 });
|
||||
await createProgram().parseAsync(["secrets", "reload"], { from: "user" });
|
||||
expect(callGatewayFromCli).toHaveBeenCalledWith(
|
||||
"secrets.reload",
|
||||
expect.anything(),
|
||||
undefined,
|
||||
expect.objectContaining({ expectFinal: false }),
|
||||
);
|
||||
expect(runtimeLogs.at(-1)).toBe("Secrets reloaded with 1 warning(s).");
|
||||
expect(runtimeErrors).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("prints JSON when requested", async () => {
|
||||
callGatewayFromCli.mockResolvedValue({ ok: true, warningCount: 0 });
|
||||
await createProgram().parseAsync(["secrets", "reload", "--json"], { from: "user" });
|
||||
expect(runtimeLogs.at(-1)).toContain('"ok": true');
|
||||
});
|
||||
|
||||
it("runs secrets migrate as dry-run by default", async () => {
|
||||
runSecretsMigration.mockResolvedValue({
|
||||
mode: "dry-run",
|
||||
changed: true,
|
||||
secretsFilePath: "/tmp/secrets.enc.json",
|
||||
counters: { secretsWritten: 3 },
|
||||
changedFiles: ["/tmp/openclaw.json"],
|
||||
});
|
||||
|
||||
await createProgram().parseAsync(["secrets", "migrate"], { from: "user" });
|
||||
|
||||
expect(runSecretsMigration).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ write: false, scrubEnv: true }),
|
||||
);
|
||||
expect(runtimeLogs.at(-1)).toContain("dry run");
|
||||
});
|
||||
|
||||
it("runs rollback when --rollback is provided", async () => {
|
||||
rollbackSecretsMigration.mockResolvedValue({
|
||||
backupId: "20260221T010203Z",
|
||||
restoredFiles: 2,
|
||||
deletedFiles: 1,
|
||||
});
|
||||
|
||||
await createProgram().parseAsync(["secrets", "migrate", "--rollback", "20260221T010203Z"], {
|
||||
from: "user",
|
||||
});
|
||||
|
||||
expect(rollbackSecretsMigration).toHaveBeenCalledWith({
|
||||
backupId: "20260221T010203Z",
|
||||
});
|
||||
expect(runtimeLogs.at(-1)).toContain("rollback complete");
|
||||
});
|
||||
});
|
||||
121
src/cli/secrets-cli.ts
Normal file
121
src/cli/secrets-cli.ts
Normal file
@@ -0,0 +1,121 @@
|
||||
import type { Command } from "commander";
|
||||
import { danger } from "../globals.js";
|
||||
import { defaultRuntime } from "../runtime.js";
|
||||
import {
|
||||
rollbackSecretsMigration,
|
||||
runSecretsMigration,
|
||||
type SecretsMigrationRollbackResult,
|
||||
type SecretsMigrationRunResult,
|
||||
} from "../secrets/migrate.js";
|
||||
import { formatDocsLink } from "../terminal/links.js";
|
||||
import { theme } from "../terminal/theme.js";
|
||||
import { addGatewayClientOptions, callGatewayFromCli, type GatewayRpcOpts } from "./gateway-rpc.js";
|
||||
|
||||
type SecretsReloadOptions = GatewayRpcOpts & { json?: boolean };
|
||||
type SecretsMigrateOptions = {
|
||||
write?: boolean;
|
||||
rollback?: string;
|
||||
scrubEnv?: boolean;
|
||||
json?: boolean;
|
||||
};
|
||||
|
||||
function printMigrationResult(
|
||||
result: SecretsMigrationRunResult | SecretsMigrationRollbackResult,
|
||||
json: boolean,
|
||||
): void {
|
||||
if (json) {
|
||||
defaultRuntime.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if ("restoredFiles" in result) {
|
||||
defaultRuntime.log(
|
||||
`Secrets rollback complete for backup ${result.backupId}. Restored ${result.restoredFiles} file(s), deleted ${result.deletedFiles} file(s).`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.mode === "dry-run") {
|
||||
if (!result.changed) {
|
||||
defaultRuntime.log("Secrets migrate dry run: no changes needed.");
|
||||
return;
|
||||
}
|
||||
defaultRuntime.log(
|
||||
`Secrets migrate dry run: ${result.changedFiles.length} file(s) would change, ${result.counters.secretsWritten} secret value(s) would move to ${result.secretsFilePath}.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!result.changed) {
|
||||
defaultRuntime.log("Secrets migrate: no changes applied.");
|
||||
return;
|
||||
}
|
||||
defaultRuntime.log(
|
||||
`Secrets migrated. Backup: ${result.backupId}. Moved ${result.counters.secretsWritten} secret value(s) into ${result.secretsFilePath}.`,
|
||||
);
|
||||
}
|
||||
|
||||
export function registerSecretsCli(program: Command) {
|
||||
const secrets = program
|
||||
.command("secrets")
|
||||
.description("Secrets runtime controls")
|
||||
.addHelpText(
|
||||
"after",
|
||||
() =>
|
||||
`\n${theme.muted("Docs:")} ${formatDocsLink("/gateway/security", "docs.openclaw.ai/gateway/security")}\n`,
|
||||
);
|
||||
|
||||
addGatewayClientOptions(
|
||||
secrets
|
||||
.command("reload")
|
||||
.description("Re-resolve secret references and atomically swap runtime snapshot")
|
||||
.option("--json", "Output JSON", false),
|
||||
).action(async (opts: SecretsReloadOptions) => {
|
||||
try {
|
||||
const result = await callGatewayFromCli("secrets.reload", opts, undefined, {
|
||||
expectFinal: false,
|
||||
});
|
||||
if (opts.json) {
|
||||
defaultRuntime.log(JSON.stringify(result, null, 2));
|
||||
return;
|
||||
}
|
||||
const warningCount = Number(
|
||||
(result as { warningCount?: unknown } | undefined)?.warningCount ?? 0,
|
||||
);
|
||||
if (Number.isFinite(warningCount) && warningCount > 0) {
|
||||
defaultRuntime.log(`Secrets reloaded with ${warningCount} warning(s).`);
|
||||
return;
|
||||
}
|
||||
defaultRuntime.log("Secrets reloaded.");
|
||||
} catch (err) {
|
||||
defaultRuntime.error(danger(String(err)));
|
||||
defaultRuntime.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
secrets
|
||||
.command("migrate")
|
||||
.description("Migrate plaintext secrets to file-backed SecretRefs (sops)")
|
||||
.option("--write", "Apply migration changes (default is dry-run)", false)
|
||||
.option("--rollback <backup-id>", "Rollback a previous migration backup id")
|
||||
.option("--no-scrub-env", "Keep matching plaintext values in ~/.openclaw/.env")
|
||||
.option("--json", "Output JSON", false)
|
||||
.action(async (opts: SecretsMigrateOptions) => {
|
||||
try {
|
||||
if (typeof opts.rollback === "string" && opts.rollback.trim()) {
|
||||
const result = await rollbackSecretsMigration({ backupId: opts.rollback.trim() });
|
||||
printMigrationResult(result, Boolean(opts.json));
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await runSecretsMigration({
|
||||
write: Boolean(opts.write),
|
||||
scrubEnv: opts.scrubEnv ?? true,
|
||||
});
|
||||
printMigrationResult(result, Boolean(opts.json));
|
||||
} catch (err) {
|
||||
defaultRuntime.error(danger(String(err)));
|
||||
defaultRuntime.exit(1);
|
||||
}
|
||||
});
|
||||
}
|
||||
89
src/commands/auth-choice.apply.openai.test.ts
Normal file
89
src/commands/auth-choice.apply.openai.test.ts
Normal file
@@ -0,0 +1,89 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyAuthChoiceOpenAI } from "./auth-choice.apply.openai.js";
|
||||
import {
|
||||
createAuthTestLifecycle,
|
||||
createExitThrowingRuntime,
|
||||
createWizardPrompter,
|
||||
readAuthProfilesForAgent,
|
||||
setupAuthTestEnv,
|
||||
} from "./test-wizard-helpers.js";
|
||||
|
||||
describe("applyAuthChoiceOpenAI", () => {
|
||||
const lifecycle = createAuthTestLifecycle([
|
||||
"OPENCLAW_STATE_DIR",
|
||||
"OPENCLAW_AGENT_DIR",
|
||||
"PI_CODING_AGENT_DIR",
|
||||
"OPENAI_API_KEY",
|
||||
]);
|
||||
|
||||
async function setupTempState() {
|
||||
const env = await setupAuthTestEnv("openclaw-openai-");
|
||||
lifecycle.setStateDir(env.stateDir);
|
||||
return env.agentDir;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await lifecycle.cleanup();
|
||||
});
|
||||
|
||||
it("writes env-backed OpenAI key as keyRef in auth profiles", async () => {
|
||||
const agentDir = await setupTempState();
|
||||
process.env.OPENAI_API_KEY = "sk-openai-env";
|
||||
|
||||
const confirm = vi.fn(async () => true);
|
||||
const text = vi.fn(async () => "unused");
|
||||
const prompter = createWizardPrompter({ confirm, text }, { defaultSelect: "" });
|
||||
const runtime = createExitThrowingRuntime();
|
||||
|
||||
const result = await applyAuthChoiceOpenAI({
|
||||
authChoice: "openai-api-key",
|
||||
config: {},
|
||||
prompter,
|
||||
runtime,
|
||||
setDefaultModel: true,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result?.config.auth?.profiles?.["openai:default"]).toMatchObject({
|
||||
provider: "openai",
|
||||
mode: "api_key",
|
||||
});
|
||||
expect(result?.config.agents?.defaults?.model?.primary).toBe("openai/gpt-5.1-codex");
|
||||
expect(text).not.toHaveBeenCalled();
|
||||
|
||||
const parsed = await readAuthProfilesForAgent<{
|
||||
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
|
||||
}>(agentDir);
|
||||
expect(parsed.profiles?.["openai:default"]).toMatchObject({
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
});
|
||||
expect(parsed.profiles?.["openai:default"]?.key).toBeUndefined();
|
||||
});
|
||||
|
||||
it("writes explicit token input into openai auth profile", async () => {
|
||||
const agentDir = await setupTempState();
|
||||
|
||||
const prompter = createWizardPrompter({}, { defaultSelect: "" });
|
||||
const runtime = createExitThrowingRuntime();
|
||||
|
||||
const result = await applyAuthChoiceOpenAI({
|
||||
authChoice: "apiKey",
|
||||
config: {},
|
||||
prompter,
|
||||
runtime,
|
||||
setDefaultModel: true,
|
||||
opts: {
|
||||
tokenProvider: "openai",
|
||||
token: "sk-openai-token",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
|
||||
const parsed = await readAuthProfilesForAgent<{
|
||||
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
|
||||
}>(agentDir);
|
||||
expect(parsed.profiles?.["openai:default"]?.key).toBe("sk-openai-token");
|
||||
expect(parsed.profiles?.["openai:default"]?.keyRef).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,4 @@
|
||||
import { resolveEnvApiKey } from "../agents/model-auth.js";
|
||||
import { upsertSharedEnvVar } from "../infra/env-file.js";
|
||||
import {
|
||||
formatApiKeyPreview,
|
||||
normalizeApiKeyInput,
|
||||
@@ -9,7 +8,7 @@ import { createAuthChoiceAgentModelNoter } from "./auth-choice.apply-helpers.js"
|
||||
import type { ApplyAuthChoiceParams, ApplyAuthChoiceResult } from "./auth-choice.apply.js";
|
||||
import { applyDefaultModelChoice } from "./auth-choice.default-model.js";
|
||||
import { isRemoteEnvironment } from "./oauth-env.js";
|
||||
import { applyAuthProfileConfig, writeOAuthCredentials } from "./onboard-auth.js";
|
||||
import { applyAuthProfileConfig, setOpenaiApiKey, writeOAuthCredentials } from "./onboard-auth.js";
|
||||
import { openUrl } from "./onboard-helpers.js";
|
||||
import {
|
||||
applyOpenAICodexModelDefault,
|
||||
@@ -58,17 +57,12 @@ export async function applyAuthChoiceOpenAI(
|
||||
initialValue: true,
|
||||
});
|
||||
if (useExisting) {
|
||||
const result = upsertSharedEnvVar({
|
||||
key: "OPENAI_API_KEY",
|
||||
value: envKey.apiKey,
|
||||
await setOpenaiApiKey(envKey.apiKey, params.agentDir);
|
||||
nextConfig = applyAuthProfileConfig(nextConfig, {
|
||||
profileId: "openai:default",
|
||||
provider: "openai",
|
||||
mode: "api_key",
|
||||
});
|
||||
if (!process.env.OPENAI_API_KEY) {
|
||||
process.env.OPENAI_API_KEY = envKey.apiKey;
|
||||
}
|
||||
await params.prompter.note(
|
||||
`Copied OPENAI_API_KEY to ${result.path} for launchd compatibility.`,
|
||||
"OpenAI API key",
|
||||
);
|
||||
return await applyOpenAiDefaultModelChoice();
|
||||
}
|
||||
}
|
||||
@@ -84,15 +78,12 @@ export async function applyAuthChoiceOpenAI(
|
||||
}
|
||||
|
||||
const trimmed = normalizeApiKeyInput(String(key));
|
||||
const result = upsertSharedEnvVar({
|
||||
key: "OPENAI_API_KEY",
|
||||
value: trimmed,
|
||||
await setOpenaiApiKey(trimmed, params.agentDir);
|
||||
nextConfig = applyAuthProfileConfig(nextConfig, {
|
||||
profileId: "openai:default",
|
||||
provider: "openai",
|
||||
mode: "api_key",
|
||||
});
|
||||
process.env.OPENAI_API_KEY = trimmed;
|
||||
await params.prompter.note(
|
||||
`Saved OPENAI_API_KEY to ${result.path} for launchd compatibility.`,
|
||||
"OpenAI API key",
|
||||
);
|
||||
return await applyOpenAiDefaultModelChoice();
|
||||
}
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ describe("models list auth-profile sync", () => {
|
||||
const openrouter = getProviderRow(String(runtime.log.mock.calls[0]?.[0]), "openrouter/");
|
||||
expect(openrouter).toBeDefined();
|
||||
expect(openrouter?.available).toBe(true);
|
||||
expect(await pathExists(authPath)).toBe(true);
|
||||
expect(await pathExists(authPath)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -6,9 +6,6 @@ let toModelRow: typeof import("./models/list.registry.js").toModelRow;
|
||||
|
||||
const loadConfig = vi.fn();
|
||||
const ensureOpenClawModelsJson = vi.fn().mockResolvedValue(undefined);
|
||||
const ensurePiAuthJsonFromAuthProfiles = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ wrote: false, authPath: "/tmp/openclaw-agent/auth.json" });
|
||||
const resolveOpenClawAgentDir = vi.fn().mockReturnValue("/tmp/openclaw-agent");
|
||||
const ensureAuthProfileStore = vi.fn().mockReturnValue({ version: 1, profiles: {} });
|
||||
const listProfilesForProvider = vi.fn().mockReturnValue([]);
|
||||
@@ -38,10 +35,6 @@ vi.mock("../agents/models-config.js", () => ({
|
||||
ensureOpenClawModelsJson,
|
||||
}));
|
||||
|
||||
vi.mock("../agents/pi-auth-json.js", () => ({
|
||||
ensurePiAuthJsonFromAuthProfiles,
|
||||
}));
|
||||
|
||||
vi.mock("../agents/agent-paths.js", () => ({
|
||||
resolveOpenClawAgentDir,
|
||||
}));
|
||||
@@ -110,7 +103,6 @@ beforeEach(() => {
|
||||
modelRegistryState.getAllError = undefined;
|
||||
modelRegistryState.getAvailableError = undefined;
|
||||
listProfilesForProvider.mockReturnValue([]);
|
||||
ensurePiAuthJsonFromAuthProfiles.mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -279,13 +271,12 @@ describe("models list/status", () => {
|
||||
({ loadModelRegistry, toModelRow } = await import("./models/list.registry.js"));
|
||||
});
|
||||
|
||||
it("models list syncs auth-profiles into auth.json before availability checks", async () => {
|
||||
it("models list runs model discovery without auth.json sync", async () => {
|
||||
setDefaultZaiRegistry();
|
||||
const runtime = makeRuntime();
|
||||
|
||||
await modelsListCommand({ all: true, json: true }, runtime);
|
||||
|
||||
expect(ensurePiAuthJsonFromAuthProfiles).toHaveBeenCalledWith("/tmp/openclaw-agent");
|
||||
expect(runtime.error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("models list outputs canonical zai key for configured z.ai model", async () => {
|
||||
|
||||
@@ -12,7 +12,6 @@ import {
|
||||
resolveForwardCompatModel,
|
||||
} from "../../agents/model-forward-compat.js";
|
||||
import { ensureOpenClawModelsJson } from "../../agents/models-config.js";
|
||||
import { ensurePiAuthJsonFromAuthProfiles } from "../../agents/pi-auth-json.js";
|
||||
import type { ModelRegistry } from "../../agents/pi-model-discovery.js";
|
||||
import { discoverAuthStorage, discoverModels } from "../../agents/pi-model-discovery.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
@@ -102,7 +101,6 @@ function loadAvailableModels(registry: ModelRegistry): Model<Api>[] {
|
||||
export async function loadModelRegistry(cfg: OpenClawConfig) {
|
||||
await ensureOpenClawModelsJson(cfg);
|
||||
const agentDir = resolveOpenClawAgentDir();
|
||||
await ensurePiAuthJsonFromAuthProfiles(agentDir);
|
||||
const authStorage = discoverAuthStorage(agentDir);
|
||||
const registry = discoverModels(authStorage, agentDir);
|
||||
const appended = appendAntigravityForwardCompatModels(registry.getAll(), registry);
|
||||
|
||||
106
src/commands/onboard-auth.credentials.test.ts
Normal file
106
src/commands/onboard-auth.credentials.test.ts
Normal file
@@ -0,0 +1,106 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
setCloudflareAiGatewayConfig,
|
||||
setMoonshotApiKey,
|
||||
setOpenaiApiKey,
|
||||
} from "./onboard-auth.js";
|
||||
import {
|
||||
createAuthTestLifecycle,
|
||||
readAuthProfilesForAgent,
|
||||
setupAuthTestEnv,
|
||||
} from "./test-wizard-helpers.js";
|
||||
|
||||
describe("onboard auth credentials secret refs", () => {
|
||||
const lifecycle = createAuthTestLifecycle([
|
||||
"OPENCLAW_STATE_DIR",
|
||||
"OPENCLAW_AGENT_DIR",
|
||||
"PI_CODING_AGENT_DIR",
|
||||
"MOONSHOT_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"CLOUDFLARE_AI_GATEWAY_API_KEY",
|
||||
]);
|
||||
|
||||
afterEach(async () => {
|
||||
await lifecycle.cleanup();
|
||||
});
|
||||
|
||||
it("stores env-backed moonshot key as keyRef", async () => {
|
||||
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-");
|
||||
lifecycle.setStateDir(env.stateDir);
|
||||
process.env.MOONSHOT_API_KEY = "sk-moonshot-env";
|
||||
|
||||
await setMoonshotApiKey("sk-moonshot-env");
|
||||
|
||||
const parsed = await readAuthProfilesForAgent<{
|
||||
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
|
||||
}>(env.agentDir);
|
||||
expect(parsed.profiles?.["moonshot:default"]).toMatchObject({
|
||||
keyRef: { source: "env", id: "MOONSHOT_API_KEY" },
|
||||
});
|
||||
expect(parsed.profiles?.["moonshot:default"]?.key).toBeUndefined();
|
||||
});
|
||||
|
||||
it("stores ${ENV} moonshot input as keyRef even when env value is unset", async () => {
|
||||
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-inline-ref-");
|
||||
lifecycle.setStateDir(env.stateDir);
|
||||
|
||||
await setMoonshotApiKey("${MOONSHOT_API_KEY}");
|
||||
|
||||
const parsed = await readAuthProfilesForAgent<{
|
||||
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
|
||||
}>(env.agentDir);
|
||||
expect(parsed.profiles?.["moonshot:default"]).toMatchObject({
|
||||
keyRef: { source: "env", id: "MOONSHOT_API_KEY" },
|
||||
});
|
||||
expect(parsed.profiles?.["moonshot:default"]?.key).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps plaintext moonshot key when no env ref applies", async () => {
|
||||
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-plaintext-");
|
||||
lifecycle.setStateDir(env.stateDir);
|
||||
process.env.MOONSHOT_API_KEY = "sk-moonshot-other";
|
||||
|
||||
await setMoonshotApiKey("sk-moonshot-plaintext");
|
||||
|
||||
const parsed = await readAuthProfilesForAgent<{
|
||||
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
|
||||
}>(env.agentDir);
|
||||
expect(parsed.profiles?.["moonshot:default"]).toMatchObject({
|
||||
key: "sk-moonshot-plaintext",
|
||||
});
|
||||
expect(parsed.profiles?.["moonshot:default"]?.keyRef).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves cloudflare metadata when storing keyRef", async () => {
|
||||
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-cloudflare-");
|
||||
lifecycle.setStateDir(env.stateDir);
|
||||
process.env.CLOUDFLARE_AI_GATEWAY_API_KEY = "cf-secret";
|
||||
|
||||
await setCloudflareAiGatewayConfig("account-1", "gateway-1", "cf-secret");
|
||||
|
||||
const parsed = await readAuthProfilesForAgent<{
|
||||
profiles?: Record<string, { key?: string; keyRef?: unknown; metadata?: unknown }>;
|
||||
}>(env.agentDir);
|
||||
expect(parsed.profiles?.["cloudflare-ai-gateway:default"]).toMatchObject({
|
||||
keyRef: { source: "env", id: "CLOUDFLARE_AI_GATEWAY_API_KEY" },
|
||||
metadata: { accountId: "account-1", gatewayId: "gateway-1" },
|
||||
});
|
||||
expect(parsed.profiles?.["cloudflare-ai-gateway:default"]?.key).toBeUndefined();
|
||||
});
|
||||
|
||||
it("stores env-backed openai key as keyRef", async () => {
|
||||
const env = await setupAuthTestEnv("openclaw-onboard-auth-credentials-openai-");
|
||||
lifecycle.setStateDir(env.stateDir);
|
||||
process.env.OPENAI_API_KEY = "sk-openai-env";
|
||||
|
||||
await setOpenaiApiKey("sk-openai-env");
|
||||
|
||||
const parsed = await readAuthProfilesForAgent<{
|
||||
profiles?: Record<string, { key?: string; keyRef?: unknown }>;
|
||||
}>(env.agentDir);
|
||||
expect(parsed.profiles?.["openai:default"]).toMatchObject({
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
});
|
||||
expect(parsed.profiles?.["openai:default"]?.key).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -4,11 +4,85 @@ import type { OAuthCredentials } from "@mariozechner/pi-ai";
|
||||
import { resolveOpenClawAgentDir } from "../agents/agent-paths.js";
|
||||
import { upsertAuthProfile } from "../agents/auth-profiles.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { isSecretRef, type SecretInput, type SecretRef } from "../config/types.secrets.js";
|
||||
import { PROVIDER_ENV_VARS } from "../secrets/provider-env-vars.js";
|
||||
import { normalizeSecretInput } from "../utils/normalize-secret-input.js";
|
||||
export { CLOUDFLARE_AI_GATEWAY_DEFAULT_MODEL_REF } from "../agents/cloudflare-ai-gateway.js";
|
||||
export { XAI_DEFAULT_MODEL_REF } from "./onboard-auth.models.js";
|
||||
|
||||
const resolveAuthAgentDir = (agentDir?: string) => agentDir ?? resolveOpenClawAgentDir();
|
||||
|
||||
const ENV_REF_PATTERN = /^\$\{([A-Z][A-Z0-9_]*)\}$/;
|
||||
function buildEnvSecretRef(id: string): SecretRef {
|
||||
return { source: "env", id };
|
||||
}
|
||||
|
||||
function parseEnvSecretRef(value: string): SecretRef | null {
|
||||
const match = ENV_REF_PATTERN.exec(value);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return buildEnvSecretRef(match[1]);
|
||||
}
|
||||
|
||||
function inferProviderEnvSecretRef(provider: string, value: string): SecretRef | null {
|
||||
const envVars = PROVIDER_ENV_VARS[provider];
|
||||
if (!envVars || value.length === 0) {
|
||||
return null;
|
||||
}
|
||||
for (const envVar of envVars) {
|
||||
const envValue = normalizeSecretInput(process.env[envVar] ?? "");
|
||||
if (envValue && envValue === value) {
|
||||
return buildEnvSecretRef(envVar);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveApiKeySecretInput(provider: string, input: SecretInput): SecretInput {
|
||||
if (isSecretRef(input)) {
|
||||
return input;
|
||||
}
|
||||
const normalized = normalizeSecretInput(input);
|
||||
const inlineEnvRef = parseEnvSecretRef(normalized);
|
||||
if (inlineEnvRef) {
|
||||
return inlineEnvRef;
|
||||
}
|
||||
const inferredEnvRef = inferProviderEnvSecretRef(provider, normalized);
|
||||
if (inferredEnvRef) {
|
||||
return inferredEnvRef;
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function buildApiKeyCredential(
|
||||
provider: string,
|
||||
input: SecretInput,
|
||||
metadata?: Record<string, string>,
|
||||
): {
|
||||
type: "api_key";
|
||||
provider: string;
|
||||
key?: string;
|
||||
keyRef?: SecretRef;
|
||||
metadata?: Record<string, string>;
|
||||
} {
|
||||
const secretInput = resolveApiKeySecretInput(provider, input);
|
||||
if (typeof secretInput === "string") {
|
||||
return {
|
||||
type: "api_key",
|
||||
provider,
|
||||
key: secretInput,
|
||||
...(metadata ? { metadata } : {}),
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: "api_key",
|
||||
provider,
|
||||
keyRef: secretInput,
|
||||
...(metadata ? { metadata } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export type WriteOAuthCredentialsOptions = {
|
||||
syncSiblingAgents?: boolean;
|
||||
};
|
||||
@@ -110,34 +184,34 @@ export async function writeOAuthCredentials(
|
||||
return profileId;
|
||||
}
|
||||
|
||||
export async function setAnthropicApiKey(key: string, agentDir?: string) {
|
||||
export async function setAnthropicApiKey(key: SecretInput, agentDir?: string) {
|
||||
// Write to resolved agent dir so gateway finds credentials on startup.
|
||||
upsertAuthProfile({
|
||||
profileId: "anthropic:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "anthropic",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("anthropic", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setGeminiApiKey(key: string, agentDir?: string) {
|
||||
export async function setOpenaiApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "openai:default",
|
||||
credential: buildApiKeyCredential("openai", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setGeminiApiKey(key: SecretInput, agentDir?: string) {
|
||||
// Write to resolved agent dir so gateway finds credentials on startup.
|
||||
upsertAuthProfile({
|
||||
profileId: "google:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "google",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("google", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setMinimaxApiKey(
|
||||
key: string,
|
||||
key: SecretInput,
|
||||
agentDir?: string,
|
||||
profileId: string = "minimax:default",
|
||||
) {
|
||||
@@ -145,63 +219,43 @@ export async function setMinimaxApiKey(
|
||||
// Write to resolved agent dir so gateway finds credentials on startup.
|
||||
upsertAuthProfile({
|
||||
profileId,
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider,
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential(provider, key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setMoonshotApiKey(key: string, agentDir?: string) {
|
||||
export async function setMoonshotApiKey(key: SecretInput, agentDir?: string) {
|
||||
// Write to resolved agent dir so gateway finds credentials on startup.
|
||||
upsertAuthProfile({
|
||||
profileId: "moonshot:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "moonshot",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("moonshot", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setKimiCodingApiKey(key: string, agentDir?: string) {
|
||||
export async function setKimiCodingApiKey(key: SecretInput, agentDir?: string) {
|
||||
// Write to resolved agent dir so gateway finds credentials on startup.
|
||||
upsertAuthProfile({
|
||||
profileId: "kimi-coding:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "kimi-coding",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("kimi-coding", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setSyntheticApiKey(key: string, agentDir?: string) {
|
||||
export async function setSyntheticApiKey(key: SecretInput, agentDir?: string) {
|
||||
// Write to resolved agent dir so gateway finds credentials on startup.
|
||||
upsertAuthProfile({
|
||||
profileId: "synthetic:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "synthetic",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("synthetic", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setVeniceApiKey(key: string, agentDir?: string) {
|
||||
export async function setVeniceApiKey(key: SecretInput, agentDir?: string) {
|
||||
// Write to resolved agent dir so gateway finds credentials on startup.
|
||||
upsertAuthProfile({
|
||||
profileId: "venice:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "venice",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("venice", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
@@ -214,41 +268,29 @@ export const TOGETHER_DEFAULT_MODEL_REF = "together/moonshotai/Kimi-K2.5";
|
||||
export const LITELLM_DEFAULT_MODEL_REF = "litellm/claude-opus-4-6";
|
||||
export const VERCEL_AI_GATEWAY_DEFAULT_MODEL_REF = "vercel-ai-gateway/anthropic/claude-opus-4.6";
|
||||
|
||||
export async function setZaiApiKey(key: string, agentDir?: string) {
|
||||
export async function setZaiApiKey(key: SecretInput, agentDir?: string) {
|
||||
// Write to resolved agent dir so gateway finds credentials on startup.
|
||||
upsertAuthProfile({
|
||||
profileId: "zai:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "zai",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("zai", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setXiaomiApiKey(key: string, agentDir?: string) {
|
||||
export async function setXiaomiApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "xiaomi:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "xiaomi",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("xiaomi", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setOpenrouterApiKey(key: string, agentDir?: string) {
|
||||
export async function setOpenrouterApiKey(key: SecretInput, agentDir?: string) {
|
||||
// Never persist the literal "undefined" (e.g. when prompt returns undefined and caller used String(key)).
|
||||
const safeKey = key === "undefined" ? "" : key;
|
||||
const safeKey = typeof key === "string" && key === "undefined" ? "" : key;
|
||||
upsertAuthProfile({
|
||||
profileId: "openrouter:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "openrouter",
|
||||
key: safeKey,
|
||||
},
|
||||
credential: buildApiKeyCredential("openrouter", safeKey),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
@@ -256,107 +298,73 @@ export async function setOpenrouterApiKey(key: string, agentDir?: string) {
|
||||
export async function setCloudflareAiGatewayConfig(
|
||||
accountId: string,
|
||||
gatewayId: string,
|
||||
apiKey: string,
|
||||
apiKey: SecretInput,
|
||||
agentDir?: string,
|
||||
) {
|
||||
const normalizedAccountId = accountId.trim();
|
||||
const normalizedGatewayId = gatewayId.trim();
|
||||
const normalizedKey = apiKey.trim();
|
||||
upsertAuthProfile({
|
||||
profileId: "cloudflare-ai-gateway:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "cloudflare-ai-gateway",
|
||||
key: normalizedKey,
|
||||
metadata: {
|
||||
accountId: normalizedAccountId,
|
||||
gatewayId: normalizedGatewayId,
|
||||
},
|
||||
},
|
||||
credential: buildApiKeyCredential("cloudflare-ai-gateway", apiKey, {
|
||||
accountId: normalizedAccountId,
|
||||
gatewayId: normalizedGatewayId,
|
||||
}),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setLitellmApiKey(key: string, agentDir?: string) {
|
||||
export async function setLitellmApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "litellm:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "litellm",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("litellm", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setVercelAiGatewayApiKey(key: string, agentDir?: string) {
|
||||
export async function setVercelAiGatewayApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "vercel-ai-gateway:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "vercel-ai-gateway",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("vercel-ai-gateway", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setOpencodeZenApiKey(key: string, agentDir?: string) {
|
||||
export async function setOpencodeZenApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "opencode:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "opencode",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("opencode", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setTogetherApiKey(key: string, agentDir?: string) {
|
||||
export async function setTogetherApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "together:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "together",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("together", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export async function setHuggingfaceApiKey(key: string, agentDir?: string) {
|
||||
export async function setHuggingfaceApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "huggingface:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "huggingface",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("huggingface", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export function setQianfanApiKey(key: string, agentDir?: string) {
|
||||
export function setQianfanApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "qianfan:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "qianfan",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("qianfan", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
export function setXaiApiKey(key: string, agentDir?: string) {
|
||||
export function setXaiApiKey(key: SecretInput, agentDir?: string) {
|
||||
upsertAuthProfile({
|
||||
profileId: "xai:default",
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider: "xai",
|
||||
key,
|
||||
},
|
||||
credential: buildApiKeyCredential("xai", key),
|
||||
agentDir: resolveAuthAgentDir(agentDir),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -55,6 +55,7 @@ export {
|
||||
CLOUDFLARE_AI_GATEWAY_DEFAULT_MODEL_REF,
|
||||
LITELLM_DEFAULT_MODEL_REF,
|
||||
OPENROUTER_DEFAULT_MODEL_REF,
|
||||
setOpenaiApiKey,
|
||||
setAnthropicApiKey,
|
||||
setCloudflareAiGatewayConfig,
|
||||
setQianfanApiKey,
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { ModelProviderConfig } from "../config/types.models.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import { fetchWithTimeout } from "../utils/fetch-timeout.js";
|
||||
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
|
||||
import type { WizardPrompter } from "../wizard/prompts.js";
|
||||
import { applyPrimaryModel } from "./model-picker.js";
|
||||
import { normalizeAlias } from "./models/shared.js";
|
||||
@@ -521,7 +522,7 @@ export function applyCustomApiConfig(params: ApplyCustomApiConfigParams): Custom
|
||||
const mergedModels = hasModel ? existingModels : [...existingModels, nextModel];
|
||||
const { apiKey: existingApiKey, ...existingProviderRest } = existingProvider ?? {};
|
||||
const normalizedApiKey =
|
||||
params.apiKey?.trim() || (existingApiKey ? existingApiKey.trim() : undefined);
|
||||
normalizeOptionalSecretInput(params.apiKey) ?? normalizeOptionalSecretInput(existingApiKey);
|
||||
|
||||
let config: OpenClawConfig = {
|
||||
...params.config,
|
||||
|
||||
@@ -38,6 +38,7 @@ import {
|
||||
setLitellmApiKey,
|
||||
setMinimaxApiKey,
|
||||
setMoonshotApiKey,
|
||||
setOpenaiApiKey,
|
||||
setOpencodeZenApiKey,
|
||||
setOpenrouterApiKey,
|
||||
setSyntheticApiKey,
|
||||
@@ -381,15 +382,16 @@ export async function applyNonInteractiveAuthChoice(params: {
|
||||
flagName: "--openai-api-key",
|
||||
envVar: "OPENAI_API_KEY",
|
||||
runtime,
|
||||
allowProfile: false,
|
||||
});
|
||||
if (!resolved) {
|
||||
return null;
|
||||
}
|
||||
const key = resolved.key;
|
||||
const result = upsertSharedEnvVar({ key: "OPENAI_API_KEY", value: key });
|
||||
process.env.OPENAI_API_KEY = key;
|
||||
runtime.log(`Saved OPENAI_API_KEY to ${shortenHomePath(result.path)}`);
|
||||
await setOpenaiApiKey(resolved.key);
|
||||
nextConfig = applyAuthProfileConfig(nextConfig, {
|
||||
profileId: "openai:default",
|
||||
provider: "openai",
|
||||
mode: "api_key",
|
||||
});
|
||||
return applyOpenAIConfig(nextConfig);
|
||||
}
|
||||
|
||||
|
||||
74
src/config/config.secrets-schema.test.ts
Normal file
74
src/config/config.secrets-schema.test.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { validateConfigObjectRaw } from "./validation.js";
|
||||
|
||||
describe("config secret refs schema", () => {
|
||||
it("accepts top-level secrets sources and model apiKey refs", () => {
|
||||
const result = validateConfigObjectRaw({
|
||||
secrets: {
|
||||
sources: {
|
||||
env: { type: "env" },
|
||||
file: { type: "sops", path: "~/.openclaw/secrets.enc.json", timeoutMs: 10_000 },
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", id: "OPENAI_API_KEY" },
|
||||
models: [{ id: "gpt-5", name: "gpt-5" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts googlechat serviceAccount refs", () => {
|
||||
const result = validateConfigObjectRaw({
|
||||
channels: {
|
||||
googlechat: {
|
||||
serviceAccountRef: { source: "file", id: "/channels/googlechat/serviceAccount" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts skills entry apiKey refs", () => {
|
||||
const result = validateConfigObjectRaw({
|
||||
skills: {
|
||||
entries: {
|
||||
"review-pr": {
|
||||
enabled: true,
|
||||
apiKey: { source: "env", id: "SKILL_REVIEW_PR_API_KEY" },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects invalid secret ref id", () => {
|
||||
const result = validateConfigObjectRaw({
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", id: "bad id with spaces" },
|
||||
models: [{ id: "gpt-5", name: "gpt-5" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) {
|
||||
expect(
|
||||
result.issues.some((issue) => issue.path.includes("models.providers.openai.apiKey")),
|
||||
).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,11 +1,14 @@
|
||||
export {
|
||||
clearConfigCache,
|
||||
clearRuntimeConfigSnapshot,
|
||||
createConfigIO,
|
||||
getRuntimeConfigSnapshot,
|
||||
loadConfig,
|
||||
parseConfigJson5,
|
||||
readConfigFileSnapshot,
|
||||
readConfigFileSnapshotForWrite,
|
||||
resolveConfigSnapshotHash,
|
||||
setRuntimeConfigSnapshot,
|
||||
writeConfigFile,
|
||||
} from "./io.js";
|
||||
export { migrateLegacyConfig } from "./legacy-migrate.js";
|
||||
|
||||
63
src/config/io.runtime-snapshot-write.test.ts
Normal file
63
src/config/io.runtime-snapshot-write.test.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { withTempHome } from "./home-env.test-harness.js";
|
||||
import {
|
||||
clearConfigCache,
|
||||
clearRuntimeConfigSnapshot,
|
||||
loadConfig,
|
||||
setRuntimeConfigSnapshot,
|
||||
writeConfigFile,
|
||||
} from "./io.js";
|
||||
import type { OpenClawConfig } from "./types.js";
|
||||
|
||||
describe("runtime config snapshot writes", () => {
|
||||
it("preserves source secret refs when writeConfigFile receives runtime-resolved config", async () => {
|
||||
await withTempHome("openclaw-config-runtime-write-", async (home) => {
|
||||
const configPath = path.join(home, ".openclaw", "openclaw.json");
|
||||
const sourceConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", id: "OPENAI_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const runtimeConfig: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: "sk-runtime-resolved",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
await fs.mkdir(path.dirname(configPath), { recursive: true });
|
||||
await fs.writeFile(configPath, `${JSON.stringify(sourceConfig, null, 2)}\n`, "utf8");
|
||||
|
||||
try {
|
||||
setRuntimeConfigSnapshot(runtimeConfig, sourceConfig);
|
||||
expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-runtime-resolved");
|
||||
|
||||
await writeConfigFile(loadConfig());
|
||||
|
||||
const persisted = JSON.parse(await fs.readFile(configPath, "utf8")) as {
|
||||
models?: { providers?: { openai?: { apiKey?: unknown } } };
|
||||
};
|
||||
expect(persisted.models?.providers?.openai?.apiKey).toEqual({
|
||||
source: "env",
|
||||
id: "OPENAI_API_KEY",
|
||||
});
|
||||
} finally {
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1061,6 +1061,8 @@ let configCache: {
|
||||
expiresAt: number;
|
||||
config: OpenClawConfig;
|
||||
} | null = null;
|
||||
let runtimeConfigSnapshot: OpenClawConfig | null = null;
|
||||
let runtimeConfigSourceSnapshot: OpenClawConfig | null = null;
|
||||
|
||||
function resolveConfigCacheMs(env: NodeJS.ProcessEnv): number {
|
||||
const raw = env.OPENCLAW_CONFIG_CACHE_MS?.trim();
|
||||
@@ -1088,7 +1090,29 @@ export function clearConfigCache(): void {
|
||||
configCache = null;
|
||||
}
|
||||
|
||||
export function setRuntimeConfigSnapshot(
|
||||
config: OpenClawConfig,
|
||||
sourceConfig?: OpenClawConfig,
|
||||
): void {
|
||||
runtimeConfigSnapshot = config;
|
||||
runtimeConfigSourceSnapshot = sourceConfig ?? null;
|
||||
clearConfigCache();
|
||||
}
|
||||
|
||||
export function clearRuntimeConfigSnapshot(): void {
|
||||
runtimeConfigSnapshot = null;
|
||||
runtimeConfigSourceSnapshot = null;
|
||||
clearConfigCache();
|
||||
}
|
||||
|
||||
export function getRuntimeConfigSnapshot(): OpenClawConfig | null {
|
||||
return runtimeConfigSnapshot;
|
||||
}
|
||||
|
||||
export function loadConfig(): OpenClawConfig {
|
||||
if (runtimeConfigSnapshot) {
|
||||
return runtimeConfigSnapshot;
|
||||
}
|
||||
const io = createConfigIO();
|
||||
const configPath = io.configPath;
|
||||
const now = Date.now();
|
||||
@@ -1125,9 +1149,14 @@ export async function writeConfigFile(
|
||||
options: ConfigWriteOptions = {},
|
||||
): Promise<void> {
|
||||
const io = createConfigIO();
|
||||
let nextCfg = cfg;
|
||||
if (runtimeConfigSnapshot && runtimeConfigSourceSnapshot) {
|
||||
const runtimePatch = createMergePatch(runtimeConfigSnapshot, cfg);
|
||||
nextCfg = coerceConfig(applyMergePatch(runtimeConfigSourceSnapshot, runtimePatch));
|
||||
}
|
||||
const sameConfigPath =
|
||||
options.expectedConfigPath === undefined || options.expectedConfigPath === io.configPath;
|
||||
await io.writeConfigFile(cfg, {
|
||||
await io.writeConfigFile(nextCfg, {
|
||||
envSnapshotForRestore: sameConfigPath ? options.envSnapshotForRestore : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -74,7 +74,6 @@ describe("redactConfigSnapshot", () => {
|
||||
},
|
||||
shortSecret: { token: "short" },
|
||||
});
|
||||
|
||||
const result = redactConfigSnapshot(snapshot);
|
||||
const cfg = result.config as typeof snapshot.config;
|
||||
|
||||
@@ -91,6 +90,45 @@ describe("redactConfigSnapshot", () => {
|
||||
expect(cfg.shortSecret.token).toBe(REDACTED_SENTINEL);
|
||||
});
|
||||
|
||||
it("redacts googlechat serviceAccount object payloads", () => {
|
||||
const snapshot = makeSnapshot({
|
||||
channels: {
|
||||
googlechat: {
|
||||
serviceAccount: {
|
||||
type: "service_account",
|
||||
client_email: "bot@example.iam.gserviceaccount.com",
|
||||
private_key: "-----BEGIN PRIVATE KEY-----secret-----END PRIVATE KEY-----",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = redactConfigSnapshot(snapshot);
|
||||
const channels = result.config.channels as Record<string, Record<string, unknown>>;
|
||||
expect(channels.googlechat.serviceAccount).toBe(REDACTED_SENTINEL);
|
||||
});
|
||||
|
||||
it("redacts object-valued apiKey refs in model providers", () => {
|
||||
const snapshot = makeSnapshot({
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: { source: "env", id: "OPENAI_API_KEY" },
|
||||
baseUrl: "https://api.openai.com",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = redactConfigSnapshot(snapshot);
|
||||
const models = result.config.models as Record<string, Record<string, Record<string, unknown>>>;
|
||||
expect(models.providers.openai.apiKey).toEqual({
|
||||
source: REDACTED_SENTINEL,
|
||||
id: REDACTED_SENTINEL,
|
||||
});
|
||||
expect(models.providers.openai.baseUrl).toBe("https://api.openai.com");
|
||||
});
|
||||
|
||||
it("preserves non-sensitive fields", () => {
|
||||
const snapshot = makeSnapshot({
|
||||
ui: { seamColor: "#0088cc" },
|
||||
|
||||
@@ -17,6 +17,31 @@ function isEnvVarPlaceholder(value: string): boolean {
|
||||
return ENV_VAR_PLACEHOLDER_PATTERN.test(value.trim());
|
||||
}
|
||||
|
||||
function isWholeObjectSensitivePath(path: string): boolean {
|
||||
const lowered = path.toLowerCase();
|
||||
return lowered.endsWith("serviceaccount") || lowered.endsWith("serviceaccountref");
|
||||
}
|
||||
|
||||
function collectSensitiveStrings(value: unknown, values: string[]): void {
|
||||
if (typeof value === "string") {
|
||||
if (!isEnvVarPlaceholder(value)) {
|
||||
values.push(value);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) {
|
||||
collectSensitiveStrings(item, values);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
for (const item of Object.values(value as Record<string, unknown>)) {
|
||||
collectSensitiveStrings(item, values);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function isExtensionPath(path: string): boolean {
|
||||
return (
|
||||
path === "plugins" ||
|
||||
@@ -159,7 +184,19 @@ function redactObjectWithLookup(
|
||||
result[key] = REDACTED_SENTINEL;
|
||||
values.push(value);
|
||||
} else if (typeof value === "object" && value !== null) {
|
||||
result[key] = redactObjectWithLookup(value, lookup, candidate, values, hints);
|
||||
if (hints[candidate]?.sensitive === true && !Array.isArray(value)) {
|
||||
collectSensitiveStrings(value, values);
|
||||
result[key] = REDACTED_SENTINEL;
|
||||
} else {
|
||||
result[key] = redactObjectWithLookup(value, lookup, candidate, values, hints);
|
||||
}
|
||||
} else if (
|
||||
hints[candidate]?.sensitive === true &&
|
||||
value !== undefined &&
|
||||
value !== null
|
||||
) {
|
||||
// Keep primitives at explicitly-sensitive paths fully redacted.
|
||||
result[key] = REDACTED_SENTINEL;
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -228,6 +265,16 @@ function redactObjectGuessing(
|
||||
) {
|
||||
result[key] = REDACTED_SENTINEL;
|
||||
values.push(value);
|
||||
} else if (
|
||||
!isExplicitlyNonSensitivePath(hints, [dotPath, wildcardPath]) &&
|
||||
isSensitivePath(dotPath) &&
|
||||
isWholeObjectSensitivePath(dotPath) &&
|
||||
value &&
|
||||
typeof value === "object" &&
|
||||
!Array.isArray(value)
|
||||
) {
|
||||
collectSensitiveStrings(value, values);
|
||||
result[key] = REDACTED_SENTINEL;
|
||||
} else if (typeof value === "object" && value !== null) {
|
||||
result[key] = redactObjectGuessing(value, dotPath, values, hints);
|
||||
} else {
|
||||
|
||||
@@ -109,6 +109,7 @@ describe("mapSensitivePaths", () => {
|
||||
expect(hints["agents.defaults.memorySearch.remote.apiKey"]?.sensitive).toBe(true);
|
||||
expect(hints["agents.list[].memorySearch.remote.apiKey"]?.sensitive).toBe(true);
|
||||
expect(hints["channels.discord.accounts.*.token"]?.sensitive).toBe(true);
|
||||
expect(hints["channels.googlechat.serviceAccount"]?.sensitive).toBe(true);
|
||||
expect(hints["gateway.auth.token"]?.sensitive).toBe(true);
|
||||
expect(hints["skills.entries.*.apiKey"]?.sensitive).toBe(true);
|
||||
});
|
||||
|
||||
@@ -107,7 +107,13 @@ const NORMALIZED_SENSITIVE_KEY_WHITELIST_SUFFIXES = SENSITIVE_KEY_WHITELIST_SUFF
|
||||
suffix.toLowerCase(),
|
||||
);
|
||||
|
||||
const SENSITIVE_PATTERNS = [/token$/i, /password/i, /secret/i, /api.?key/i];
|
||||
const SENSITIVE_PATTERNS = [
|
||||
/token$/i,
|
||||
/password/i,
|
||||
/secret/i,
|
||||
/api.?key/i,
|
||||
/serviceaccount(?:ref)?$/i,
|
||||
];
|
||||
|
||||
function isWhitelistedSensitivePath(path: string): boolean {
|
||||
const lowerPath = path.toLowerCase();
|
||||
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
ReplyToMode,
|
||||
} from "./types.base.js";
|
||||
import type { DmConfig } from "./types.messages.js";
|
||||
import type { SecretRef } from "./types.secrets.js";
|
||||
|
||||
export type GoogleChatDmConfig = {
|
||||
/** If false, ignore all incoming Google Chat DMs. Default: true. */
|
||||
@@ -58,8 +59,10 @@ export type GoogleChatAccountConfig = {
|
||||
defaultTo?: string;
|
||||
/** Per-space configuration keyed by space id or name. */
|
||||
groups?: Record<string, GoogleChatGroupConfig>;
|
||||
/** Service account JSON (inline string or object). */
|
||||
serviceAccount?: string | Record<string, unknown>;
|
||||
/** Service account JSON (inline string, object, or secret reference). */
|
||||
serviceAccount?: string | Record<string, unknown> | SecretRef;
|
||||
/** Explicit secret reference for service account JSON. */
|
||||
serviceAccountRef?: SecretRef;
|
||||
/** Service account JSON file path. */
|
||||
serviceAccountFile?: string;
|
||||
/** Webhook audience type (app-url or project-number). */
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { SecretInput } from "./types.secrets.js";
|
||||
|
||||
export type ModelApi =
|
||||
| "openai-completions"
|
||||
| "openai-responses"
|
||||
@@ -43,7 +45,7 @@ export type ModelDefinitionConfig = {
|
||||
|
||||
export type ModelProviderConfig = {
|
||||
baseUrl: string;
|
||||
apiKey?: string;
|
||||
apiKey?: SecretInput;
|
||||
auth?: ModelProviderAuthMode;
|
||||
api?: ModelApi;
|
||||
headers?: Record<string, string>;
|
||||
|
||||
@@ -22,6 +22,7 @@ import type {
|
||||
import type { ModelsConfig } from "./types.models.js";
|
||||
import type { NodeHostConfig } from "./types.node-host.js";
|
||||
import type { PluginsConfig } from "./types.plugins.js";
|
||||
import type { SecretsConfig } from "./types.secrets.js";
|
||||
import type { SkillsConfig } from "./types.skills.js";
|
||||
import type { ToolsConfig } from "./types.tools.js";
|
||||
|
||||
@@ -75,6 +76,7 @@ export type OpenClawConfig = {
|
||||
avatar?: string;
|
||||
};
|
||||
};
|
||||
secrets?: SecretsConfig;
|
||||
skills?: SkillsConfig;
|
||||
plugins?: PluginsConfig;
|
||||
models?: ModelsConfig;
|
||||
|
||||
49
src/config/types.secrets.ts
Normal file
49
src/config/types.secrets.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
export type SecretRefSource = "env" | "file";
|
||||
|
||||
/**
|
||||
* Stable identifier for a secret in a configured source.
|
||||
* Examples:
|
||||
* - env source: "OPENAI_API_KEY"
|
||||
* - file source: "/providers/openai/api_key" (JSON pointer)
|
||||
*/
|
||||
export type SecretRef = {
|
||||
source: SecretRefSource;
|
||||
id: string;
|
||||
};
|
||||
|
||||
export type SecretInput = string | SecretRef;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
export function isSecretRef(value: unknown): value is SecretRef {
|
||||
if (!isRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
if (Object.keys(value).length !== 2) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
(value.source === "env" || value.source === "file") &&
|
||||
typeof value.id === "string" &&
|
||||
value.id.trim().length > 0
|
||||
);
|
||||
}
|
||||
|
||||
export type EnvSecretSourceConfig = {
|
||||
type?: "env";
|
||||
};
|
||||
|
||||
export type SopsSecretSourceConfig = {
|
||||
type: "sops";
|
||||
path: string;
|
||||
timeoutMs?: number;
|
||||
};
|
||||
|
||||
export type SecretsConfig = {
|
||||
sources?: {
|
||||
env?: EnvSecretSourceConfig;
|
||||
file?: SopsSecretSourceConfig;
|
||||
};
|
||||
};
|
||||
@@ -1,6 +1,8 @@
|
||||
import type { SecretInput } from "./types.secrets.js";
|
||||
|
||||
export type SkillConfig = {
|
||||
enabled?: boolean;
|
||||
apiKey?: string;
|
||||
apiKey?: SecretInput;
|
||||
env?: Record<string, string>;
|
||||
config?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
@@ -22,6 +22,7 @@ export * from "./types.msteams.js";
|
||||
export * from "./types.plugins.js";
|
||||
export * from "./types.queue.js";
|
||||
export * from "./types.sandbox.js";
|
||||
export * from "./types.secrets.js";
|
||||
export * from "./types.signal.js";
|
||||
export * from "./types.skills.js";
|
||||
export * from "./types.slack.js";
|
||||
|
||||
@@ -3,6 +3,49 @@ import { isSafeExecutableValue } from "../infra/exec-safety.js";
|
||||
import { createAllowDenyChannelRulesSchema } from "./zod-schema.allowdeny.js";
|
||||
import { sensitive } from "./zod-schema.sensitive.js";
|
||||
|
||||
const SECRET_REF_ID_PATTERN = /^[A-Za-z0-9_./:=-](?:[A-Za-z0-9_./:=~-]{0,127})$/;
|
||||
|
||||
export const SecretRefSchema = z
|
||||
.object({
|
||||
source: z.enum(["env", "file"]),
|
||||
id: z
|
||||
.string()
|
||||
.regex(
|
||||
SECRET_REF_ID_PATTERN,
|
||||
"Secret reference id must match /^[A-Za-z0-9_./:=-](?:[A-Za-z0-9_./:=~-]{0,127})$/",
|
||||
),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const SecretInputSchema = z.union([z.string(), SecretRefSchema]);
|
||||
|
||||
const SecretsEnvSourceSchema = z
|
||||
.object({
|
||||
type: z.literal("env").optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const SecretsFileSourceSchema = z
|
||||
.object({
|
||||
type: z.literal("sops"),
|
||||
path: z.string().min(1),
|
||||
timeoutMs: z.number().int().positive().max(120000).optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const SecretsConfigSchema = z
|
||||
.object({
|
||||
sources: z
|
||||
.object({
|
||||
env: SecretsEnvSourceSchema.optional(),
|
||||
file: SecretsFileSourceSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional();
|
||||
|
||||
export const ModelApiSchema = z.union([
|
||||
z.literal("openai-completions"),
|
||||
z.literal("openai-responses"),
|
||||
@@ -58,7 +101,7 @@ export const ModelDefinitionSchema = z
|
||||
export const ModelProviderSchema = z
|
||||
.object({
|
||||
baseUrl: z.string().min(1),
|
||||
apiKey: z.string().optional().register(sensitive),
|
||||
apiKey: SecretInputSchema.optional().register(sensitive),
|
||||
auth: z
|
||||
.union([z.literal("api-key"), z.literal("aws-sdk"), z.literal("oauth"), z.literal("token")])
|
||||
.optional(),
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
MarkdownConfigSchema,
|
||||
MSTeamsReplyStyleSchema,
|
||||
ProviderCommandsSchema,
|
||||
SecretRefSchema,
|
||||
ReplyToModeSchema,
|
||||
RetryConfigSchema,
|
||||
TtsConfigSchema,
|
||||
@@ -519,7 +520,11 @@ export const GoogleChatAccountSchema = z
|
||||
groupAllowFrom: z.array(z.union([z.string(), z.number()])).optional(),
|
||||
groups: z.record(z.string(), GoogleChatGroupSchema.optional()).optional(),
|
||||
defaultTo: z.string().optional(),
|
||||
serviceAccount: z.union([z.string(), z.record(z.string(), z.unknown())]).optional(),
|
||||
serviceAccount: z
|
||||
.union([z.string(), z.record(z.string(), z.unknown()), SecretRefSchema])
|
||||
.optional()
|
||||
.register(sensitive),
|
||||
serviceAccountRef: SecretRefSchema.optional().register(sensitive),
|
||||
serviceAccountFile: z.string().optional(),
|
||||
audienceType: z.enum(["app-url", "project-number"]).optional(),
|
||||
audience: z.string().optional(),
|
||||
|
||||
@@ -2,7 +2,12 @@ import { z } from "zod";
|
||||
import { ToolsSchema } from "./zod-schema.agent-runtime.js";
|
||||
import { AgentsSchema, AudioSchema, BindingsSchema, BroadcastSchema } from "./zod-schema.agents.js";
|
||||
import { ApprovalsSchema } from "./zod-schema.approvals.js";
|
||||
import { HexColorSchema, ModelsConfigSchema } from "./zod-schema.core.js";
|
||||
import {
|
||||
HexColorSchema,
|
||||
ModelsConfigSchema,
|
||||
SecretInputSchema,
|
||||
SecretsConfigSchema,
|
||||
} from "./zod-schema.core.js";
|
||||
import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-schema.hooks.js";
|
||||
import { InstallRecordShape } from "./zod-schema.installs.js";
|
||||
import { ChannelsSchema } from "./zod-schema.providers.js";
|
||||
@@ -263,6 +268,7 @@ export const OpenClawSchema = z
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
secrets: SecretsConfigSchema,
|
||||
auth: z
|
||||
.object({
|
||||
profiles: z
|
||||
@@ -612,7 +618,7 @@ export const OpenClawSchema = z
|
||||
z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
apiKey: z.string().optional().register(sensitive),
|
||||
apiKey: SecretInputSchema.optional().register(sensitive),
|
||||
env: z.record(z.string(), z.string()).optional(),
|
||||
config: z.record(z.string(), z.unknown()).optional(),
|
||||
})
|
||||
|
||||
@@ -250,7 +250,7 @@ export function startGatewayConfigReloader(opts: {
|
||||
initialConfig: OpenClawConfig;
|
||||
readSnapshot: () => Promise<ConfigFileSnapshot>;
|
||||
onHotReload: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
|
||||
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void;
|
||||
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
|
||||
log: {
|
||||
info: (msg: string) => void;
|
||||
warn: (msg: string) => void;
|
||||
@@ -301,42 +301,53 @@ export function startGatewayConfigReloader(opts: {
|
||||
}
|
||||
const nextConfig = snapshot.config;
|
||||
const changedPaths = diffConfigPaths(currentConfig, nextConfig);
|
||||
currentConfig = nextConfig;
|
||||
settings = resolveGatewayReloadSettings(nextConfig);
|
||||
const nextSettings = resolveGatewayReloadSettings(nextConfig);
|
||||
if (changedPaths.length === 0) {
|
||||
currentConfig = nextConfig;
|
||||
settings = nextSettings;
|
||||
return;
|
||||
}
|
||||
|
||||
opts.log.info(`config change detected; evaluating reload (${changedPaths.join(", ")})`);
|
||||
const plan = buildGatewayReloadPlan(changedPaths);
|
||||
if (settings.mode === "off") {
|
||||
if (nextSettings.mode === "off") {
|
||||
opts.log.info("config reload disabled (gateway.reload.mode=off)");
|
||||
currentConfig = nextConfig;
|
||||
settings = nextSettings;
|
||||
return;
|
||||
}
|
||||
if (settings.mode === "restart") {
|
||||
if (nextSettings.mode === "restart") {
|
||||
if (!restartQueued) {
|
||||
restartQueued = true;
|
||||
opts.onRestart(plan, nextConfig);
|
||||
await opts.onRestart(plan, nextConfig);
|
||||
}
|
||||
currentConfig = nextConfig;
|
||||
settings = nextSettings;
|
||||
return;
|
||||
}
|
||||
if (plan.restartGateway) {
|
||||
if (settings.mode === "hot") {
|
||||
if (nextSettings.mode === "hot") {
|
||||
opts.log.warn(
|
||||
`config reload requires gateway restart; hot mode ignoring (${plan.restartReasons.join(
|
||||
", ",
|
||||
)})`,
|
||||
);
|
||||
currentConfig = nextConfig;
|
||||
settings = nextSettings;
|
||||
return;
|
||||
}
|
||||
if (!restartQueued) {
|
||||
restartQueued = true;
|
||||
opts.onRestart(plan, nextConfig);
|
||||
await opts.onRestart(plan, nextConfig);
|
||||
}
|
||||
currentConfig = nextConfig;
|
||||
settings = nextSettings;
|
||||
return;
|
||||
}
|
||||
|
||||
await opts.onHotReload(plan, nextConfig);
|
||||
currentConfig = nextConfig;
|
||||
settings = nextSettings;
|
||||
} catch (err) {
|
||||
opts.log.error(`config reload failed: ${String(err)}`);
|
||||
} finally {
|
||||
|
||||
@@ -97,6 +97,7 @@ const METHOD_SCOPE_GROUPS: Record<OperatorScope, readonly string[]> = {
|
||||
"agents.delete",
|
||||
"skills.install",
|
||||
"skills.update",
|
||||
"secrets.reload",
|
||||
"cron.add",
|
||||
"cron.update",
|
||||
"cron.remove",
|
||||
|
||||
@@ -48,6 +48,7 @@ const BASE_METHODS = [
|
||||
"update.run",
|
||||
"voicewake.get",
|
||||
"voicewake.set",
|
||||
"secrets.reload",
|
||||
"sessions.list",
|
||||
"sessions.preview",
|
||||
"sessions.patch",
|
||||
|
||||
43
src/gateway/server-methods/secrets.test.ts
Normal file
43
src/gateway/server-methods/secrets.test.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createSecretsHandlers } from "./secrets.js";
|
||||
|
||||
describe("secrets handlers", () => {
|
||||
it("responds with warning count on successful reload", async () => {
|
||||
const handlers = createSecretsHandlers({
|
||||
reloadSecrets: vi.fn().mockResolvedValue({ warningCount: 2 }),
|
||||
});
|
||||
const respond = vi.fn();
|
||||
await handlers["secrets.reload"]({
|
||||
req: { type: "req", id: "1", method: "secrets.reload" },
|
||||
params: {},
|
||||
client: null,
|
||||
isWebchatConnect: () => false,
|
||||
respond,
|
||||
context: {} as never,
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 2 });
|
||||
});
|
||||
|
||||
it("returns unavailable when reload fails", async () => {
|
||||
const handlers = createSecretsHandlers({
|
||||
reloadSecrets: vi.fn().mockRejectedValue(new Error("reload failed")),
|
||||
});
|
||||
const respond = vi.fn();
|
||||
await handlers["secrets.reload"]({
|
||||
req: { type: "req", id: "1", method: "secrets.reload" },
|
||||
params: {},
|
||||
client: null,
|
||||
isWebchatConnect: () => false,
|
||||
respond,
|
||||
context: {} as never,
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: "UNAVAILABLE",
|
||||
message: "Error: reload failed",
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
17
src/gateway/server-methods/secrets.ts
Normal file
17
src/gateway/server-methods/secrets.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { ErrorCodes, errorShape } from "../protocol/index.js";
|
||||
import type { GatewayRequestHandlers } from "./types.js";
|
||||
|
||||
export function createSecretsHandlers(params: {
|
||||
reloadSecrets: () => Promise<{ warningCount: number }>;
|
||||
}): GatewayRequestHandlers {
|
||||
return {
|
||||
"secrets.reload": async ({ respond }) => {
|
||||
try {
|
||||
const result = await params.reloadSecrets();
|
||||
respond(true, { ok: true, warningCount: result.warningCount });
|
||||
} catch (err) {
|
||||
respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, String(err)));
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { createDefaultDeps } from "../cli/deps.js";
|
||||
import { isRestartEnabled } from "../config/commands.js";
|
||||
import {
|
||||
CONFIG_PATH,
|
||||
type OpenClawConfig,
|
||||
isNixMode,
|
||||
loadConfig,
|
||||
migrateLegacyConfig,
|
||||
@@ -45,6 +46,12 @@ import { createEmptyPluginRegistry } from "../plugins/registry.js";
|
||||
import type { PluginServicesHandle } from "../plugins/services.js";
|
||||
import { getTotalQueueSize } from "../process/command-queue.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import {
|
||||
activateSecretsRuntimeSnapshot,
|
||||
clearSecretsRuntimeSnapshot,
|
||||
getActiveSecretsRuntimeSnapshot,
|
||||
prepareSecretsRuntimeSnapshot,
|
||||
} from "../secrets/runtime.js";
|
||||
import { runOnboardingWizard } from "../wizard/onboarding.js";
|
||||
import { createAuthRateLimiter, type AuthRateLimiter } from "./auth-rate-limit.js";
|
||||
import { startChannelHealthMonitor } from "./channel-health-monitor.js";
|
||||
@@ -68,6 +75,7 @@ import { GATEWAY_EVENTS, listGatewayMethods } from "./server-methods-list.js";
|
||||
import { coreGatewayHandlers } from "./server-methods.js";
|
||||
import { createExecApprovalHandlers } from "./server-methods/exec-approval.js";
|
||||
import { safeParseJson } from "./server-methods/nodes.helpers.js";
|
||||
import { createSecretsHandlers } from "./server-methods/secrets.js";
|
||||
import { hasConnectedMobileNode } from "./server-mobile-nodes.js";
|
||||
import { loadGatewayModelCatalog } from "./server-model-catalog.js";
|
||||
import { createNodeSubscriptionManager } from "./server-node-subscriptions.js";
|
||||
@@ -107,6 +115,7 @@ const logReload = log.child("reload");
|
||||
const logHooks = log.child("hooks");
|
||||
const logPlugins = log.child("plugins");
|
||||
const logWsControl = log.child("ws");
|
||||
const logSecrets = log.child("secrets");
|
||||
const gatewayRuntime = runtimeForLogger(log);
|
||||
const canvasRuntime = runtimeForLogger(logCanvas);
|
||||
|
||||
@@ -233,7 +242,63 @@ export async function startGatewayServer(
|
||||
}
|
||||
}
|
||||
|
||||
let cfgAtStart = loadConfig();
|
||||
let secretsDegraded = false;
|
||||
const activateRuntimeSecrets = async (
|
||||
config: OpenClawConfig,
|
||||
params: { reason: "startup" | "reload" | "restart-check"; activate: boolean },
|
||||
) => {
|
||||
try {
|
||||
const prepared = await prepareSecretsRuntimeSnapshot({ config });
|
||||
if (params.activate) {
|
||||
activateSecretsRuntimeSnapshot(prepared);
|
||||
}
|
||||
for (const warning of prepared.warnings) {
|
||||
logSecrets.warn(`[${warning.code}] ${warning.message}`);
|
||||
}
|
||||
if (secretsDegraded) {
|
||||
logSecrets.info(
|
||||
"[SECRETS_RELOADER_RECOVERED] Secret resolution recovered; runtime remained on last-known-good during the outage.",
|
||||
);
|
||||
}
|
||||
secretsDegraded = false;
|
||||
return prepared;
|
||||
} catch (err) {
|
||||
const details = String(err);
|
||||
if (!secretsDegraded) {
|
||||
logSecrets.error(`[SECRETS_RELOADER_DEGRADED] ${details}`);
|
||||
} else {
|
||||
logSecrets.warn(`[SECRETS_RELOADER_DEGRADED] ${details}`);
|
||||
}
|
||||
secretsDegraded = true;
|
||||
if (params.reason === "startup") {
|
||||
throw new Error(`Startup failed: required secrets are unavailable. ${details}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// Fail fast before startup if required refs are unresolved.
|
||||
let cfgAtStart: OpenClawConfig;
|
||||
{
|
||||
const freshSnapshot = await readConfigFileSnapshot();
|
||||
if (!freshSnapshot.valid) {
|
||||
const issues =
|
||||
freshSnapshot.issues.length > 0
|
||||
? freshSnapshot.issues
|
||||
.map((issue) => `${issue.path || "<root>"}: ${issue.message}`)
|
||||
.join("\n")
|
||||
: "Unknown validation issue.";
|
||||
throw new Error(`Invalid config at ${freshSnapshot.path}.\n${issues}`);
|
||||
}
|
||||
await activateRuntimeSecrets(freshSnapshot.config, {
|
||||
reason: "startup",
|
||||
activate: false,
|
||||
});
|
||||
}
|
||||
|
||||
cfgAtStart = loadConfig();
|
||||
const authBootstrap = await ensureGatewayStartupAuth({
|
||||
cfg: cfgAtStart,
|
||||
env: process.env,
|
||||
@@ -253,6 +318,12 @@ export async function startGatewayServer(
|
||||
);
|
||||
}
|
||||
}
|
||||
cfgAtStart = (
|
||||
await activateRuntimeSecrets(cfgAtStart, {
|
||||
reason: "startup",
|
||||
activate: true,
|
||||
})
|
||||
).config;
|
||||
const diagnosticsEnabled = isDiagnosticsEnabled(cfgAtStart);
|
||||
if (diagnosticsEnabled) {
|
||||
startDiagnosticHeartbeat();
|
||||
@@ -560,6 +631,19 @@ export async function startGatewayServer(
|
||||
const execApprovalHandlers = createExecApprovalHandlers(execApprovalManager, {
|
||||
forwarder: execApprovalForwarder,
|
||||
});
|
||||
const secretsHandlers = createSecretsHandlers({
|
||||
reloadSecrets: async () => {
|
||||
const active = getActiveSecretsRuntimeSnapshot();
|
||||
if (!active) {
|
||||
throw new Error("Secrets runtime snapshot is not active.");
|
||||
}
|
||||
const prepared = await activateRuntimeSecrets(active.sourceConfig, {
|
||||
reason: "reload",
|
||||
activate: true,
|
||||
});
|
||||
return { warningCount: prepared.warnings.length };
|
||||
},
|
||||
});
|
||||
|
||||
const canvasHostServerPort = (canvasHostServer as CanvasHostServer | null)?.port;
|
||||
|
||||
@@ -580,6 +664,7 @@ export async function startGatewayServer(
|
||||
extraHandlers: {
|
||||
...pluginRegistry.gatewayHandlers,
|
||||
...execApprovalHandlers,
|
||||
...secretsHandlers,
|
||||
},
|
||||
broadcast,
|
||||
context: {
|
||||
@@ -710,8 +795,27 @@ export async function startGatewayServer(
|
||||
return startGatewayConfigReloader({
|
||||
initialConfig: cfgAtStart,
|
||||
readSnapshot: readConfigFileSnapshot,
|
||||
onHotReload: applyHotReload,
|
||||
onRestart: requestGatewayRestart,
|
||||
onHotReload: async (plan, nextConfig) => {
|
||||
const previousSnapshot = getActiveSecretsRuntimeSnapshot();
|
||||
const prepared = await activateRuntimeSecrets(nextConfig, {
|
||||
reason: "reload",
|
||||
activate: true,
|
||||
});
|
||||
try {
|
||||
await applyHotReload(plan, prepared.config);
|
||||
} catch (err) {
|
||||
if (previousSnapshot) {
|
||||
activateSecretsRuntimeSnapshot(previousSnapshot);
|
||||
} else {
|
||||
clearSecretsRuntimeSnapshot();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
onRestart: async (plan, nextConfig) => {
|
||||
await activateRuntimeSecrets(nextConfig, { reason: "restart-check", activate: false });
|
||||
requestGatewayRestart(plan, nextConfig);
|
||||
},
|
||||
log: {
|
||||
info: (msg) => logReload.info(msg),
|
||||
warn: (msg) => logReload.warn(msg),
|
||||
@@ -764,6 +868,7 @@ export async function startGatewayServer(
|
||||
skillsChangeUnsub();
|
||||
authRateLimiter?.dispose();
|
||||
channelHealthMonitor?.stop();
|
||||
clearSecretsRuntimeSnapshot();
|
||||
await close(opts);
|
||||
},
|
||||
};
|
||||
|
||||
94
src/secrets/json-pointer.ts
Normal file
94
src/secrets/json-pointer.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
function failOrUndefined(params: { onMissing: "throw" | "undefined"; message: string }): undefined {
|
||||
if (params.onMissing === "throw") {
|
||||
throw new Error(params.message);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function decodeJsonPointerToken(token: string): string {
|
||||
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
||||
}
|
||||
|
||||
export function encodeJsonPointerToken(token: string): string {
|
||||
return token.replace(/~/g, "~0").replace(/\//g, "~1");
|
||||
}
|
||||
|
||||
export function readJsonPointer(
|
||||
root: unknown,
|
||||
pointer: string,
|
||||
options: { onMissing?: "throw" | "undefined" } = {},
|
||||
): unknown {
|
||||
const onMissing = options.onMissing ?? "throw";
|
||||
if (!pointer.startsWith("/")) {
|
||||
return failOrUndefined({
|
||||
onMissing,
|
||||
message:
|
||||
'File-backed secret ids must be absolute JSON pointers (for example: "/providers/openai/apiKey").',
|
||||
});
|
||||
}
|
||||
|
||||
const tokens = pointer
|
||||
.slice(1)
|
||||
.split("/")
|
||||
.map((token) => decodeJsonPointerToken(token));
|
||||
|
||||
let current: unknown = root;
|
||||
for (const token of tokens) {
|
||||
if (Array.isArray(current)) {
|
||||
const index = Number.parseInt(token, 10);
|
||||
if (!Number.isFinite(index) || index < 0 || index >= current.length) {
|
||||
return failOrUndefined({
|
||||
onMissing,
|
||||
message: `JSON pointer segment "${token}" is out of bounds.`,
|
||||
});
|
||||
}
|
||||
current = current[index];
|
||||
continue;
|
||||
}
|
||||
if (typeof current !== "object" || current === null || Array.isArray(current)) {
|
||||
return failOrUndefined({
|
||||
onMissing,
|
||||
message: `JSON pointer segment "${token}" does not exist.`,
|
||||
});
|
||||
}
|
||||
const record = current as Record<string, unknown>;
|
||||
if (!Object.hasOwn(record, token)) {
|
||||
return failOrUndefined({
|
||||
onMissing,
|
||||
message: `JSON pointer segment "${token}" does not exist.`,
|
||||
});
|
||||
}
|
||||
current = record[token];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
export function setJsonPointer(
|
||||
root: Record<string, unknown>,
|
||||
pointer: string,
|
||||
value: unknown,
|
||||
): void {
|
||||
if (!pointer.startsWith("/")) {
|
||||
throw new Error(`Invalid JSON pointer "${pointer}".`);
|
||||
}
|
||||
|
||||
const tokens = pointer
|
||||
.slice(1)
|
||||
.split("/")
|
||||
.map((token) => decodeJsonPointerToken(token));
|
||||
|
||||
let current: Record<string, unknown> = root;
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
const token = tokens[index];
|
||||
const isLast = index === tokens.length - 1;
|
||||
if (isLast) {
|
||||
current[token] = value;
|
||||
return;
|
||||
}
|
||||
const child = current[token];
|
||||
if (typeof child !== "object" || child === null || Array.isArray(child)) {
|
||||
current[token] = {};
|
||||
}
|
||||
current = current[token] as Record<string, unknown>;
|
||||
}
|
||||
}
|
||||
204
src/secrets/migrate.test.ts
Normal file
204
src/secrets/migrate.test.ts
Normal file
@@ -0,0 +1,204 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const runExecMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../process/exec.js", () => ({
|
||||
runExec: runExecMock,
|
||||
}));
|
||||
|
||||
const { rollbackSecretsMigration, runSecretsMigration } = await import("./migrate.js");
|
||||
|
||||
describe("secrets migrate", () => {
|
||||
let baseDir = "";
|
||||
let stateDir = "";
|
||||
let configPath = "";
|
||||
let env: NodeJS.ProcessEnv;
|
||||
let authStorePath = "";
|
||||
let envPath = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
baseDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-secrets-migrate-"));
|
||||
stateDir = path.join(baseDir, ".openclaw");
|
||||
configPath = path.join(stateDir, "openclaw.json");
|
||||
authStorePath = path.join(stateDir, "agents", "main", "agent", "auth-profiles.json");
|
||||
envPath = path.join(stateDir, ".env");
|
||||
env = {
|
||||
...process.env,
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
OPENCLAW_CONFIG_PATH: configPath,
|
||||
};
|
||||
|
||||
await fs.mkdir(path.dirname(configPath), { recursive: true });
|
||||
await fs.mkdir(path.dirname(authStorePath), { recursive: true });
|
||||
|
||||
await fs.writeFile(
|
||||
configPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: "sk-openai-plaintext",
|
||||
models: [{ id: "gpt-5", name: "gpt-5" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
skills: {
|
||||
entries: {
|
||||
"review-pr": {
|
||||
enabled: true,
|
||||
apiKey: "sk-skill-plaintext",
|
||||
},
|
||||
},
|
||||
},
|
||||
channels: {
|
||||
googlechat: {
|
||||
serviceAccount: '{"type":"service_account","client_email":"bot@example.com"}',
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await fs.writeFile(
|
||||
authStorePath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
key: "sk-profile-plaintext",
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await fs.writeFile(
|
||||
envPath,
|
||||
"OPENAI_API_KEY=sk-openai-plaintext\nSKILL_KEY=sk-skill-plaintext\nUNRELATED=value\n",
|
||||
"utf8",
|
||||
);
|
||||
|
||||
runExecMock.mockReset();
|
||||
runExecMock.mockImplementation(async (_cmd: string, args: string[]) => {
|
||||
if (args[0] === "--encrypt") {
|
||||
const outputPath = args[args.indexOf("--output") + 1];
|
||||
const inputPath = args.at(-1);
|
||||
if (!outputPath || !inputPath) {
|
||||
throw new Error("missing sops encrypt paths");
|
||||
}
|
||||
await fs.copyFile(inputPath, outputPath);
|
||||
return { stdout: "", stderr: "" };
|
||||
}
|
||||
if (args[0] === "--decrypt") {
|
||||
const sourcePath = args.at(-1);
|
||||
if (!sourcePath) {
|
||||
throw new Error("missing sops decrypt source");
|
||||
}
|
||||
const raw = await fs.readFile(sourcePath, "utf8");
|
||||
return { stdout: raw, stderr: "" };
|
||||
}
|
||||
throw new Error(`unexpected sops invocation: ${args.join(" ")}`);
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(baseDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("reports a dry-run without mutating files", async () => {
|
||||
const beforeConfig = await fs.readFile(configPath, "utf8");
|
||||
const beforeAuthStore = await fs.readFile(authStorePath, "utf8");
|
||||
|
||||
const result = await runSecretsMigration({ env });
|
||||
|
||||
expect(result.mode).toBe("dry-run");
|
||||
expect(result.changed).toBe(true);
|
||||
expect(result.counters.secretsWritten).toBeGreaterThanOrEqual(3);
|
||||
|
||||
expect(await fs.readFile(configPath, "utf8")).toBe(beforeConfig);
|
||||
expect(await fs.readFile(authStorePath, "utf8")).toBe(beforeAuthStore);
|
||||
});
|
||||
|
||||
it("migrates plaintext to file-backed refs and can rollback", async () => {
|
||||
const applyResult = await runSecretsMigration({ env, write: true });
|
||||
|
||||
expect(applyResult.mode).toBe("write");
|
||||
expect(applyResult.changed).toBe(true);
|
||||
expect(applyResult.backupId).toBeTruthy();
|
||||
|
||||
const migratedConfig = JSON.parse(await fs.readFile(configPath, "utf8")) as {
|
||||
models: { providers: { openai: { apiKey: unknown } } };
|
||||
skills: { entries: { "review-pr": { apiKey: unknown } } };
|
||||
channels: { googlechat: { serviceAccount?: unknown; serviceAccountRef?: unknown } };
|
||||
secrets: { sources: { file: { type: string; path: string } } };
|
||||
};
|
||||
expect(migratedConfig.models.providers.openai.apiKey).toEqual({
|
||||
source: "file",
|
||||
id: "/providers/openai/apiKey",
|
||||
});
|
||||
expect(migratedConfig.skills.entries["review-pr"].apiKey).toEqual({
|
||||
source: "file",
|
||||
id: "/skills/entries/review-pr/apiKey",
|
||||
});
|
||||
expect(migratedConfig.channels.googlechat.serviceAccount).toBeUndefined();
|
||||
expect(migratedConfig.channels.googlechat.serviceAccountRef).toEqual({
|
||||
source: "file",
|
||||
id: "/channels/googlechat/serviceAccount",
|
||||
});
|
||||
expect(migratedConfig.secrets.sources.file.type).toBe("sops");
|
||||
|
||||
const migratedAuth = JSON.parse(await fs.readFile(authStorePath, "utf8")) as {
|
||||
profiles: { "openai:default": { key?: string; keyRef?: unknown } };
|
||||
};
|
||||
expect(migratedAuth.profiles["openai:default"].key).toBeUndefined();
|
||||
expect(migratedAuth.profiles["openai:default"].keyRef).toEqual({
|
||||
source: "file",
|
||||
id: "/auth-profiles/main/openai:default/key",
|
||||
});
|
||||
|
||||
const migratedEnv = await fs.readFile(envPath, "utf8");
|
||||
expect(migratedEnv).not.toContain("sk-openai-plaintext");
|
||||
expect(migratedEnv).toContain("SKILL_KEY=sk-skill-plaintext");
|
||||
expect(migratedEnv).toContain("UNRELATED=value");
|
||||
|
||||
const secretsPath = path.join(stateDir, "secrets.enc.json");
|
||||
const secretsPayload = JSON.parse(await fs.readFile(secretsPath, "utf8")) as {
|
||||
providers: { openai: { apiKey: string } };
|
||||
skills: { entries: { "review-pr": { apiKey: string } } };
|
||||
channels: { googlechat: { serviceAccount: string } };
|
||||
"auth-profiles": { main: { "openai:default": { key: string } } };
|
||||
};
|
||||
expect(secretsPayload.providers.openai.apiKey).toBe("sk-openai-plaintext");
|
||||
expect(secretsPayload.skills.entries["review-pr"].apiKey).toBe("sk-skill-plaintext");
|
||||
expect(secretsPayload.channels.googlechat.serviceAccount).toContain("service_account");
|
||||
expect(secretsPayload["auth-profiles"].main["openai:default"].key).toBe("sk-profile-plaintext");
|
||||
|
||||
const rollbackResult = await rollbackSecretsMigration({ env, backupId: applyResult.backupId! });
|
||||
expect(rollbackResult.restoredFiles).toBeGreaterThan(0);
|
||||
|
||||
const rolledBackConfig = await fs.readFile(configPath, "utf8");
|
||||
expect(rolledBackConfig).toContain("sk-openai-plaintext");
|
||||
expect(rolledBackConfig).toContain("sk-skill-plaintext");
|
||||
|
||||
const rolledBackAuth = await fs.readFile(authStorePath, "utf8");
|
||||
expect(rolledBackAuth).toContain("sk-profile-plaintext");
|
||||
|
||||
await expect(fs.stat(secretsPath)).rejects.toThrow();
|
||||
const rolledBackEnv = await fs.readFile(envPath, "utf8");
|
||||
expect(rolledBackEnv).toContain("OPENAI_API_KEY=sk-openai-plaintext");
|
||||
});
|
||||
});
|
||||
886
src/secrets/migrate.ts
Normal file
886
src/secrets/migrate.ts
Normal file
@@ -0,0 +1,886 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js";
|
||||
import { resolveAuthStorePath } from "../agents/auth-profiles/paths.js";
|
||||
import { createConfigIO, resolveStateDir, type OpenClawConfig } from "../config/config.js";
|
||||
import { isSecretRef } from "../config/types.secrets.js";
|
||||
import { resolveConfigDir, resolveUserPath } from "../utils.js";
|
||||
import {
|
||||
encodeJsonPointerToken,
|
||||
readJsonPointer as readJsonPointerRaw,
|
||||
setJsonPointer,
|
||||
} from "./json-pointer.js";
|
||||
import { listKnownSecretEnvVarNames } from "./provider-env-vars.js";
|
||||
import { decryptSopsJsonFile, encryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
|
||||
|
||||
const DEFAULT_SECRETS_FILE_PATH = "~/.openclaw/secrets.enc.json";
|
||||
const BACKUP_DIRNAME = "secrets-migrate";
|
||||
const BACKUP_MANIFEST_FILENAME = "manifest.json";
|
||||
const BACKUP_RETENTION = 20;
|
||||
|
||||
type MigrationCounters = {
|
||||
configRefs: number;
|
||||
authProfileRefs: number;
|
||||
plaintextRemoved: number;
|
||||
secretsWritten: number;
|
||||
envEntriesRemoved: number;
|
||||
authStoresChanged: number;
|
||||
};
|
||||
|
||||
type AuthStoreChange = {
|
||||
path: string;
|
||||
nextStore: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type EnvChange = {
|
||||
path: string;
|
||||
nextRaw: string;
|
||||
};
|
||||
|
||||
type BackupManifestEntry = {
|
||||
path: string;
|
||||
existed: boolean;
|
||||
backupPath?: string;
|
||||
mode?: number;
|
||||
};
|
||||
|
||||
type BackupManifest = {
|
||||
version: 1;
|
||||
backupId: string;
|
||||
createdAt: string;
|
||||
entries: BackupManifestEntry[];
|
||||
};
|
||||
|
||||
type MigrationPlan = {
|
||||
changed: boolean;
|
||||
counters: MigrationCounters;
|
||||
stateDir: string;
|
||||
configChanged: boolean;
|
||||
nextConfig: OpenClawConfig;
|
||||
configWriteOptions: Awaited<
|
||||
ReturnType<ReturnType<typeof createConfigIO>["readConfigFileSnapshotForWrite"]>
|
||||
>["writeOptions"];
|
||||
authStoreChanges: AuthStoreChange[];
|
||||
payloadChanged: boolean;
|
||||
nextPayload: Record<string, unknown>;
|
||||
secretsFilePath: string;
|
||||
secretsFileTimeoutMs: number;
|
||||
envChange: EnvChange | null;
|
||||
backupTargets: string[];
|
||||
};
|
||||
|
||||
export type SecretsMigrationRunOptions = {
|
||||
write?: boolean;
|
||||
scrubEnv?: boolean;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
now?: Date;
|
||||
};
|
||||
|
||||
export type SecretsMigrationRunResult = {
|
||||
mode: "dry-run" | "write";
|
||||
changed: boolean;
|
||||
backupId?: string;
|
||||
backupDir?: string;
|
||||
secretsFilePath: string;
|
||||
counters: MigrationCounters;
|
||||
changedFiles: string[];
|
||||
};
|
||||
|
||||
export type SecretsMigrationRollbackOptions = {
|
||||
backupId: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
};
|
||||
|
||||
export type SecretsMigrationRollbackResult = {
|
||||
backupId: string;
|
||||
restoredFiles: number;
|
||||
deletedFiles: number;
|
||||
};
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isNonEmptyString(value: unknown): value is string {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function normalizeSopsTimeoutMs(value: unknown): number {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return Math.max(1, Math.floor(value));
|
||||
}
|
||||
return DEFAULT_SOPS_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
function readJsonPointer(root: unknown, pointer: string): unknown {
|
||||
return readJsonPointerRaw(root, pointer, { onMissing: "undefined" });
|
||||
}
|
||||
|
||||
function formatBackupId(now: Date): string {
|
||||
const year = now.getUTCFullYear();
|
||||
const month = String(now.getUTCMonth() + 1).padStart(2, "0");
|
||||
const day = String(now.getUTCDate()).padStart(2, "0");
|
||||
const hour = String(now.getUTCHours()).padStart(2, "0");
|
||||
const minute = String(now.getUTCMinutes()).padStart(2, "0");
|
||||
const second = String(now.getUTCSeconds()).padStart(2, "0");
|
||||
return `${year}${month}${day}T${hour}${minute}${second}Z`;
|
||||
}
|
||||
|
||||
function parseEnvValue(raw: string): string {
|
||||
const trimmed = raw.trim();
|
||||
if (
|
||||
(trimmed.startsWith('"') && trimmed.endsWith('"')) ||
|
||||
(trimmed.startsWith("'") && trimmed.endsWith("'"))
|
||||
) {
|
||||
return trimmed.slice(1, -1);
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function scrubEnvRaw(
|
||||
raw: string,
|
||||
migratedValues: Set<string>,
|
||||
allowedEnvKeys: Set<string>,
|
||||
): {
|
||||
nextRaw: string;
|
||||
removed: number;
|
||||
} {
|
||||
if (migratedValues.size === 0 || allowedEnvKeys.size === 0) {
|
||||
return { nextRaw: raw, removed: 0 };
|
||||
}
|
||||
const lines = raw.split(/\r?\n/);
|
||||
const nextLines: string[] = [];
|
||||
let removed = 0;
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
|
||||
if (!match) {
|
||||
nextLines.push(line);
|
||||
continue;
|
||||
}
|
||||
const envKey = match[1] ?? "";
|
||||
if (!allowedEnvKeys.has(envKey)) {
|
||||
nextLines.push(line);
|
||||
continue;
|
||||
}
|
||||
const parsedValue = parseEnvValue(match[2] ?? "");
|
||||
if (migratedValues.has(parsedValue)) {
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
nextLines.push(line);
|
||||
}
|
||||
const hadTrailingNewline = raw.endsWith("\n");
|
||||
const joined = nextLines.join("\n");
|
||||
return {
|
||||
nextRaw:
|
||||
hadTrailingNewline || joined.length === 0
|
||||
? `${joined}${joined.endsWith("\n") ? "" : "\n"}`
|
||||
: joined,
|
||||
removed,
|
||||
};
|
||||
}
|
||||
|
||||
function ensureDirForFile(filePath: string): void {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
||||
}
|
||||
|
||||
function saveJsonFile(pathname: string, value: unknown): void {
|
||||
ensureDirForFile(pathname);
|
||||
fs.writeFileSync(pathname, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
fs.chmodSync(pathname, 0o600);
|
||||
}
|
||||
|
||||
function resolveFileSource(
|
||||
config: OpenClawConfig,
|
||||
env: NodeJS.ProcessEnv,
|
||||
): {
|
||||
path: string;
|
||||
timeoutMs: number;
|
||||
hadConfiguredSource: boolean;
|
||||
} {
|
||||
const source = config.secrets?.sources?.file;
|
||||
if (source && source.type === "sops" && isNonEmptyString(source.path)) {
|
||||
return {
|
||||
path: resolveUserPath(source.path),
|
||||
timeoutMs: normalizeSopsTimeoutMs(source.timeoutMs),
|
||||
hadConfiguredSource: true,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
path: resolveUserPath(resolveDefaultSecretsConfigPath(env)),
|
||||
timeoutMs: DEFAULT_SOPS_TIMEOUT_MS,
|
||||
hadConfiguredSource: false,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveDefaultSecretsConfigPath(env: NodeJS.ProcessEnv): string {
|
||||
if (env.OPENCLAW_STATE_DIR?.trim() || env.CLAWDBOT_STATE_DIR?.trim()) {
|
||||
return path.join(resolveStateDir(env, os.homedir), "secrets.enc.json");
|
||||
}
|
||||
return DEFAULT_SECRETS_FILE_PATH;
|
||||
}
|
||||
|
||||
async function decryptSopsJson(
|
||||
pathname: string,
|
||||
timeoutMs: number,
|
||||
): Promise<Record<string, unknown>> {
|
||||
if (!fs.existsSync(pathname)) {
|
||||
return {};
|
||||
}
|
||||
const parsed = await decryptSopsJsonFile({
|
||||
path: pathname,
|
||||
timeoutMs,
|
||||
missingBinaryMessage:
|
||||
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
|
||||
});
|
||||
if (!isRecord(parsed)) {
|
||||
throw new Error("sops decrypt failed: decrypted payload is not a JSON object");
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
async function encryptSopsJson(params: {
|
||||
pathname: string;
|
||||
timeoutMs: number;
|
||||
payload: Record<string, unknown>;
|
||||
}): Promise<void> {
|
||||
await encryptSopsJsonFile({
|
||||
path: params.pathname,
|
||||
payload: params.payload,
|
||||
timeoutMs: params.timeoutMs,
|
||||
missingBinaryMessage:
|
||||
"sops binary not found in PATH. Install sops >= 3.9.0 to run secrets migrate.",
|
||||
});
|
||||
}
|
||||
|
||||
function migrateModelProviderSecrets(params: {
|
||||
config: OpenClawConfig;
|
||||
payload: Record<string, unknown>;
|
||||
counters: MigrationCounters;
|
||||
migratedValues: Set<string>;
|
||||
}): void {
|
||||
const providers = params.config.models?.providers as
|
||||
| Record<string, { apiKey?: unknown }>
|
||||
| undefined;
|
||||
if (!providers) {
|
||||
return;
|
||||
}
|
||||
for (const [providerId, provider] of Object.entries(providers)) {
|
||||
if (isSecretRef(provider.apiKey)) {
|
||||
continue;
|
||||
}
|
||||
if (!isNonEmptyString(provider.apiKey)) {
|
||||
continue;
|
||||
}
|
||||
const value = provider.apiKey.trim();
|
||||
const id = `/providers/${encodeJsonPointerToken(providerId)}/apiKey`;
|
||||
const existing = readJsonPointer(params.payload, id);
|
||||
if (!isDeepStrictEqual(existing, value)) {
|
||||
setJsonPointer(params.payload, id, value);
|
||||
params.counters.secretsWritten += 1;
|
||||
}
|
||||
provider.apiKey = { source: "file", id };
|
||||
params.counters.configRefs += 1;
|
||||
params.migratedValues.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
function migrateSkillEntrySecrets(params: {
|
||||
config: OpenClawConfig;
|
||||
payload: Record<string, unknown>;
|
||||
counters: MigrationCounters;
|
||||
migratedValues: Set<string>;
|
||||
}): void {
|
||||
const entries = params.config.skills?.entries as Record<string, { apiKey?: unknown }> | undefined;
|
||||
if (!entries) {
|
||||
return;
|
||||
}
|
||||
for (const [skillKey, entry] of Object.entries(entries)) {
|
||||
if (!isRecord(entry) || isSecretRef(entry.apiKey)) {
|
||||
continue;
|
||||
}
|
||||
if (!isNonEmptyString(entry.apiKey)) {
|
||||
continue;
|
||||
}
|
||||
const value = entry.apiKey.trim();
|
||||
const id = `/skills/entries/${encodeJsonPointerToken(skillKey)}/apiKey`;
|
||||
const existing = readJsonPointer(params.payload, id);
|
||||
if (!isDeepStrictEqual(existing, value)) {
|
||||
setJsonPointer(params.payload, id, value);
|
||||
params.counters.secretsWritten += 1;
|
||||
}
|
||||
entry.apiKey = { source: "file", id };
|
||||
params.counters.configRefs += 1;
|
||||
params.migratedValues.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
function migrateGoogleChatServiceAccount(params: {
|
||||
account: Record<string, unknown>;
|
||||
pointerId: string;
|
||||
counters: MigrationCounters;
|
||||
payload: Record<string, unknown>;
|
||||
}): void {
|
||||
const explicitRef = isSecretRef(params.account.serviceAccountRef)
|
||||
? params.account.serviceAccountRef
|
||||
: null;
|
||||
const inlineRef = isSecretRef(params.account.serviceAccount)
|
||||
? params.account.serviceAccount
|
||||
: null;
|
||||
if (explicitRef || inlineRef) {
|
||||
if (
|
||||
params.account.serviceAccount !== undefined &&
|
||||
!isSecretRef(params.account.serviceAccount)
|
||||
) {
|
||||
delete params.account.serviceAccount;
|
||||
params.counters.plaintextRemoved += 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const value = params.account.serviceAccount;
|
||||
const hasStringValue = isNonEmptyString(value);
|
||||
const hasObjectValue = isRecord(value) && Object.keys(value).length > 0;
|
||||
if (!hasStringValue && !hasObjectValue) {
|
||||
return;
|
||||
}
|
||||
|
||||
const id = `${params.pointerId}/serviceAccount`;
|
||||
const normalizedValue = hasStringValue ? value.trim() : structuredClone(value);
|
||||
const existing = readJsonPointer(params.payload, id);
|
||||
if (!isDeepStrictEqual(existing, normalizedValue)) {
|
||||
setJsonPointer(params.payload, id, normalizedValue);
|
||||
params.counters.secretsWritten += 1;
|
||||
}
|
||||
|
||||
params.account.serviceAccountRef = { source: "file", id };
|
||||
delete params.account.serviceAccount;
|
||||
params.counters.configRefs += 1;
|
||||
}
|
||||
|
||||
function migrateGoogleChatSecrets(params: {
|
||||
config: OpenClawConfig;
|
||||
payload: Record<string, unknown>;
|
||||
counters: MigrationCounters;
|
||||
}): void {
|
||||
const googlechat = params.config.channels?.googlechat;
|
||||
if (!isRecord(googlechat)) {
|
||||
return;
|
||||
}
|
||||
|
||||
migrateGoogleChatServiceAccount({
|
||||
account: googlechat,
|
||||
pointerId: "/channels/googlechat",
|
||||
payload: params.payload,
|
||||
counters: params.counters,
|
||||
});
|
||||
|
||||
if (!isRecord(googlechat.accounts)) {
|
||||
return;
|
||||
}
|
||||
for (const [accountId, accountValue] of Object.entries(googlechat.accounts)) {
|
||||
if (!isRecord(accountValue)) {
|
||||
continue;
|
||||
}
|
||||
migrateGoogleChatServiceAccount({
|
||||
account: accountValue,
|
||||
pointerId: `/channels/googlechat/accounts/${encodeJsonPointerToken(accountId)}`,
|
||||
payload: params.payload,
|
||||
counters: params.counters,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function collectAuthStorePaths(config: OpenClawConfig, stateDir: string): string[] {
|
||||
const paths = new Set<string>();
|
||||
paths.add(resolveUserPath(resolveAuthStorePath()));
|
||||
|
||||
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
|
||||
if (fs.existsSync(agentsRoot)) {
|
||||
for (const entry of fs.readdirSync(agentsRoot, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
paths.add(path.join(agentsRoot, entry.name, "agent", "auth-profiles.json"));
|
||||
}
|
||||
}
|
||||
|
||||
for (const agentId of listAgentIds(config)) {
|
||||
const agentDir = resolveAgentDir(config, agentId);
|
||||
paths.add(resolveUserPath(resolveAuthStorePath(agentDir)));
|
||||
}
|
||||
|
||||
return [...paths];
|
||||
}
|
||||
|
||||
function deriveAuthStoreScope(authStorePath: string, stateDir: string): string {
|
||||
const agentsRoot = path.join(resolveUserPath(stateDir), "agents");
|
||||
const relative = path.relative(agentsRoot, authStorePath);
|
||||
if (!relative.startsWith("..")) {
|
||||
const segments = relative.split(path.sep);
|
||||
if (segments.length >= 3 && segments[1] === "agent" && segments[2] === "auth-profiles.json") {
|
||||
const candidate = segments[0]?.trim();
|
||||
if (candidate) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const digest = crypto.createHash("sha1").update(authStorePath).digest("hex").slice(0, 8);
|
||||
return `path-${digest}`;
|
||||
}
|
||||
|
||||
function migrateAuthStoreSecrets(params: {
|
||||
store: Record<string, unknown>;
|
||||
scope: string;
|
||||
payload: Record<string, unknown>;
|
||||
counters: MigrationCounters;
|
||||
migratedValues: Set<string>;
|
||||
}): boolean {
|
||||
const profiles = params.store.profiles;
|
||||
if (!isRecord(profiles)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let changed = false;
|
||||
for (const [profileId, profileValue] of Object.entries(profiles)) {
|
||||
if (!isRecord(profileValue)) {
|
||||
continue;
|
||||
}
|
||||
if (profileValue.type === "api_key") {
|
||||
const keyRef = isSecretRef(profileValue.keyRef) ? profileValue.keyRef : null;
|
||||
const key = isNonEmptyString(profileValue.key) ? profileValue.key.trim() : "";
|
||||
if (keyRef) {
|
||||
if (key) {
|
||||
delete profileValue.key;
|
||||
params.counters.plaintextRemoved += 1;
|
||||
changed = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!key) {
|
||||
continue;
|
||||
}
|
||||
const id = `/auth-profiles/${encodeJsonPointerToken(params.scope)}/${encodeJsonPointerToken(profileId)}/key`;
|
||||
const existing = readJsonPointer(params.payload, id);
|
||||
if (!isDeepStrictEqual(existing, key)) {
|
||||
setJsonPointer(params.payload, id, key);
|
||||
params.counters.secretsWritten += 1;
|
||||
}
|
||||
profileValue.keyRef = { source: "file", id };
|
||||
delete profileValue.key;
|
||||
params.counters.authProfileRefs += 1;
|
||||
params.migratedValues.add(key);
|
||||
changed = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (profileValue.type === "token") {
|
||||
const tokenRef = isSecretRef(profileValue.tokenRef) ? profileValue.tokenRef : null;
|
||||
const token = isNonEmptyString(profileValue.token) ? profileValue.token.trim() : "";
|
||||
if (tokenRef) {
|
||||
if (token) {
|
||||
delete profileValue.token;
|
||||
params.counters.plaintextRemoved += 1;
|
||||
changed = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!token) {
|
||||
continue;
|
||||
}
|
||||
const id = `/auth-profiles/${encodeJsonPointerToken(params.scope)}/${encodeJsonPointerToken(profileId)}/token`;
|
||||
const existing = readJsonPointer(params.payload, id);
|
||||
if (!isDeepStrictEqual(existing, token)) {
|
||||
setJsonPointer(params.payload, id, token);
|
||||
params.counters.secretsWritten += 1;
|
||||
}
|
||||
profileValue.tokenRef = { source: "file", id };
|
||||
delete profileValue.token;
|
||||
params.counters.authProfileRefs += 1;
|
||||
params.migratedValues.add(token);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
return changed;
|
||||
}
|
||||
|
||||
function resolveBackupRoot(stateDir: string): string {
|
||||
return path.join(resolveUserPath(stateDir), "backups", BACKUP_DIRNAME);
|
||||
}
|
||||
|
||||
function createBackupManifest(params: {
|
||||
stateDir: string;
|
||||
targets: string[];
|
||||
backupId: string;
|
||||
now: Date;
|
||||
}): { backupDir: string; manifestPath: string; manifest: BackupManifest } {
|
||||
const backupDir = path.join(resolveBackupRoot(params.stateDir), params.backupId);
|
||||
fs.mkdirSync(backupDir, { recursive: true, mode: 0o700 });
|
||||
|
||||
const entries: BackupManifestEntry[] = [];
|
||||
let index = 0;
|
||||
for (const target of params.targets) {
|
||||
const normalized = resolveUserPath(target);
|
||||
const exists = fs.existsSync(normalized);
|
||||
if (!exists) {
|
||||
entries.push({ path: normalized, existed: false });
|
||||
continue;
|
||||
}
|
||||
|
||||
const backupName = `file-${String(index).padStart(4, "0")}.bak`;
|
||||
const backupPath = path.join(backupDir, backupName);
|
||||
fs.copyFileSync(normalized, backupPath);
|
||||
const stats = fs.statSync(normalized);
|
||||
entries.push({
|
||||
path: normalized,
|
||||
existed: true,
|
||||
backupPath,
|
||||
mode: stats.mode & 0o777,
|
||||
});
|
||||
index += 1;
|
||||
}
|
||||
|
||||
const manifest: BackupManifest = {
|
||||
version: 1,
|
||||
backupId: params.backupId,
|
||||
createdAt: params.now.toISOString(),
|
||||
entries,
|
||||
};
|
||||
const manifestPath = path.join(backupDir, BACKUP_MANIFEST_FILENAME);
|
||||
fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
||||
fs.chmodSync(manifestPath, 0o600);
|
||||
|
||||
return { backupDir, manifestPath, manifest };
|
||||
}
|
||||
|
||||
function restoreFromManifest(manifest: BackupManifest): {
|
||||
restoredFiles: number;
|
||||
deletedFiles: number;
|
||||
} {
|
||||
let restoredFiles = 0;
|
||||
let deletedFiles = 0;
|
||||
|
||||
for (const entry of manifest.entries) {
|
||||
if (!entry.existed) {
|
||||
if (fs.existsSync(entry.path)) {
|
||||
fs.rmSync(entry.path, { force: true });
|
||||
deletedFiles += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.backupPath || !fs.existsSync(entry.backupPath)) {
|
||||
throw new Error(`Backup file is missing for ${entry.path}.`);
|
||||
}
|
||||
ensureDirForFile(entry.path);
|
||||
fs.copyFileSync(entry.backupPath, entry.path);
|
||||
fs.chmodSync(entry.path, entry.mode ?? 0o600);
|
||||
restoredFiles += 1;
|
||||
}
|
||||
|
||||
return { restoredFiles, deletedFiles };
|
||||
}
|
||||
|
||||
function pruneOldBackups(stateDir: string): void {
|
||||
const backupRoot = resolveBackupRoot(stateDir);
|
||||
if (!fs.existsSync(backupRoot)) {
|
||||
return;
|
||||
}
|
||||
const dirs = fs
|
||||
.readdirSync(backupRoot, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.toSorted();
|
||||
|
||||
if (dirs.length <= BACKUP_RETENTION) {
|
||||
return;
|
||||
}
|
||||
|
||||
const toDelete = dirs.slice(0, Math.max(0, dirs.length - BACKUP_RETENTION));
|
||||
for (const dir of toDelete) {
|
||||
fs.rmSync(path.join(backupRoot, dir), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function buildMigrationPlan(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
scrubEnv: boolean;
|
||||
}): Promise<MigrationPlan> {
|
||||
const io = createConfigIO({ env: params.env });
|
||||
const { snapshot, writeOptions } = await io.readConfigFileSnapshotForWrite();
|
||||
if (!snapshot.valid) {
|
||||
const issues =
|
||||
snapshot.issues.length > 0
|
||||
? snapshot.issues.map((issue) => `${issue.path || "<root>"}: ${issue.message}`).join("\n")
|
||||
: "Unknown validation issue.";
|
||||
throw new Error(`Cannot migrate secrets because config is invalid:\n${issues}`);
|
||||
}
|
||||
|
||||
const stateDir = resolveStateDir(params.env, os.homedir);
|
||||
const nextConfig = structuredClone(snapshot.config);
|
||||
const fileSource = resolveFileSource(nextConfig, params.env);
|
||||
const previousPayload = await decryptSopsJson(fileSource.path, fileSource.timeoutMs);
|
||||
const nextPayload = structuredClone(previousPayload);
|
||||
|
||||
const counters: MigrationCounters = {
|
||||
configRefs: 0,
|
||||
authProfileRefs: 0,
|
||||
plaintextRemoved: 0,
|
||||
secretsWritten: 0,
|
||||
envEntriesRemoved: 0,
|
||||
authStoresChanged: 0,
|
||||
};
|
||||
|
||||
const migratedValues = new Set<string>();
|
||||
|
||||
migrateModelProviderSecrets({
|
||||
config: nextConfig,
|
||||
payload: nextPayload,
|
||||
counters,
|
||||
migratedValues,
|
||||
});
|
||||
migrateSkillEntrySecrets({
|
||||
config: nextConfig,
|
||||
payload: nextPayload,
|
||||
counters,
|
||||
migratedValues,
|
||||
});
|
||||
migrateGoogleChatSecrets({
|
||||
config: nextConfig,
|
||||
payload: nextPayload,
|
||||
counters,
|
||||
});
|
||||
|
||||
const authStoreChanges: AuthStoreChange[] = [];
|
||||
for (const authStorePath of collectAuthStorePaths(nextConfig, stateDir)) {
|
||||
if (!fs.existsSync(authStorePath)) {
|
||||
continue;
|
||||
}
|
||||
const raw = fs.readFileSync(authStorePath, "utf8");
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(raw) as unknown;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!isRecord(parsed)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextStore = structuredClone(parsed);
|
||||
const scope = deriveAuthStoreScope(authStorePath, stateDir);
|
||||
const changed = migrateAuthStoreSecrets({
|
||||
store: nextStore,
|
||||
scope,
|
||||
payload: nextPayload,
|
||||
counters,
|
||||
migratedValues,
|
||||
});
|
||||
if (!changed) {
|
||||
continue;
|
||||
}
|
||||
authStoreChanges.push({ path: authStorePath, nextStore });
|
||||
}
|
||||
counters.authStoresChanged = authStoreChanges.length;
|
||||
|
||||
if (counters.secretsWritten > 0 && !fileSource.hadConfiguredSource) {
|
||||
const defaultConfigPath = resolveDefaultSecretsConfigPath(params.env);
|
||||
nextConfig.secrets ??= {};
|
||||
nextConfig.secrets.sources ??= {};
|
||||
nextConfig.secrets.sources.file = {
|
||||
type: "sops",
|
||||
path: defaultConfigPath,
|
||||
timeoutMs: DEFAULT_SOPS_TIMEOUT_MS,
|
||||
};
|
||||
}
|
||||
|
||||
const configChanged = !isDeepStrictEqual(snapshot.config, nextConfig);
|
||||
const payloadChanged = !isDeepStrictEqual(previousPayload, nextPayload);
|
||||
|
||||
let envChange: EnvChange | null = null;
|
||||
if (params.scrubEnv && migratedValues.size > 0) {
|
||||
const envPath = path.join(resolveConfigDir(params.env, os.homedir), ".env");
|
||||
if (fs.existsSync(envPath)) {
|
||||
const rawEnv = fs.readFileSync(envPath, "utf8");
|
||||
const scrubbed = scrubEnvRaw(rawEnv, migratedValues, new Set(listKnownSecretEnvVarNames()));
|
||||
if (scrubbed.removed > 0 && scrubbed.nextRaw !== rawEnv) {
|
||||
counters.envEntriesRemoved = scrubbed.removed;
|
||||
envChange = {
|
||||
path: envPath,
|
||||
nextRaw: scrubbed.nextRaw,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const backupTargets = new Set<string>();
|
||||
if (configChanged) {
|
||||
backupTargets.add(io.configPath);
|
||||
}
|
||||
if (payloadChanged) {
|
||||
backupTargets.add(fileSource.path);
|
||||
}
|
||||
for (const change of authStoreChanges) {
|
||||
backupTargets.add(change.path);
|
||||
}
|
||||
if (envChange) {
|
||||
backupTargets.add(envChange.path);
|
||||
}
|
||||
|
||||
return {
|
||||
changed: configChanged || payloadChanged || authStoreChanges.length > 0 || Boolean(envChange),
|
||||
counters,
|
||||
stateDir,
|
||||
configChanged,
|
||||
nextConfig,
|
||||
configWriteOptions: writeOptions,
|
||||
authStoreChanges,
|
||||
payloadChanged,
|
||||
nextPayload,
|
||||
secretsFilePath: fileSource.path,
|
||||
secretsFileTimeoutMs: fileSource.timeoutMs,
|
||||
envChange,
|
||||
backupTargets: [...backupTargets],
|
||||
};
|
||||
}
|
||||
|
||||
export async function runSecretsMigration(
|
||||
options: SecretsMigrationRunOptions = {},
|
||||
): Promise<SecretsMigrationRunResult> {
|
||||
const env = options.env ?? process.env;
|
||||
const scrubEnv = options.scrubEnv ?? true;
|
||||
const plan = await buildMigrationPlan({ env, scrubEnv });
|
||||
|
||||
if (!options.write) {
|
||||
return {
|
||||
mode: "dry-run",
|
||||
changed: plan.changed,
|
||||
secretsFilePath: plan.secretsFilePath,
|
||||
counters: plan.counters,
|
||||
changedFiles: plan.backupTargets,
|
||||
};
|
||||
}
|
||||
|
||||
if (!plan.changed) {
|
||||
return {
|
||||
mode: "write",
|
||||
changed: false,
|
||||
secretsFilePath: plan.secretsFilePath,
|
||||
counters: plan.counters,
|
||||
changedFiles: [],
|
||||
};
|
||||
}
|
||||
|
||||
const now = options.now ?? new Date();
|
||||
const backupId = formatBackupId(now);
|
||||
const backup = createBackupManifest({
|
||||
stateDir: plan.stateDir,
|
||||
targets: plan.backupTargets,
|
||||
backupId,
|
||||
now,
|
||||
});
|
||||
|
||||
try {
|
||||
if (plan.payloadChanged) {
|
||||
await encryptSopsJson({
|
||||
pathname: plan.secretsFilePath,
|
||||
timeoutMs: plan.secretsFileTimeoutMs,
|
||||
payload: plan.nextPayload,
|
||||
});
|
||||
}
|
||||
|
||||
if (plan.configChanged) {
|
||||
const io = createConfigIO({ env });
|
||||
await io.writeConfigFile(plan.nextConfig, plan.configWriteOptions);
|
||||
}
|
||||
|
||||
for (const change of plan.authStoreChanges) {
|
||||
saveJsonFile(change.path, change.nextStore);
|
||||
}
|
||||
|
||||
if (plan.envChange) {
|
||||
ensureDirForFile(plan.envChange.path);
|
||||
fs.writeFileSync(plan.envChange.path, plan.envChange.nextRaw, "utf8");
|
||||
fs.chmodSync(plan.envChange.path, 0o600);
|
||||
}
|
||||
} catch (err) {
|
||||
restoreFromManifest(backup.manifest);
|
||||
throw new Error(
|
||||
`Secrets migration failed and was rolled back from backup ${backupId}: ${String(err)}`,
|
||||
{
|
||||
cause: err,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pruneOldBackups(plan.stateDir);
|
||||
|
||||
return {
|
||||
mode: "write",
|
||||
changed: true,
|
||||
backupId,
|
||||
backupDir: backup.backupDir,
|
||||
secretsFilePath: plan.secretsFilePath,
|
||||
counters: plan.counters,
|
||||
changedFiles: plan.backupTargets,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveSecretsMigrationBackupRoot(env: NodeJS.ProcessEnv = process.env): string {
|
||||
return resolveBackupRoot(resolveStateDir(env, os.homedir));
|
||||
}
|
||||
|
||||
export function listSecretsMigrationBackups(env: NodeJS.ProcessEnv = process.env): string[] {
|
||||
const root = resolveSecretsMigrationBackupRoot(env);
|
||||
if (!fs.existsSync(root)) {
|
||||
return [];
|
||||
}
|
||||
return fs
|
||||
.readdirSync(root, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.toSorted();
|
||||
}
|
||||
|
||||
export async function rollbackSecretsMigration(
|
||||
options: SecretsMigrationRollbackOptions,
|
||||
): Promise<SecretsMigrationRollbackResult> {
|
||||
const env = options.env ?? process.env;
|
||||
const backupDir = path.join(resolveSecretsMigrationBackupRoot(env), options.backupId);
|
||||
const manifestPath = path.join(backupDir, BACKUP_MANIFEST_FILENAME);
|
||||
if (!fs.existsSync(manifestPath)) {
|
||||
const available = listSecretsMigrationBackups(env);
|
||||
const suffix =
|
||||
available.length > 0
|
||||
? ` Available backups: ${available.slice(-10).join(", ")}`
|
||||
: " No backups were found.";
|
||||
throw new Error(`Backup "${options.backupId}" was not found.${suffix}`);
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as unknown;
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to read backup manifest at ${manifestPath}: ${String(err)}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
|
||||
if (!isRecord(parsed) || !Array.isArray(parsed.entries)) {
|
||||
throw new Error(`Backup manifest at ${manifestPath} is invalid.`);
|
||||
}
|
||||
|
||||
const manifest = parsed as BackupManifest;
|
||||
const restored = restoreFromManifest(manifest);
|
||||
return {
|
||||
backupId: options.backupId,
|
||||
restoredFiles: restored.restoredFiles,
|
||||
deletedFiles: restored.deletedFiles,
|
||||
};
|
||||
}
|
||||
28
src/secrets/provider-env-vars.ts
Normal file
28
src/secrets/provider-env-vars.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
export const PROVIDER_ENV_VARS: Record<string, readonly string[]> = {
|
||||
openai: ["OPENAI_API_KEY"],
|
||||
anthropic: ["ANTHROPIC_API_KEY"],
|
||||
google: ["GEMINI_API_KEY"],
|
||||
minimax: ["MINIMAX_API_KEY"],
|
||||
"minimax-cn": ["MINIMAX_API_KEY"],
|
||||
moonshot: ["MOONSHOT_API_KEY"],
|
||||
"kimi-coding": ["KIMI_API_KEY", "KIMICODE_API_KEY"],
|
||||
synthetic: ["SYNTHETIC_API_KEY"],
|
||||
venice: ["VENICE_API_KEY"],
|
||||
zai: ["ZAI_API_KEY", "Z_AI_API_KEY"],
|
||||
xiaomi: ["XIAOMI_API_KEY"],
|
||||
openrouter: ["OPENROUTER_API_KEY"],
|
||||
"cloudflare-ai-gateway": ["CLOUDFLARE_AI_GATEWAY_API_KEY"],
|
||||
litellm: ["LITELLM_API_KEY"],
|
||||
"vercel-ai-gateway": ["AI_GATEWAY_API_KEY"],
|
||||
opencode: ["OPENCODE_API_KEY", "OPENCODE_ZEN_API_KEY"],
|
||||
together: ["TOGETHER_API_KEY"],
|
||||
huggingface: ["HUGGINGFACE_HUB_TOKEN", "HF_TOKEN"],
|
||||
qianfan: ["QIANFAN_API_KEY"],
|
||||
xai: ["XAI_API_KEY"],
|
||||
volcengine: ["VOLCANO_ENGINE_API_KEY"],
|
||||
byteplus: ["BYTEPLUS_API_KEY"],
|
||||
};
|
||||
|
||||
export function listKnownSecretEnvVarNames(): string[] {
|
||||
return [...new Set(Object.values(PROVIDER_ENV_VARS).flatMap((keys) => keys))];
|
||||
}
|
||||
169
src/secrets/runtime.test.ts
Normal file
169
src/secrets/runtime.test.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ensureAuthProfileStore } from "../agents/auth-profiles.js";
|
||||
import { loadConfig, type OpenClawConfig } from "../config/config.js";
|
||||
import {
|
||||
activateSecretsRuntimeSnapshot,
|
||||
clearSecretsRuntimeSnapshot,
|
||||
prepareSecretsRuntimeSnapshot,
|
||||
} from "./runtime.js";
|
||||
|
||||
const runExecMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../process/exec.js", () => ({
|
||||
runExec: runExecMock,
|
||||
}));
|
||||
|
||||
describe("secrets runtime snapshot", () => {
|
||||
afterEach(() => {
|
||||
runExecMock.mockReset();
|
||||
clearSecretsRuntimeSnapshot();
|
||||
});
|
||||
|
||||
it("resolves env refs for config and auth profiles", async () => {
|
||||
const config: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", id: "OPENAI_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
skills: {
|
||||
entries: {
|
||||
"review-pr": {
|
||||
enabled: true,
|
||||
apiKey: { source: "env", id: "REVIEW_SKILL_API_KEY" },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config,
|
||||
env: {
|
||||
OPENAI_API_KEY: "sk-env-openai",
|
||||
GITHUB_TOKEN: "ghp-env-token",
|
||||
REVIEW_SKILL_API_KEY: "sk-skill-ref",
|
||||
},
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: () => ({
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
key: "old-openai",
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
},
|
||||
"github-copilot:default": {
|
||||
type: "token",
|
||||
provider: "github-copilot",
|
||||
token: "old-gh",
|
||||
tokenRef: { source: "env", id: "GITHUB_TOKEN" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-env-openai");
|
||||
expect(snapshot.config.skills?.entries?.["review-pr"]?.apiKey).toBe("sk-skill-ref");
|
||||
expect(snapshot.warnings).toHaveLength(2);
|
||||
expect(snapshot.authStores[0]?.store.profiles["openai:default"]).toMatchObject({
|
||||
type: "api_key",
|
||||
key: "sk-env-openai",
|
||||
});
|
||||
expect(snapshot.authStores[0]?.store.profiles["github-copilot:default"]).toMatchObject({
|
||||
type: "token",
|
||||
token: "ghp-env-token",
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves file refs via sops json payload", async () => {
|
||||
runExecMock.mockResolvedValueOnce({
|
||||
stdout: JSON.stringify({
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: "sk-from-sops",
|
||||
},
|
||||
},
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const config: OpenClawConfig = {
|
||||
secrets: {
|
||||
sources: {
|
||||
file: {
|
||||
type: "sops",
|
||||
path: "~/.openclaw/secrets.enc.json",
|
||||
timeoutMs: 7000,
|
||||
},
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "file", id: "/providers/openai/apiKey" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config,
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: () => ({ version: 1, profiles: {} }),
|
||||
});
|
||||
|
||||
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-from-sops");
|
||||
expect(runExecMock).toHaveBeenCalledWith(
|
||||
"sops",
|
||||
["--decrypt", "--output-type", "json", expect.stringContaining("secrets.enc.json")],
|
||||
{
|
||||
timeoutMs: 7000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("activates runtime snapshots for loadConfig and ensureAuthProfileStore", async () => {
|
||||
const prepared = await prepareSecretsRuntimeSnapshot({
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", id: "OPENAI_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
env: { OPENAI_API_KEY: "sk-runtime" },
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: () => ({
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
activateSecretsRuntimeSnapshot(prepared);
|
||||
|
||||
expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-runtime");
|
||||
const store = ensureAuthProfileStore("/tmp/openclaw-agent-main");
|
||||
expect(store.profiles["openai:default"]).toMatchObject({
|
||||
type: "api_key",
|
||||
key: "sk-runtime",
|
||||
});
|
||||
});
|
||||
});
|
||||
337
src/secrets/runtime.ts
Normal file
337
src/secrets/runtime.ts
Normal file
@@ -0,0 +1,337 @@
|
||||
import { resolveOpenClawAgentDir } from "../agents/agent-paths.js";
|
||||
import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js";
|
||||
import type { AuthProfileCredential, AuthProfileStore } from "../agents/auth-profiles.js";
|
||||
import {
|
||||
clearRuntimeAuthProfileStoreSnapshots,
|
||||
loadAuthProfileStoreForRuntime,
|
||||
replaceRuntimeAuthProfileStoreSnapshots,
|
||||
} from "../agents/auth-profiles.js";
|
||||
import {
|
||||
clearRuntimeConfigSnapshot,
|
||||
setRuntimeConfigSnapshot,
|
||||
type OpenClawConfig,
|
||||
} from "../config/config.js";
|
||||
import { isSecretRef, type SecretRef } from "../config/types.secrets.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
import { readJsonPointer } from "./json-pointer.js";
|
||||
import { decryptSopsJsonFile, DEFAULT_SOPS_TIMEOUT_MS } from "./sops.js";
|
||||
|
||||
type SecretResolverWarningCode = "SECRETS_REF_OVERRIDES_PLAINTEXT";
|
||||
|
||||
export type SecretResolverWarning = {
|
||||
code: SecretResolverWarningCode;
|
||||
path: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type PreparedSecretsRuntimeSnapshot = {
|
||||
sourceConfig: OpenClawConfig;
|
||||
config: OpenClawConfig;
|
||||
authStores: Array<{ agentDir: string; store: AuthProfileStore }>;
|
||||
warnings: SecretResolverWarning[];
|
||||
};
|
||||
|
||||
type ResolverContext = {
|
||||
config: OpenClawConfig;
|
||||
env: NodeJS.ProcessEnv;
|
||||
fileSecretsPromise: Promise<unknown> | null;
|
||||
};
|
||||
|
||||
type ProviderLike = {
|
||||
apiKey?: unknown;
|
||||
};
|
||||
|
||||
type SkillEntryLike = {
|
||||
apiKey?: unknown;
|
||||
};
|
||||
|
||||
type GoogleChatAccountLike = {
|
||||
serviceAccount?: unknown;
|
||||
serviceAccountRef?: unknown;
|
||||
accounts?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type ApiKeyCredentialLike = AuthProfileCredential & {
|
||||
type: "api_key";
|
||||
key?: string;
|
||||
keyRef?: unknown;
|
||||
};
|
||||
|
||||
type TokenCredentialLike = AuthProfileCredential & {
|
||||
type: "token";
|
||||
token?: string;
|
||||
tokenRef?: unknown;
|
||||
};
|
||||
|
||||
let activeSnapshot: PreparedSecretsRuntimeSnapshot | null = null;
|
||||
|
||||
function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecretsRuntimeSnapshot {
|
||||
return {
|
||||
sourceConfig: structuredClone(snapshot.sourceConfig),
|
||||
config: structuredClone(snapshot.config),
|
||||
authStores: snapshot.authStores.map((entry) => ({
|
||||
agentDir: entry.agentDir,
|
||||
store: structuredClone(entry.store),
|
||||
})),
|
||||
warnings: snapshot.warnings.map((warning) => ({ ...warning })),
|
||||
};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isNonEmptyString(value: unknown): value is string {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
async function decryptSopsFile(config: OpenClawConfig): Promise<unknown> {
|
||||
const fileSource = config.secrets?.sources?.file;
|
||||
if (!fileSource) {
|
||||
throw new Error(
|
||||
`Secret reference source "file" is not configured. Configure secrets.sources.file first.`,
|
||||
);
|
||||
}
|
||||
if (fileSource.type !== "sops") {
|
||||
throw new Error(`Unsupported secrets.sources.file.type "${String(fileSource.type)}".`);
|
||||
}
|
||||
|
||||
const resolvedPath = resolveUserPath(fileSource.path);
|
||||
const timeoutMs =
|
||||
typeof fileSource.timeoutMs === "number" && Number.isFinite(fileSource.timeoutMs)
|
||||
? Math.max(1, Math.floor(fileSource.timeoutMs))
|
||||
: DEFAULT_SOPS_TIMEOUT_MS;
|
||||
return await decryptSopsJsonFile({
|
||||
path: resolvedPath,
|
||||
timeoutMs,
|
||||
missingBinaryMessage:
|
||||
"sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.",
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext): Promise<unknown> {
|
||||
const id = ref.id.trim();
|
||||
if (!id) {
|
||||
throw new Error(`Secret reference id is empty.`);
|
||||
}
|
||||
|
||||
if (ref.source === "env") {
|
||||
const envValue = context.env[id];
|
||||
if (!isNonEmptyString(envValue)) {
|
||||
throw new Error(`Environment variable "${id}" is missing or empty.`);
|
||||
}
|
||||
return envValue;
|
||||
}
|
||||
|
||||
if (ref.source === "file") {
|
||||
context.fileSecretsPromise ??= decryptSopsFile(context.config);
|
||||
const payload = await context.fileSecretsPromise;
|
||||
return readJsonPointer(payload, id, { onMissing: "throw" });
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`);
|
||||
}
|
||||
|
||||
async function resolveGoogleChatServiceAccount(
|
||||
target: GoogleChatAccountLike,
|
||||
path: string,
|
||||
context: ResolverContext,
|
||||
warnings: SecretResolverWarning[],
|
||||
): Promise<void> {
|
||||
const explicitRef = isSecretRef(target.serviceAccountRef) ? target.serviceAccountRef : null;
|
||||
const inlineRef = isSecretRef(target.serviceAccount) ? target.serviceAccount : null;
|
||||
const ref = explicitRef ?? inlineRef;
|
||||
if (!ref) {
|
||||
return;
|
||||
}
|
||||
if (explicitRef && target.serviceAccount !== undefined && !isSecretRef(target.serviceAccount)) {
|
||||
warnings.push({
|
||||
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
|
||||
path,
|
||||
message: `${path}: serviceAccountRef is set; runtime will ignore plaintext serviceAccount.`,
|
||||
});
|
||||
}
|
||||
target.serviceAccount = await resolveSecretRefValue(ref, context);
|
||||
}
|
||||
|
||||
async function resolveConfigSecretRefs(params: {
|
||||
config: OpenClawConfig;
|
||||
context: ResolverContext;
|
||||
warnings: SecretResolverWarning[];
|
||||
}): Promise<OpenClawConfig> {
|
||||
const resolved = structuredClone(params.config);
|
||||
const providers = resolved.models?.providers as Record<string, ProviderLike> | undefined;
|
||||
if (providers) {
|
||||
for (const [providerId, provider] of Object.entries(providers)) {
|
||||
if (!isSecretRef(provider.apiKey)) {
|
||||
continue;
|
||||
}
|
||||
const resolvedValue = await resolveSecretRefValue(provider.apiKey, params.context);
|
||||
if (!isNonEmptyString(resolvedValue)) {
|
||||
throw new Error(
|
||||
`models.providers.${providerId}.apiKey resolved to a non-string or empty value.`,
|
||||
);
|
||||
}
|
||||
provider.apiKey = resolvedValue;
|
||||
}
|
||||
}
|
||||
|
||||
const skillEntries = resolved.skills?.entries as Record<string, SkillEntryLike> | undefined;
|
||||
if (skillEntries) {
|
||||
for (const [skillKey, entry] of Object.entries(skillEntries)) {
|
||||
if (!isSecretRef(entry.apiKey)) {
|
||||
continue;
|
||||
}
|
||||
const resolvedValue = await resolveSecretRefValue(entry.apiKey, params.context);
|
||||
if (!isNonEmptyString(resolvedValue)) {
|
||||
throw new Error(
|
||||
`skills.entries.${skillKey}.apiKey resolved to a non-string or empty value.`,
|
||||
);
|
||||
}
|
||||
entry.apiKey = resolvedValue;
|
||||
}
|
||||
}
|
||||
|
||||
const googleChat = resolved.channels?.googlechat as GoogleChatAccountLike | undefined;
|
||||
if (googleChat) {
|
||||
await resolveGoogleChatServiceAccount(
|
||||
googleChat,
|
||||
"channels.googlechat",
|
||||
params.context,
|
||||
params.warnings,
|
||||
);
|
||||
if (isRecord(googleChat.accounts)) {
|
||||
for (const [accountId, account] of Object.entries(googleChat.accounts)) {
|
||||
if (!isRecord(account)) {
|
||||
continue;
|
||||
}
|
||||
await resolveGoogleChatServiceAccount(
|
||||
account as GoogleChatAccountLike,
|
||||
`channels.googlechat.accounts.${accountId}`,
|
||||
params.context,
|
||||
params.warnings,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function resolveAuthStoreSecretRefs(params: {
|
||||
store: AuthProfileStore;
|
||||
context: ResolverContext;
|
||||
warnings: SecretResolverWarning[];
|
||||
agentDir: string;
|
||||
}): Promise<AuthProfileStore> {
|
||||
const resolvedStore = structuredClone(params.store);
|
||||
for (const [profileId, profile] of Object.entries(resolvedStore.profiles)) {
|
||||
if (profile.type === "api_key") {
|
||||
const apiProfile = profile as ApiKeyCredentialLike;
|
||||
const keyRef = isSecretRef(apiProfile.keyRef) ? apiProfile.keyRef : null;
|
||||
if (keyRef && isNonEmptyString(apiProfile.key)) {
|
||||
params.warnings.push({
|
||||
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
|
||||
path: `${params.agentDir}.auth-profiles.${profileId}.key`,
|
||||
message: `auth-profiles ${profileId}: keyRef is set; runtime will ignore plaintext key.`,
|
||||
});
|
||||
}
|
||||
if (keyRef) {
|
||||
const resolvedValue = await resolveSecretRefValue(keyRef, params.context);
|
||||
if (!isNonEmptyString(resolvedValue)) {
|
||||
throw new Error(`auth profile "${profileId}" keyRef resolved to an empty value.`);
|
||||
}
|
||||
apiProfile.key = resolvedValue;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (profile.type === "token") {
|
||||
const tokenProfile = profile as TokenCredentialLike;
|
||||
const tokenRef = isSecretRef(tokenProfile.tokenRef) ? tokenProfile.tokenRef : null;
|
||||
if (tokenRef && isNonEmptyString(tokenProfile.token)) {
|
||||
params.warnings.push({
|
||||
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
|
||||
path: `${params.agentDir}.auth-profiles.${profileId}.token`,
|
||||
message: `auth-profiles ${profileId}: tokenRef is set; runtime will ignore plaintext token.`,
|
||||
});
|
||||
}
|
||||
if (tokenRef) {
|
||||
const resolvedValue = await resolveSecretRefValue(tokenRef, params.context);
|
||||
if (!isNonEmptyString(resolvedValue)) {
|
||||
throw new Error(`auth profile "${profileId}" tokenRef resolved to an empty value.`);
|
||||
}
|
||||
tokenProfile.token = resolvedValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
return resolvedStore;
|
||||
}
|
||||
|
||||
function collectCandidateAgentDirs(config: OpenClawConfig): string[] {
|
||||
const dirs = new Set<string>();
|
||||
dirs.add(resolveUserPath(resolveOpenClawAgentDir()));
|
||||
for (const agentId of listAgentIds(config)) {
|
||||
dirs.add(resolveUserPath(resolveAgentDir(config, agentId)));
|
||||
}
|
||||
return [...dirs];
|
||||
}
|
||||
|
||||
export async function prepareSecretsRuntimeSnapshot(params: {
|
||||
config: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
agentDirs?: string[];
|
||||
loadAuthStore?: (agentDir?: string) => AuthProfileStore;
|
||||
}): Promise<PreparedSecretsRuntimeSnapshot> {
|
||||
const warnings: SecretResolverWarning[] = [];
|
||||
const context: ResolverContext = {
|
||||
config: params.config,
|
||||
env: params.env ?? process.env,
|
||||
fileSecretsPromise: null,
|
||||
};
|
||||
const resolvedConfig = await resolveConfigSecretRefs({
|
||||
config: params.config,
|
||||
context,
|
||||
warnings,
|
||||
});
|
||||
|
||||
const loadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreForRuntime;
|
||||
const candidateDirs = params.agentDirs?.length
|
||||
? [...new Set(params.agentDirs.map((entry) => resolveUserPath(entry)))]
|
||||
: collectCandidateAgentDirs(resolvedConfig);
|
||||
const authStores: Array<{ agentDir: string; store: AuthProfileStore }> = [];
|
||||
for (const agentDir of candidateDirs) {
|
||||
const rawStore = loadAuthStore(agentDir);
|
||||
const resolvedStore = await resolveAuthStoreSecretRefs({
|
||||
store: rawStore,
|
||||
context,
|
||||
warnings,
|
||||
agentDir,
|
||||
});
|
||||
authStores.push({ agentDir, store: resolvedStore });
|
||||
}
|
||||
|
||||
return {
|
||||
sourceConfig: structuredClone(params.config),
|
||||
config: resolvedConfig,
|
||||
authStores,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void {
|
||||
const next = cloneSnapshot(snapshot);
|
||||
setRuntimeConfigSnapshot(next.config, next.sourceConfig);
|
||||
replaceRuntimeAuthProfileStoreSnapshots(next.authStores);
|
||||
activeSnapshot = next;
|
||||
}
|
||||
|
||||
export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapshot | null {
|
||||
return activeSnapshot ? cloneSnapshot(activeSnapshot) : null;
|
||||
}
|
||||
|
||||
export function clearSecretsRuntimeSnapshot(): void {
|
||||
activeSnapshot = null;
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearRuntimeAuthProfileStoreSnapshots();
|
||||
}
|
||||
120
src/secrets/sops.ts
Normal file
120
src/secrets/sops.ts
Normal file
@@ -0,0 +1,120 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { runExec } from "../process/exec.js";
|
||||
|
||||
export const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
|
||||
const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
function normalizeTimeoutMs(value: number | undefined): number {
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
return Math.max(1, Math.floor(value));
|
||||
}
|
||||
return DEFAULT_SOPS_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
function isTimeoutError(message: string | undefined): boolean {
|
||||
return typeof message === "string" && message.toLowerCase().includes("timed out");
|
||||
}
|
||||
|
||||
type SopsErrorContext = {
|
||||
missingBinaryMessage: string;
|
||||
operationLabel: string;
|
||||
};
|
||||
|
||||
function toSopsError(err: unknown, params: SopsErrorContext): Error {
|
||||
const error = err as NodeJS.ErrnoException & { message?: string };
|
||||
if (error.code === "ENOENT") {
|
||||
return new Error(params.missingBinaryMessage, { cause: err });
|
||||
}
|
||||
return new Error(`${params.operationLabel}: ${String(error.message ?? err)}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
|
||||
function ensureDirForFile(filePath: string): void {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
||||
}
|
||||
|
||||
export async function decryptSopsJsonFile(params: {
|
||||
path: string;
|
||||
timeoutMs?: number;
|
||||
missingBinaryMessage: string;
|
||||
}): Promise<unknown> {
|
||||
const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
|
||||
try {
|
||||
const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", params.path], {
|
||||
timeoutMs,
|
||||
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
|
||||
});
|
||||
return JSON.parse(stdout) as unknown;
|
||||
} catch (err) {
|
||||
const error = err as NodeJS.ErrnoException & { message?: string };
|
||||
if (isTimeoutError(error.message)) {
|
||||
throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${params.path}.`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
throw toSopsError(err, {
|
||||
missingBinaryMessage: params.missingBinaryMessage,
|
||||
operationLabel: `sops decrypt failed for ${params.path}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function encryptSopsJsonFile(params: {
|
||||
path: string;
|
||||
payload: Record<string, unknown>;
|
||||
timeoutMs?: number;
|
||||
missingBinaryMessage: string;
|
||||
}): Promise<void> {
|
||||
ensureDirForFile(params.path);
|
||||
const timeoutMs = normalizeTimeoutMs(params.timeoutMs);
|
||||
const tmpPlain = path.join(
|
||||
path.dirname(params.path),
|
||||
`${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.plain.tmp`,
|
||||
);
|
||||
const tmpEncrypted = path.join(
|
||||
path.dirname(params.path),
|
||||
`${path.basename(params.path)}.${process.pid}.${crypto.randomUUID()}.enc.tmp`,
|
||||
);
|
||||
|
||||
fs.writeFileSync(tmpPlain, `${JSON.stringify(params.payload, null, 2)}\n`, "utf8");
|
||||
fs.chmodSync(tmpPlain, 0o600);
|
||||
|
||||
try {
|
||||
await runExec(
|
||||
"sops",
|
||||
[
|
||||
"--encrypt",
|
||||
"--input-type",
|
||||
"json",
|
||||
"--output-type",
|
||||
"json",
|
||||
"--output",
|
||||
tmpEncrypted,
|
||||
tmpPlain,
|
||||
],
|
||||
{
|
||||
timeoutMs,
|
||||
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
|
||||
},
|
||||
);
|
||||
fs.renameSync(tmpEncrypted, params.path);
|
||||
fs.chmodSync(params.path, 0o600);
|
||||
} catch (err) {
|
||||
const error = err as NodeJS.ErrnoException & { message?: string };
|
||||
if (isTimeoutError(error.message)) {
|
||||
throw new Error(`sops encrypt timed out after ${timeoutMs}ms for ${params.path}.`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
throw toSopsError(err, {
|
||||
missingBinaryMessage: params.missingBinaryMessage,
|
||||
operationLabel: `sops encrypt failed for ${params.path}`,
|
||||
});
|
||||
} finally {
|
||||
fs.rmSync(tmpPlain, { force: true });
|
||||
fs.rmSync(tmpEncrypted, { force: true });
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user