packages\/evals："test": "vitest run –config vitest.test.config.ts"（\u6ce8\u610f\u8fd9\u662f evals \u5305\u81ea\u8eab\u57fa\u7840\u8bbe\u65bd\u7684\u5355\u5143\u6d4b\u8bd5，\u4e0d\u662f\u8fd0\u884c eval \u4efb\u52a1）<\/li>\n<\/ul>\n\u8dd1\u5355\u4e2a\u6d4b\u8bd5\u6587\u4ef6<\/h4>\n
AGENTS.md \u7ed9\u51fa\u7684\u51c6\u786e\u547d\u4ee4（\u6309\u5305\u7c7b\u578b\u533a\u5206）：<\/p>\n
\u5bf9\u4f7f\u7528 vitest \u7684\u5305（\u4f8b\u5982 packages\/coding-agent\u3001packages\/ai\u3001packages\/agent），\u4ece\u5305\u6839\u76ee\u5f55\u6267\u884c：<\/p>\n
node "$(git rev-parse –show-toplevel)\/node_modules\/vitest\/dist\/cli.js" –run test\/specific.test.ts<\/p>\n
\u4e4b\u6240\u4ee5\u4e0d\u76f4\u63a5\u5199 npx vitest –run test\/specific.test.ts，\u662f\u4e3a\u4e86\u660e\u786e\u6307\u5411\u4ed3\u5e93\u6839 node_modules \u91cc\u7684 vitest \u53ef\u6267\u884c\u6587\u4ef6（monorepo \u573a\u666f\u4e0b npx \u6709\u65f6\u4f1a\u89e3\u6790\u5230\u9519\u8bef\u7684\u7248\u672c\u6216\u63d0\u793a\u4ea4\u4e92\u5f0f\u5b89\u88c5）\u3002<\/p>\n
\u5bf9 packages\/tui（\u7528 node:test，\u4e0d\u662f vitest），\u4ece packages\/tui \u76ee\u5f55\u6267\u884c：<\/p>\n
node –test test\/specific.test.ts<\/p>\n
\u4e24\u8005\u547d\u4ee4\u5f62\u6001\u5b8c\u5168\u4e0d\u540c，\u6df7\u7528\u4f1a\u76f4\u63a5\u62a5\u9519\u2014\u2014\u8fd9\u4e5f\u662f\u4e3a\u4ec0\u4e48\u8fd9\u6761\u89c4\u5219\u5728 AGENTS.md \u91cc\u88ab\u5355\u72ec\u5217\u51fa\u6765\u3002<\/p>\n
\u4e09\u3001Faux Provider：\u4e0d\u6d88\u8017\u771f\u5b9e\u914d\u989d\u7684\u5047\u6a21\u578b<\/h3>\n
packages\/coding-agent\/test\/suite\/ \u662f\u4ed3\u5e93\u91cc"\u65b0\u4e00\u4ee3"\u7684 Agent \u6d4b\u8bd5\u76ee\u5f55，test\/suite\/README.md \u660e\u786e\u5199\u4e86\u89c4\u5219：<\/p>\n
Use `test\/suite\/` for the new harness-based test suite around `AgentSession` and `AgentSessionRuntime`.<\/p>\n
Rules:
\n– Use `test\/suite\/harness.ts`
\n– Use the faux provider from `packages\/ai\/src\/providers\/faux.ts`
\n– Do not use real provider APIs, real API keys, network calls, or paid tokens
\n– Keep these tests CI-safe and deterministic
\n– Do not use or extend the legacy `test\/test-harness.ts` path unless a missing capability forces it<\/p>\n
\u6700\u540e\u4e00\u6761\u662f\u540e\u6765\u8865\u7684：\u4ed3\u5e93\u91cc\u8fd8\u7559\u7740\u4e00\u6761\u66f4\u8001\u7684\u6d4b\u8bd5\u811a\u624b\u67b6\u8def\u5f84 test\/test-harness.ts，\u660e\u786e\u8981\u6c42\u65b0\u6d4b\u8bd5\u4e0d\u8981\u518d\u7528\u5b83\u3001\u4e5f\u4e0d\u8981\u7ee7\u7eed\u6269\u5c55\u5b83，\u9664\u975e test\/suite\/harness.ts \u786e\u5b9e\u7f3a\u4e86\u67d0\u4e2a\u80fd\u529b\u2014\u2014\u8fd9\u662f\u4e00\u6761\u5178\u578b\u7684"\u53ea\u7559\u9000\u8def\u3001\u4e0d\u9f13\u52b1\u4f7f\u7528"\u5f0f\u8fc1\u79fb\u63d0\u793a，\u8bf4\u660e test\/suite\/ \u8fd9\u5957\u57fa\u4e8e faux provider \u7684\u811a\u624b\u67b6\u662f\u5f53\u524d\u552f\u4e00\u88ab\u5b98\u65b9\u63a8\u8350\u7684\u5199\u6cd5\u3002<\/p>\n
\u4e5f\u5c31\u662f\u8bf4，\u4efb\u4f55\u8981\u6d4b\u8bd5 AgentSession（\u5bf9\u8bdd\u751f\u547d\u5468\u671f\u3001\u5de5\u5177\u8c03\u7528\u3001\u538b\u7f29\u3001\u6269\u5c55\u7b49）\u884c\u4e3a\u7684\u65b0\u6d4b\u8bd5，\u4e00\u5f8b\u4e0d\u5141\u8bb8\u63a5\u89e6\u771f\u5b9e\u7684\u6a21\u578b API，\u800c\u8981\u7528 faux provider \u6a21\u62df\u4e00\u4e2a"\u542c\u8bdd\u7684\u5047\u6a21\u578b"\u3002<\/p>\n
faux provider \u7684\u6838\u5fc3\u601d\u8def<\/h4>\n
packages\/ai\/src\/providers\/faux.ts（708 \u884c）\u5b9e\u73b0\u4e86\u4e00\u4e2a\u5b8c\u6574\u7684\u3001\u9075\u5faa pi \u5185\u90e8 provider \u63a5\u53e3\u7684\u5047 provider\u3002\u5b83\u4e0d\u4f1a\u771f\u7684\u53d1\u7f51\u7edc\u8bf7\u6c42，\u800c\u662f\u4ece\u4e00\u4e2a\u9884\u5148\u8bbe\u7f6e\u597d\u7684"\u54cd\u5e94\u811a\u672c"\u961f\u5217\u91cc\u6309\u987a\u5e8f\u53d6\u51fa\u7ed3\u679c\u8fd4\u56de\u7ed9\u8c03\u7528\u65b9，\u5e76\u4e14\u5b8c\u6574\u6a21\u62df\u4e86\u6d41\u5f0f\u589e\u91cf（text_start\/text_delta\/text_end \u7b49\u4e8b\u4ef6）\u3001\u601d\u8003\u5185\u5bb9（thinking）\u3001\u5de5\u5177\u8c03\u7528\u3001usage \u4f30\u7b97\u3001\u751a\u81f3 deferred（\u5f02\u6b65\u8f6e\u8be2）\u548c abort（\u4e2d\u65ad）\u7b49\u771f\u5b9e provider \u624d\u6709\u7684\u590d\u6742\u884c\u4e3a\u3002<\/p>\n
\u51e0\u4e2a\u5173\u952e\u7684\u6784\u5efa\u5757：<\/p>\n
export function fauxToolCall(name: string, arguments_: ToolCall["arguments"], options: { id?: string } = {}): ToolCall {
\n return { type: "toolCall", id: options.id ?? randomId("tool"), name, arguments: arguments_ };
\n}<\/p>\n
export function fauxAssistantMessage(
\n content: string | FauxContentBlock | FauxContentBlock[],
\n options: { stopReason?: AssistantMessage["stopReason"]; … } = {},
\n): AssistantMessage { … }<\/p>\n
export type FauxResponseFactory = (
\n context: Context,
\n options: SimpleStreamOptions | undefined,
\n state: FauxProviderState,
\n model: Model<string>,
\n) => AssistantMessage | Promise<AssistantMessage>;<\/p>\n
export type FauxResponseStep = AssistantMessage | FauxResponseFactory;<\/p>\n
fauxAssistantMessage + fauxToolCall \u7ec4\u5408\u8d77\u6765，\u5c31\u80fd\u50cf\u7f16\u6392\u5267\u672c\u4e00\u6837，\u58f0\u660e"\u6a21\u578b\u7b2c\u4e00\u8f6e\u8981\u8c03\u7528\u54ea\u4e2a\u5de5\u5177\u3001\u7b2c\u4e8c\u8f6e\u8bf4\u4ec0\u4e48\u8bdd"\u3002\u54cd\u5e94\u4e5f\u652f\u6301\u4f20\u5165\u4e00\u4e2a\u51fd\u6570（FauxResponseFactory），\u6839\u636e\u5f53\u524d\u4e0a\u4e0b\u6587\u52a8\u6001\u751f\u6210\u56de\u590d，\u7528\u4e8e\u6d4b\u8bd5\u9700\u8981"\u770b\u5230\u7528\u6237\u8bf4\u4e86\u4ec0\u4e48\u518d\u51b3\u5b9a\u600e\u4e48\u56de"\u7684\u573a\u666f\u3002\u6d41\u5f0f\u56de\u590d\u662f\u9010 token \u6a21\u62df\u51fa\u6765\u7684，\u771f\u5b9e\u8fd8\u539f\u4e86\u6253\u5b57\u673a\u5f0f\u7684\u589e\u91cf\u63a8\u9001（streamWithDeltas \u5185\u90e8\u6309 splitStringByTokenSize \u628a\u6587\u672c\u5207\u6210\u82e5\u5e72"\u5047 token"，\u9010\u5757 push \u5230\u4e8b\u4ef6\u6d41\u91cc），\u8fd9\u8ba9\u4f9d\u8d56\u6d41\u5f0f\u4e8b\u4ef6\u7684 UI\/\u65e5\u5fd7\u903b\u8f91\u4e5f\u80fd\u88ab\u8986\u76d6\u5230，\u800c\u4e0d\u53ea\u662f\u6d4b\u8bd5"\u6700\u7ec8\u7ed3\u679c\u5bf9\u4e0d\u5bf9"\u3002<\/p>\n
test\/suite\/harness.ts：\u628a faux provider \u63a5\u5230\u771f\u5b9e\u7684 AgentSession \u4e0a<\/h4>\n
harness.ts \u91cc\u7684 createHarness() \u505a\u7684\u4e8b\u60c5，\u662f\u642d\u4e00\u4e2a"\u4e94\u810f\u4ff1\u5168\u4f46\u5b8c\u5168\u5185\u5b58\u5316"\u7684\u6d4b\u8bd5\u73af\u5883：\u6ce8\u518c faux provider\u3001\u4f2a\u9020\u4e00\u4e2a provider \u7684 API Key（"faux-key"）\u3001\u7528\u5185\u5b58\u7248\u7684 SessionManager\/SettingsManager\/AuthStorage，\u6700\u540e\u5b9e\u4f8b\u5316\u4e00\u4e2a\u771f\u5b9e\u7684 AgentSession：<\/p>\n
export async function createHarness(options: HarnessOptions = {}): Promise<Harness> {
\n const tempDir = createTempDir();
\n const fauxProvider: FauxProviderRegistration = registerFauxProvider({ models: options.models });
\n fauxProvider.setResponses([]);
\n const model = fauxProvider.getModel();
\n const sessionManager = SessionManager.inMemory();
\n const settingsManager = SettingsManager.inMemory(options.settings);
\n const authStorage = AuthStorage.inMemory();
\n if (withConfiguredAuth) {
\n await authStorage.modify(model.provider, async () => ({ type: "api_key", key: "faux-key" }));
\n }
\n const agent = new Agent({
\n getApiKey: () => (withConfiguredAuth ? "faux-key" : undefined),
\n streamFn: streamSimple,
\n initialState: { model, systemPrompt: options.systemPrompt ?? "You are a test assistant.", tools: [] },
\n …
\n });
\n const session = new AgentSession({ agent, sessionManager, settingsManager, cwd: tempDir, … });
\n return {
\n session, sessionManager, settingsManager, authStorage,
\n faux: fauxProvider, setResponses: fauxProvider.setResponses, events, eventsOfType, tempDir,
\n cleanup() {
\n session.dispose();
\n fauxProvider.unregister();
\n if (existsSync(tempDir)) rmSync(tempDir, { recursive: true });
\n },
\n };
\n}<\/p>\n
\u6d4b\u8bd5\u6587\u4ef6\u62ff\u5230 Harness \u4e4b\u540e，\u5178\u578b\u7528\u6cd5\u662f：\u5148\u7528 harness.setResponses([…]) \u7f16\u6392\u597d"\u6a21\u578b\u63a5\u4e0b\u6765\u8981\u8bf4\u4ec0\u4e48\/\u8c03\u7528\u54ea\u4e2a\u5de5\u5177"，\u518d\u8c03\u7528 harness.session.prompt(…) \u89e6\u53d1\u771f\u5b9e\u7684 Agent \u4e3b\u5faa\u73af，\u6700\u540e\u7528 harness.events（\u8ba2\u9605 AgentSession \u629b\u51fa\u7684\u4e8b\u4ef6）\u6216 getAssistantTexts(harness)\/getUserTexts(harness) \u4e4b\u7c7b\u7684\u8f85\u52a9\u51fd\u6570\u65ad\u8a00\u7ed3\u679c\u3002packages\/coding-agent\/test\/suite\/regressions\/2023-queued-slash-command-followup.test.ts \u662f\u4e00\u4e2a\u5f88\u5178\u578b\u7684\u4f8b\u5b50：<\/p>\n
harness.setResponses([
\n fauxAssistantMessage(fauxToolCall("wait", {}), { stopReason: "toolUse" }),
\n fauxAssistantMessage("first turn complete"),
\n fauxAssistantMessage("queued follow-up handled by model"),
\n]);<\/p>\n
const promptPromise = harness.session.prompt("start");
\nawait sawToolStart;
\nextensionApi?.sendUserMessage("\/testcmd queued", { deliverAs: "followUp" });
\nreleaseToolExecution?.();
\nawait promptPromise;<\/p>\n
expect(commandRuns).toEqual([]);
\nexpect(getAssistantTexts(harness)).toContain("queued follow-up handled by model");<\/p>\n
\u8fd9\u4e2a\u6d4b\u8bd5\u5b8c\u5168\u6ca1\u6709\u8054\u7f51，\u4e5f\u6ca1\u6709\u7528\u5230\u4efb\u4f55\u771f\u5b9e\u5bc6\u94a5，\u5374\u5b8c\u6574\u8dd1\u901a\u4e86"\u7528\u6237\u8f93\u5165 \u2192 \u6a21\u578b\u51b3\u5b9a\u8c03\u7528\u5de5\u5177 \u2192 \u5de5\u5177\u6267\u884c\u4e2d\u6392\u961f\u4e00\u6761 slash \u547d\u4ee4 \u2192 \u5de5\u5177\u6267\u884c\u5b8c\u6bd5 \u2192 \u6a21\u578b\u7ee7\u7eed\u56de\u590d"\u8fd9\u6837\u4e00\u6761\u771f\u5b9e\u7684\u3001\u6709\u65f6\u5e8f\u7ade\u4e89\u7684\u4e1a\u52a1\u94fe\u8def\u3002\u8fd9\u6b63\u662f faux provider \u6a21\u5f0f\u7684\u4ef7\u503c\u6240\u5728：\u628a"\u6a21\u578b\u4f1a\u600e\u4e48\u56de\u590d"\u53d8\u6210\u4e00\u4e2a\u53ef\u4ee5\u7531\u6d4b\u8bd5\u4ee3\u7801\u5b8c\u5168\u638c\u63a7\u7684\u786e\u5b9a\u6027\u8f93\u5165，\u4ece\u800c\u8ba9\u539f\u672c\u9ad8\u5ea6\u4f9d\u8d56\u5916\u90e8\u4e0d\u786e\u5b9a\u6027（\u771f\u5b9e\u6a21\u578b\u7684\u8f93\u51fa）\u7684 Agent \u903b\u8f91，\u53d8\u6210\u53ef\u4ee5\u7528\u4f20\u7edf\u5355\u5143\u6d4b\u8bd5\u624b\u6cd5\u8986\u76d6\u7684\u786e\u5b9a\u6027\u4ee3\u7801\u3002<\/p>\n
\u56db\u3001Evals：\u8bc4\u6d4b\u884c\u4e3a\u8d28\u91cf，\u800c\u4e0d\u662f\u65ad\u8a00\u5bf9\u9519<\/h3>\n
packages\/evals\/ \u662f\u4e00\u4e2a\u72ec\u7acb\u7684\u5305，packages\/evals\/README.md \u5f00\u7bc7\u5c31\u628a\u5b83\u548c\u666e\u901a\u6d4b\u8bd5\u533a\u5206\u5f00：<\/p>\n
Pi evals are behavioral, model-backed checks for Pi workflows. They adapt a real AgentSession to vitest-evals, run it in isolated temporary project and agent directories, and attach native Pi session artifacts. Use them to measure end-to-end behavior and compare prompts, tools, skills, models, or other harness configurations.<\/p>\n
\u5173\u952e\u533a\u522b\u5728\u4e8e：faux provider \u6d4b\u8bd5\u7528\u5047\u6a21\u578b\u9a8c\u8bc1"\u4ee3\u7801\u903b\u8f91\u5bf9\u4e0d\u5bf9"，\u800c evals \u7528\u771f\u5b9e\u6a21\u578b\u9a8c\u8bc1"\u7ed9\u5b9a\u8fd9\u5957\u63d0\u793a\u8bcd\u3001\u5de5\u5177\u3001\u6a21\u578b\u7ec4\u5408，Agent \u5b8c\u6210\u771f\u5b9e\u4efb\u52a1\u7684\u6548\u679c\u597d\u4e0d\u597d"\u3002\u8fd9\u662f\u4e24\u4e2a\u5b8c\u5168\u4e0d\u540c\u7ef4\u5ea6\u7684\u95ee\u9898\u2014\u2014\u4ee3\u7801\u903b\u8f91\u6b63\u786e\u4e0d\u4ee3\u8868\u63d0\u793a\u8bcd\u5199\u5f97\u597d，\u53cd\u4e4b\u4ea6\u7136\u3002<\/p>\n
pi-harness.ts：\u628a\u771f\u5b9e AgentSession \u63a5\u5165 vitest-evals<\/h4>\n
packages\/evals\/src\/pi-harness.ts \u63d0\u4f9b\u7684 createPiCodingAgentHarness(…) \u662f evals \u7684\u6838\u5fc3\u811a\u624b\u67b6\u3002\u5b83\u5728\u4e34\u65f6\u76ee\u5f55\u91cc\u521b\u5efa\u4e00\u4e2a\u771f\u5b9e\u7684\u5de5\u4f5c\u533a\u548c agent \u76ee\u5f55，\u7528\u771f\u5b9e\u7684 ModelRuntime \u89e3\u6790\u51fa\u771f\u5b9e\u7684\u6a21\u578b（provider + model id \u6765\u81ea CLI \u53c2\u6570\u6216\u73af\u5883\u53d8\u91cf PI_PROVIDER\/PI_MODEL），\u5b9e\u4f8b\u5316\u4e00\u4e2a\u771f\u5b9e\u7684 AgentSession \u5e76\u5bf9\u5176\u4e0b\u8fbe prompt：<\/p>\n
export function resolveModelSelection(
\n explicitModel: PiCodingAgentModelSelection | undefined,
\n environment: { PI_PROVIDER?: string; PI_MODEL?: string } = process.env,
\n): PiCodingAgentModelSelection {
\n const provider = (explicitModel?.provider ?? environment.PI_PROVIDER)?.trim();
\n const id = (explicitModel?.id ?? environment.PI_MODEL)?.trim();
\n if (!provider || !id) {
\n throw new Error("Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults.");
\n }
\n return { provider, id };
\n}<\/p>\n
\u8fd0\u884c\u65f6\u901a\u8fc7：<\/p>\n
npm run eval — –provider openai –model gpt-5.6-sol
\n# \u7b49\u4ef7\u4e8e
\nPI_PROVIDER=openai PI_MODEL=gpt-5.6-sol npm run eval<\/p>\n
\u4e5f\u5c31\u662f\u8bf4，\u8fd0\u884c evals \u5fc5\u987b\u63d0\u4f9b\u4e00\u4e2a\u771f\u5b9e\u53ef\u7528\u7684 provider \u548c\u6a21\u578b（\u4ee5\u53ca\u5bf9\u5e94\u7684\u8ba4\u8bc1\u4fe1\u606f，\u8d70 pi \u6b63\u5e38\u7684 ModelRuntime \u903b\u8f91，\u53ef\u4ee5\u662f\u8ba2\u9605\u51ed\u8bc1\u4e5f\u53ef\u4ee5\u662f API Key \u73af\u5883\u53d8\u91cf）\u3002\u8fd9\u4e0e\u524d\u9762\u5355\u5143\u6d4b\u8bd5"\u7edd\u4e0d\u4f7f\u7528\u771f\u5b9e\u5bc6\u94a5"\u7684\u539f\u5219\u5f62\u6210\u4e86\u9c9c\u660e\u5bf9\u6bd4\u2014\u2014evals \u5b58\u5728\u7684\u610f\u4e49\u5c31\u662f\u89c2\u5bdf\u771f\u5b9e\u6a21\u578b\u7684\u771f\u5b9e\u884c\u4e3a\u3002<\/p>\n
\u6bcf\u6b21\u8fd0\u884c\u4f1a\u5728 packages\/evals\/.eval\/ \u4e0b\u751f\u6210\u4e00\u4e2a\u5e26\u65f6\u95f4\u6233\u7684 artifact \u76ee\u5f55，runs.jsonl \u8bb0\u5f55\u6bcf\u6b21 harness \u8fd0\u884c，\u5e76\u628a\u539f\u751f\u7684 pi \u4f1a\u8bdd JSONL \u9644\u52a0\u5728 sessions\/ \u4e0b\u2014\u2014\u8fd9\u610f\u5473\u7740 eval \u4ea7\u7269\u91cc\u53ef\u80fd\u5305\u542b\u771f\u5b9e\u7684 prompt\u3001\u6a21\u578b\u8f93\u51fa\u3001\u5de5\u5177\u8c03\u7528\u7ec6\u8282，\u9700\u8981\u6309\u654f\u611f\u6570\u636e\u5bf9\u5f85\u3002<\/p>\n
smoke.eval.ts：\u6700\u57fa\u7840\u7684\u7aef\u5230\u7aef\u53ef\u7528\u6027\u68c0\u67e5<\/h4>\n
packages\/evals\/src\/smoke.eval.ts \u662f\u6700\u7b80\u5355\u7684\u4e00\u4e2a eval，\u53ea\u9a8c\u8bc1"Agent \u80fd\u4e0d\u80fd\u6b63\u5e38\u8dd1\u5b8c\u4e00\u8f6e\u771f\u5b9e\u5bf9\u8bdd\u5e76\u7ed9\u51fa\u6b63\u786e\u7b54\u6848"：<\/p>\n
const piCodingAgentHarness = createPiCodingAgentHarness({ noTools: "all" });<\/p>\n
describeEval("Pi Coding Agent smoke", { harness: piCodingAgentHarness }, (it) => {
\n it("runs a basic prompt end to end", async ({ run }) => {
\n const result = await run("What's the capital of France? Respond with only the city name.");<\/p>\n
expect(result.output.trim()).toBe("Paris");
\n expect(result.errors).toEqual([]);
\n expect(result.usage.provider).toBe(process.env.PI_PROVIDER);
\n expect(result.usage.model).toBe(process.env.PI_MODEL);
\n expect(result.usage.totalTokens).toBeGreaterThan(0);
\n });
\n});<\/p>\n
\u5b83\u7528 noTools: "all" \u5173\u95ed\u4e86\u5168\u90e8\u5de5\u5177，\u53ea\u6d4b\u6700\u57fa\u7840\u7684"\u53d1\u6d88\u606f\u3001\u6536\u56de\u7b54"\u94fe\u8def，\u672c\u8d28\u4e0a\u662f\u7ed9\u6574\u6761\u771f\u5b9e\u8c03\u7528\u94fe（\u6a21\u578b\u89e3\u6790 \u2192 \u8ba4\u8bc1 \u2192 \u8bf7\u6c42 \u2192 \u89e3\u6790\u54cd\u5e94 \u2192 session \u8bb0\u8d26）\u505a\u4e00\u6b21\u5192\u70df\u6d4b\u8bd5（smoke test）\u3002<\/p>\n
extensions.eval.ts：\u8bc4\u6d4b\u66f4\u590d\u6742\u7684\u3001\u5e26\u5de5\u5177\u8c03\u7528\u7684\u5b8c\u6574\u5de5\u4f5c\u6d41<\/h4>\n
packages\/evals\/src\/extensions.eval.ts \u590d\u6742\u5f97\u591a，\u5b83\u9a8c\u8bc1\u7684\u662f"Agent \u80fd\u5426\u6839\u636e\u7cfb\u7edf\u63d0\u793a\u8bcd\u7684\u6307\u5f15，\u6b63\u786e\u5730\u521b\u5efa\u4e00\u4e2a pi \u6269\u5c55\u3001reload \u52a0\u8f7d\u5b83\u3001\u518d\u8c03\u7528\u6269\u5c55\u6ce8\u518c\u7684\u5de5\u5177\u5b8c\u6210\u4efb\u52a1"\u3002\u5b83\u7528 createJudge<…>(…) \u5b9a\u4e49\u4e86\u4e00\u4e2a\u57fa\u4e8e\u89c4\u5219\u7684\u8bc4\u59d4\u51fd\u6570，\u68c0\u67e5\u751f\u6210\u7684\u6269\u5c55\u6e90\u7801\u662f\u5426 import \u4e86\u6b63\u786e\u7684\u5305\u3001\u6269\u5c55\u662f\u5426\u6210\u529f\u52a0\u8f7d\u3001\u5de5\u5177\u8c03\u7528\u53c2\u6570\u548c\u8fd4\u56de\u503c\u662f\u5426\u7b26\u5408\u9884\u671f：<\/p>\n
const ExtensionAuthoringJudge = createJudge<PiCodingAgentInput, ExtensionAuthoringOutput>(
\n "ExtensionAuthoringJudge",
\n ({ output, toolCalls }) => {
\n const failures: string[] = [];
\n if (output.extensionSource === null) {
\n failures.push("generated extension source is unavailable");
\n } else if (!imports.includes("@earendil-works\/pi-coding-agent")) {
\n failures.push("extension does not import the canonical @earendil-works\/pi-coding-agent package");
\n }
\n return {
\n score: failures.length === 0 ? 1 : 0,
\n metadata: { rationale: failures.length === 0 ? "Extension authoring workflow completed." : failures.join("; ") },
\n };
\n },
\n);<\/p>\n
\u66f4\u503c\u5f97\u6ce8\u610f\u7684\u662f\u5b83\u7528 evalHarnessTable(…) \u505a\u4e86\u4e00\u6b21\u5bf9\u7167\u5b9e\u9a8c：\u628a"\u7cfb\u7edf\u63d0\u793a\u8bcd\u91cc\u5305\u542b\u6269\u5c55\u5f00\u53d1\u6307\u5357\u548c pi \u6587\u6863" vs "\u4e0d\u5305\u542b"\u8fd9\u4e24\u79cd\u60c5\u51b5\u5404\u8dd1\u4e00\u904d\u76f8\u540c\u7684\u4efb\u52a1，\u6bd4\u8f83\u4e24\u7ec4\u7684\u901a\u8fc7\u7387\u5dee\u5f02：<\/p>\n
const extensionHarnessTable = evalHarnessTable("Pi extension authoring system prompt", {
\n baseline: createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation),
\n candidate: createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride),
\n});<\/p>\n
\u8fd9\u6b63\u662f"\u8bc4\u6d4b Agent \u884c\u4e3a\u8d28\u91cf"\u548c"\u4f20\u7edf\u65ad\u8a00\u5f0f\u5355\u5143\u6d4b\u8bd5"\u7684\u672c\u8d28\u533a\u522b：\u5355\u5143\u6d4b\u8bd5\u95ee\u7684\u662f"\u8fd9\u6bb5\u4ee3\u7801\u662f\u5426\u7b26\u5408\u89c4\u683c"，\u662f\u975e\u9ed1\u5373\u767d\u7684；evals \u95ee\u7684\u662f"\u8fd9\u5957\u63d0\u793a\u8bcd\/\u5de5\u5177\/\u6a21\u578b\u7684\u7ec4\u5408，\u5728\u771f\u5b9e\u6a21\u578b\u7684\u4e0d\u786e\u5b9a\u6027\u4e0b，\u5b8c\u6210\u4efb\u52a1\u7684\u6210\u529f\u7387\u6709\u591a\u9ad8"，\u662f\u6982\u7387\u6027\u7684\u3001\u9700\u8981\u7528\u5bf9\u7167\u7ec4\u548c\u8bc4\u5206\u673a\u5236\u6765\u8861\u91cf\u7684\u3002README.md \u91cc\u4e5f\u660e\u786e\u5efa\u8bae\u5bf9\u7167\u5b9e\u9a8c\u7c7b\u7684 eval \u628a judgeThreshold: null，\u8ba9\u4f4e\u5206\u53ea\u4f5c\u4e3a\u89c2\u5bdf\u8bb0\u5f55\u800c\u4e0d\u76f4\u63a5\u5224\u5931\u8d25\u2014\u2014\u56e0\u4e3a\u771f\u5b9e\u6a21\u578b\u8f93\u51fa\u672c\u8eab\u5c31\u6709\u6ce2\u52a8\u3002<\/p>\n
docs.eval.ts：\u8ba9\u771f\u5b9e Agent \u81ea\u5df1\u5ba1\u8ba1\u6587\u6863\u662f\u5426\u4e0e\u5b9e\u73b0\u4e00\u81f4<\/h4>\n
packages\/evals\/src\/ \u540e\u6765\u65b0\u589e\u4e86\u4e00\u4e2a\u5f88\u6709\u610f\u601d\u7684 eval：docs.eval.ts\u3002\u5b83\u7528 globSync("**\/*.md", { cwd: docsRoot }) \u626b\u51fa packages\/coding-agent\/docs\/ \u4e0b\u7684\u6bcf\u4e00\u7bc7\u6587\u6863，\u5bf9\u6bcf\u4e00\u7bc7\u90fd\u8dd1\u4e00\u6b21\u771f\u5b9e\u7684 AgentSession，\u8ba9 Agent \u81ea\u5df1\u53bb\u8bfb\u6587\u6863\u5185\u5bb9\u3001\u518d\u7528 read\/grep\/find\/ls \u8fd9\u51e0\u4e2a\u53ea\u8bfb\u5de5\u5177\u53bb\u7ffb\u6e90\u7801，\u6700\u540e\u8c03\u7528\u4e00\u4e2a\u4e13\u95e8\u5b9a\u4e49\u7684 submit_documentation_audit \u5de5\u5177\u63d0\u4ea4\u7ed3\u6784\u5316\u7684\u5ba1\u8ba1\u7ed3\u8bba（verdict: "match" | "mismatch"，\u9644\u5e26\u6587\u6863\u8bc1\u636e\u548c\u5b9e\u73b0\u8bc1\u636e）：<\/p>\n
const submitDocumentationAuditTool = defineTool({
\n name: "submit_documentation_audit",
\n description: "Submit the final verdict after completing the documentation investigation.",
\n parameters: Type.Object({
\n verdict: Type.Union([Type.Literal("match"), Type.Literal("mismatch")]),
\n explanation: Type.String({ minLength: 1, maxLength: 2000 }),
\n documentationEvidence: Type.String({ minLength: 1, maxLength: 2000 }),
\n implementationEvidence: Type.String({ minLength: 1, maxLength: 3000 }),
\n }, { additionalProperties: false }),
\n constrainedSampling: { type: "json_schema", strict: "prefer" },
\n \/\/ …
\n});<\/p>\n
\u8fd9\u662f\u4e00\u4e2a\u8df3\u51fa"\u4eba\u5199\u65ad\u8a00\u3001\u4ee3\u7801\u8dd1\u65ad\u8a00"\u4f20\u7edf\u601d\u8def\u7684\u7528\u6cd5：\u4e0e\u5176\u9760\u4eba\u5de5 review \u4fdd\u8bc1\u6587\u6863\u4e0d\u8fc7\u65f6,\u4e0d\u5982\u8ba9 Agent \u672c\u8eab（\u65e2\u7136\u5b83\u5c31\u662f\u8981\u53bb\u8bfb\u8fd9\u4e9b\u6587\u6863\u6765\u7406\u89e3\u81ea\u5df1\u600e\u4e48\u7528）\u5145\u5f53\u5ba1\u8ba1\u5458，\u7528\u5b83\u81ea\u5df1\u7684\u9605\u8bfb\u7406\u89e3\u80fd\u529b\u53bb\u6bd4\u5bf9"\u6587\u6863\u8bf4\u7684"\u548c"\u4ee3\u7801\u5b9e\u9645\u505a\u7684"\u662f\u5426\u4e00\u81f4，\u518d\u7528\u4e00\u4e2a constrainedSampling: { type: "json_schema" } \u7684\u81ea\u5b9a\u4e49\u5de5\u5177\u5f3a\u5236\u5b83\u8f93\u51fa\u7ed3\u6784\u5316\u3001\u53ef\u7f16\u7a0b\u6d88\u8d39\u7684\u7ed3\u8bba,\u800c\u4e0d\u662f\u4e00\u6bb5\u81ea\u7531\u6587\u672c\u3002\u8fd9\u4e5f\u89e3\u91ca\u4e86\u672c\u8bfe\u7a0b\u524d\u9762\u51e0\u7bc7\u4e3a\u4ec0\u4e48\u53cd\u590d\u5f3a\u8c03"\u6587\u6863\u63cf\u8ff0\u5728\u5feb\u901f\u8fed\u4ee3\u7684\u4ed3\u5e93\u91cc\u5bb9\u6613\u548c\u5b9e\u73b0\u8131\u8282"\u2014\u2014pi \u9879\u76ee\u81ea\u5df1\u4e5f\u5728\u7528 evals \u8fd9\u5957\u57fa\u7840\u8bbe\u65bd\u6765\u5bf9\u6297\u8fd9\u4e2a\u95ee\u9898\u3002<\/p>\n
packages\/evals\/vitest.config.ts \u548c packages\/evals\/vitest.test.config.ts \u662f\u4e24\u4efd\u4e0d\u540c\u914d\u7f6e：\u524d\u8005\u7528\u4e8e\u8dd1\u771f\u6b63\u7684 eval \u4efb\u52a1（npm run eval），\u540e\u8005\u7528\u4e8e\u8dd1 evals \u8fd9\u4e2a\u5305\u81ea\u8eab\u57fa\u7840\u8bbe\u65bd\u4ee3\u7801\u7684\u5355\u5143\u6d4b\u8bd5（npm run test，\u9a8c\u8bc1 harness-table.ts\u3001summary.ts\u3001artifacts.ts \u8fd9\u4e9b\u5de5\u5177\u51fd\u6570\u672c\u8eab\u6ca1\u5199\u9519）\u2014\u2014\u8fd9\u4e24\u8005\u4e0d\u8981\u6df7\u6dc6：\u4e00\u4e2a\u662f"\u7528 evals \u8bc4\u6d4b pi"，\u4e00\u4e2a\u662f"\u6d4b\u8bd5 evals \u5de5\u5177\u672c\u8eab"\u3002<\/p>\n
\u4e94\u3001\u56de\u5f52\u6d4b\u8bd5\u7684\u7ec4\u7ec7\u7ea6\u5b9a（\u73b0\u5df2\u4ece\u6587\u6863\u89c4\u5219\u964d\u7ea7\u4e3a\u7ea6\u5b9a\u4fd7\u6210）<\/h3>\n
\u9700\u8981\u66f4\u65b0\u4e00\u5904\u8ba4\u77e5：\u65e9\u671f packages\/coding-agent\/test\/suite\/README.md \u66fe\u7528\u4e00\u6bb5"Organization"\u660e\u786e\u5199\u51fa\u56de\u5f52\u6d4b\u8bd5\u7684\u7ec4\u7ec7\u89c4\u5219（broad \u6d4b\u8bd5\u653e test\/suite\/ \u4e0b\u3001issue \u76f8\u5173\u7684\u56de\u5f52\u6d4b\u8bd5\u653e test\/suite\/regressions\/\u3001\u6309 <issue-number>-<short-slug>.test.ts \u547d\u540d），\u4f46\u8fd9\u6bb5\u89c4\u5219\u5728\u540e\u7eed\u4e00\u6b21\u6587\u6863\u6e05\u7406（docs(coding-agent): remove issue-specific regression test placement rule）\u4e2d\u88ab\u4ece README \u91cc\u6574\u6bb5\u5220\u9664\u4e86，\u73b0\u5728\u7684 README \u53ea\u4fdd\u7559\u4e86\u4e0a\u4e00\u8282\u5217\u51fa\u7684\u90a3\u51e0\u6761\u786c\u6027\u89c4\u5219，\u4e0d\u518d\u5bf9\u76ee\u5f55\u7ec4\u7ec7\u65b9\u5f0f\u505a\u6587\u6863\u5c42\u9762\u7684\u5f3a\u5236\u7ea6\u5b9a\u3002<\/p>\n
\u4e0d\u8fc7\u5b9e\u9645\u4ee3\u7801\u5e93\u91cc\u7684\u7ec4\u7ec7\u65b9\u5f0f\u5e76\u6ca1\u6709\u53d8：test\/suite\/regressions\/ \u76ee\u5f55\u4f9d\u7136\u5b58\u5728，\u5199\u4f5c\u672c\u6587\u65f6\u91cc\u9762\u4ecd\u6709 70 \u591a\u4e2a\u6d4b\u8bd5\u6587\u4ef6，\u6587\u4ef6\u540d\u4f9d\u7136\u6e05\u4e00\u8272\u662f <issue \u7f16\u53f7>-<\u7b80\u77ed\u63cf\u8ff0>.test.ts \u7684\u683c\u5f0f，\u4f8b\u5982：<\/p>\n
2023-queued-slash-command-followup.test.ts
\n3317-network-connection-lost-retry.test.ts
\n5109-exclude-tools.test.ts
\n7290-json-stream-linear.test.ts<\/p>\n
\u4e5f\u5b58\u5728\u4e2a\u522b\u6ca1\u6709\u7f16\u53f7\u524d\u7f00\u7684\u6587\u4ef6（\u5982 pre-prompt-compaction-no-continue.test.ts\u3001startup-session-rebind-duplicate-subscription.test.ts），\u8fd9\u5370\u8bc1\u4e86\u76ee\u5f55\u7ec4\u7ec7\u672c\u8eab\u73b0\u5728\u53ea\u662f\u7ea6\u5b9a\u4fd7\u6210\u7684\u5b9e\u8df5,\u4e0d\u518d\u662f\u6587\u6863\u5f3a\u5236\u7684\u89c4\u5219\u3002\u8fd9\u79cd\u547d\u540d\u65b9\u5f0f\u7684\u597d\u5904\u4f9d\u7136\u5f88\u76f4\u63a5：\u51e0\u5e74\u540e\u518d\u770b\u5230\u8fd9\u4e2a\u6d4b\u8bd5\u6587\u4ef6，\u4e0d\u9700\u8981\u770b\u6d4b\u8bd5\u5185\u5bb9\u5c31\u77e5\u9053\u5b83\u5728\u9632\u6b62\u54ea\u4e2a\u5386\u53f2 bug \u590d\u73b0\u3002<\/p>\n
\u5f53\u524d AGENTS.md \u91cc\u5173\u4e8e\u56de\u5f52\u6d4b\u8bd5\u7684\u8981\u6c42\u5df2\u7ecf\u4ece"\u6309\u56fa\u5b9a\u76ee\u5f55 + \u6587\u4ef6\u540d\u683c\u5f0f\u7ec4\u7ec7"\u6539\u6210\u4e86\u66f4\u8f7b\u91cf\u7684\u4e00\u53e5\u8bdd："\u4fee\u590d\u67d0\u4e2a GitHub issue \u7684\u56de\u5f52\u6d4b\u8bd5，\u8981\u5728\u6d4b\u8bd5\u65c1\u8fb9\u52a0\u4e00\u6761\u5e26 issue \u7f16\u53f7\u7684\u6ce8\u91ca"（When regressions tests for fixing a github issue, add a comment with the github issue number next to the test.）\u2014\u2014\u4ece"\u9760\u6587\u4ef6\u8def\u5f84\u548c\u547d\u540d\u89c4\u8303\u4f20\u9012\u53ef\u8ffd\u6eaf\u6027"，\u53d8\u6210"\u9760\u4ee3\u7801\u6ce8\u91ca\u4f20\u9012\u53ef\u8ffd\u6eaf\u6027"，\u7ea6\u675f\u76ee\u6807\u6ca1\u53d8（\u51e0\u5e74\u540e\u8fd8\u80fd\u987a\u85e4\u6478\u74dc\u627e\u5230\u95ee\u9898\u80cc\u666f），\u4f46\u8f7d\u4f53\u4ece\u6587\u4ef6\u7cfb\u7edf\u7ea6\u5b9a\u653e\u5bbd\u6210\u4e86\u6e90\u7801\u5185\u6ce8\u91ca,\u8fd9\u5bf9\u8d21\u732e\u8005\u6765\u8bf4\u95e8\u69db\u66f4\u4f4e，\u4e5f\u4e0d\u518d\u8981\u6c42\u65b0\u6d4b\u8bd5\u5fc5\u987b\u4e25\u683c\u5957\u8fdb test\/suite\/regressions\/ \u8fd9\u4e00\u4e2a\u76ee\u5f55\u3002\u5f53\u7136，test\/suite\/harness.ts + faux provider\u3001\u4e0d\u80fd\u5f15\u5165\u771f\u5b9e provider \u4f9d\u8d56\u8fd9\u4e24\u6761\u786c\u7ea6\u675f\u6ca1\u6709\u53d8\u5316\u3002<\/p>\n
\u52a8\u624b\u7ec3\u4e60<\/h3>\n