fix(core): support number type for aiInput value field (#1339)

quanru · claude · web-flow · commit 13b4f1ddee7a · 2025-10-20T15:10:53.000+08:00
* fix(core): support number type for aiInput value field This change allows aiInput.value to accept both string and number types, addressing scenarios where: 1. AI models return numeric values instead of strings 2. YAML files contain unquoted numbers that parse as number type Changes: - Updated type definitions to accept string | number - Added Zod schema transformation to convert numbers to strings - Updated runtime validation to accept both types - Added explicit conversion in YAML player as fallback All conversions happen internally and are transparent to users. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> * fix(core): update aiInput type signatures to accept number values Update the TypeScript method signatures for aiInput to accept string | number for the value parameter, matching the runtime implementation. Changes: - New signature: opt parameter now accepts { value: string | number } - Legacy signature: first parameter now accepts string | number - Implementation signature: locatePromptOrValue now accepts TUserPrompt | string | number - Type assertion updated from `as string` to `as string | number` This ensures type safety and allows users to pass number values directly without TypeScript errors, while maintaining backward compatibility with existing string-based usage. Fixes type errors in test cases that use number values. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com> --------- Co-authored-by: Claude <noreply@anthropic.com>
diff --git a/packages/core/src/agent/agent.ts b/packages/core/src/agent/agent.ts
@@ -496,7 +496,7 @@ export class Agent<
   // New signature, always use locatePrompt as the first param
   async aiInput(
     locatePrompt: TUserPrompt,
-    opt: LocateOption & { value: string } & {
+    opt: LocateOption & { value: string | number } & {
       autoDismissKeyboard?: boolean;
     } & { mode?: 'replace' | 'clear' | 'append' },
   ): Promise<any>;
@@ -506,7 +506,7 @@ export class Agent<
    * @deprecated Use aiInput(locatePrompt, opt) instead where opt contains the value
    */
   async aiInput(
-    value: string,
+    value: string | number,
     locatePrompt: TUserPrompt,
     opt?: LocateOption & { autoDismissKeyboard?: boolean } & {
       mode?: 'replace' | 'clear' | 'append';
@@ -515,19 +515,19 @@ export class Agent<
 
   // Implementation
   async aiInput(
-    locatePromptOrValue: TUserPrompt | string,
+    locatePromptOrValue: TUserPrompt | string | number,
     locatePromptOrOpt:
       | TUserPrompt
-      | (LocateOption & { value: string } & {
+      | (LocateOption & { value: string | number } & {
           autoDismissKeyboard?: boolean;
         } & { mode?: 'replace' | 'clear' | 'append' }) // AndroidDeviceInputOpt &
       | undefined,
     optOrUndefined?: LocateOption, // AndroidDeviceInputOpt &
   ) {
-    let value: string;
+    let value: string | number;
     let locatePrompt: TUserPrompt;
     let opt:
-      | (LocateOption & { value: string } & {
+      | (LocateOption & { value: string | number } & {
           autoDismissKeyboard?: boolean;
         } & { mode?: 'replace' | 'clear' | 'append' }) // AndroidDeviceInputOpt &
       | undefined;
@@ -542,14 +542,14 @@ export class Agent<
       locatePrompt = locatePromptOrValue as TUserPrompt;
       const optWithValue = locatePromptOrOpt as LocateOption & {
         // AndroidDeviceInputOpt &
-        value: string;
+        value: string | number;
         autoDismissKeyboard?: boolean;
       };
       value = optWithValue.value;
       opt = optWithValue;
     } else {
       // Legacy signature: aiInput(value, locatePrompt, opt)
-      value = locatePromptOrValue as string;
+      value = locatePromptOrValue as string | number;
       locatePrompt = locatePromptOrOpt as TUserPrompt;
       opt = {
         ...optOrUndefined,
@@ -558,8 +558,8 @@ export class Agent<
     }
 
     assert(
-      typeof value === 'string',
-      'input value must be a string, use empty string if you want to clear the input',
+      typeof value === 'string' || typeof value === 'number',
+      'input value must be a string or number, use empty string if you want to clear the input',
     );
     assert(locatePrompt, 'missing locate prompt for input');
 
diff --git a/packages/core/src/device/index.ts b/packages/core/src/device/index.ts
@@ -155,7 +155,8 @@ export const defineActionHover = (
 // Input
 export const actionInputParamSchema = z.object({
   value: z
-    .string()
+    .union([z.string(), z.number()])
+    .transform((val) => String(val))
     .describe(
       'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',
     ),
diff --git a/packages/core/src/yaml/player.ts b/packages/core/src/yaml/player.ts
@@ -8,7 +8,7 @@ interface MidsceneYamlFlowItemAIInput extends LocateOption {
   // aiInput: string; // value to input
   // locate: TUserPrompt; // where to input
   aiInput: TUserPrompt | undefined; // where to input
-  value: string; // value to input
+  value: string | number; // value to input
 }
 
 interface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {
@@ -333,10 +333,10 @@ export class ScriptPlayer<T extends MidsceneYamlScriptEnv> {
 
         // Compatibility with previous version:
         // Old format: { aiInput: string (value), locate: TUserPrompt }
-        // New format - 1: { aiInput: TUserPrompt, value: string }
-        // New format - 2: { aiInput: undefined, locate: TUserPrompt, value: string }
+        // New format - 1: { aiInput: TUserPrompt, value: string | number }
+        // New format - 2: { aiInput: undefined, locate: TUserPrompt, value: string | number }
         let locatePrompt: TUserPrompt | undefined;
-        let value: string | undefined;
+        let value: string | number | undefined;
         if ((inputTask as any).locate) {
           // Old format - aiInput is the value, locate is the prompt
           value = (aiInput as string) || inputTask.value;
@@ -349,7 +349,7 @@ export class ScriptPlayer<T extends MidsceneYamlScriptEnv> {
 
         await agent.callActionInActionSpace('Input', {
           ...inputTask,
-          ...(value !== undefined ? { value } : {}),
+          ...(value !== undefined ? { value: String(value) } : {}),
           ...(locatePrompt
             ? { locate: buildDetailedLocateParam(locatePrompt, inputTask) }
             : {}),
diff --git a/packages/web-integration/tests/unit-test/ai-input-number-value.test.ts b/packages/web-integration/tests/unit-test/ai-input-number-value.test.ts
@@ -0,0 +1,230 @@
+import type { AbstractWebPage } from '@/web-page';
+import { Agent as PageAgent } from '@midscene/core/agent';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+declare const __VERSION__: string;
+
+// Mock only the necessary parts to avoid side effects
+vi.mock('@midscene/core/utils', async () => {
+  const actual = await vi.importActual('@midscene/core/utils');
+  return {
+    ...actual,
+    writeLogFile: vi.fn(() => null),
+    reportHTMLContent: vi.fn(() => ''),
+    stringifyDumpData: vi.fn(() => '{}'),
+    groupedActionDumpFileExt: '.json',
+    getVersion: () => __VERSION__,
+    sleep: vi.fn(() => Promise.resolve()),
+  };
+});
+
+vi.mock('@midscene/shared/logger', () => ({
+  getDebug: vi.fn(() => vi.fn()),
+  logMsg: vi.fn(),
+}));
+
+vi.mock('@midscene/core', async () => {
+  const actual = await vi.importActual('@midscene/core');
+  return {
+    ...actual,
+    Insight: vi.fn().mockImplementation(() => ({})),
+  };
+});
+
+// Partial mock for utils - only mock the async functions that need mocking
+vi.mock('@/common/utils', async () => {
+  const actual = await vi.importActual('@/common/utils');
+  return {
+    ...actual,
+    WebPageContextParser: vi.fn().mockResolvedValue({}),
+    trimContextByViewport: vi.fn((execution) => execution),
+    printReportMsg: vi.fn(),
+  };
+});
+
+// Mock page implementation
+const mockPage = {
+  interfaceType: 'puppeteer',
+  keyboard: {
+    type: vi.fn(),
+  },
+  screenshotBase64: vi.fn().mockResolvedValue('mock-screenshot'),
+  evaluateJavaScript: vi.fn(),
+  size: vi.fn().mockResolvedValue({ dpr: 1 }),
+  destroy: vi.fn(),
+} as unknown as AbstractWebPage;
+
+const mockedModelConfigFnResult = {
+  MIDSCENE_MODEL_NAME: 'mock-model',
+  MIDSCENE_OPENAI_API_KEY: 'mock-api-key',
+  MIDSCENE_OPENAI_BASE_URL: 'mock-base-url',
+};
+
+// Mock task executor
+const mockTaskExecutor = {
+  runPlans: vi.fn(),
+} as any;
+
+describe('PageAgent aiInput with number value', () => {
+  let agent: PageAgent;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+
+    // Create agent instance
+    agent = new PageAgent(mockPage, {
+      generateReport: false,
+      autoPrintReportMsg: false,
+      modelConfig: () => mockedModelConfigFnResult,
+    });
+
+    // Replace the taskExecutor with our mock
+    agent.taskExecutor = mockTaskExecutor;
+  });
+
+  it('should accept string value for aiInput', async () => {
+    const mockPlans = [
+      {
+        type: 'Locate' as const,
+        locate: {
+          prompt: 'input field',
+        },
+        param: {
+          prompt: 'input field',
+        },
+        thought: '',
+      },
+      {
+        type: 'Input' as const,
+        locate: {
+          prompt: 'input field',
+        },
+        param: {
+          value: 'test string',
+        },
+        thought: '',
+      },
+    ];
+
+    const mockExecutorResult = {
+      executor: {
+        dump: () => ({ name: 'test', tasks: [] }),
+        isInErrorState: () => false,
+      },
+      output: {},
+    };
+
+    mockTaskExecutor.runPlans.mockResolvedValue(mockExecutorResult);
+
+    // Call aiInput with string value (should work as before)
+    await expect(
+      agent.aiInput('input field', { value: 'test string' }),
+    ).resolves.not.toThrow();
+  });
+
+  it('should accept number value for aiInput', async () => {
+    const mockPlans = [
+      {
+        type: 'Locate' as const,
+        locate: {
+          prompt: 'input field',
+        },
+        param: {
+          prompt: 'input field',
+        },
+        thought: '',
+      },
+      {
+        type: 'Input' as const,
+        locate: {
+          prompt: 'input field',
+        },
+        param: {
+          value: '123456',
+        },
+        thought: '',
+      },
+    ];
+
+    const mockExecutorResult = {
+      executor: {
+        dump: () => ({ name: 'test', tasks: [] }),
+        isInErrorState: () => false,
+      },
+      output: {},
+    };
+
+    mockTaskExecutor.runPlans.mockResolvedValue(mockExecutorResult);
+
+    // Call aiInput with number value (should not throw error)
+    await expect(
+      agent.aiInput('input field', { value: 123456 }),
+    ).resolves.not.toThrow();
+  });
+
+  it('should accept integer zero for aiInput', async () => {
+    const mockExecutorResult = {
+      executor: {
+        dump: () => ({ name: 'test', tasks: [] }),
+        isInErrorState: () => false,
+      },
+      output: {},
+    };
+
+    mockTaskExecutor.runPlans.mockResolvedValue(mockExecutorResult);
+
+    // Call aiInput with number value 0 (should not throw error)
+    await expect(
+      agent.aiInput('input field', { value: 0 }),
+    ).resolves.not.toThrow();
+  });
+
+  it('should accept negative number for aiInput', async () => {
+    const mockExecutorResult = {
+      executor: {
+        dump: () => ({ name: 'test', tasks: [] }),
+        isInErrorState: () => false,
+      },
+      output: {},
+    };
+
+    mockTaskExecutor.runPlans.mockResolvedValue(mockExecutorResult);
+
+    // Call aiInput with negative number value (should not throw error)
+    await expect(
+      agent.aiInput('input field', { value: -999 }),
+    ).resolves.not.toThrow();
+  });
+
+  it('should accept decimal number for aiInput', async () => {
+    const mockExecutorResult = {
+      executor: {
+        dump: () => ({ name: 'test', tasks: [] }),
+        isInErrorState: () => false,
+      },
+      output: {},
+    };
+
+    mockTaskExecutor.runPlans.mockResolvedValue(mockExecutorResult);
+
+    // Call aiInput with decimal number value (should not throw error)
+    await expect(
+      agent.aiInput('input field', { value: 3.14 }),
+    ).resolves.not.toThrow();
+  });
+
+  it('should use legacy aiInput(value, locatePrompt) signature with number', async () => {
+    const mockExecutorResult = {
+      executor: {
+        dump: () => ({ name: 'test', tasks: [] }),
+        isInErrorState: () => false,
+      },
+      output: {},
+    };
+
+    mockTaskExecutor.runPlans.mockResolvedValue(mockExecutorResult);
+
+    // Test legacy signature: aiInput(value, locatePrompt) with number value
+    await expect(agent.aiInput(88888, 'input field')).resolves.not.toThrow();
+  });
+});
diff --git a/packages/web-integration/tests/unit-test/yaml/player.test.ts b/packages/web-integration/tests/unit-test/yaml/player.test.ts
