From 0fae5ae7ca58e3e11ba31d25aa07a282de1808ee Mon Sep 17 00:00:00 2001 From: Kazuhiro Sera Date: Fri, 21 Nov 2025 13:56:01 +0900 Subject: [PATCH] feat: #678 Add a list of per-request usage data to Usage --- .changeset/three-islands-pump.md | 7 + packages/agents-core/src/index.ts | 2 +- packages/agents-core/src/runState.ts | 40 ++++- packages/agents-core/src/types/protocol.ts | 25 ++- packages/agents-core/src/usage.ts | 155 ++++++++++++++++-- packages/agents-core/test/run.test.ts | 34 ++++ packages/agents-core/test/usage.test.ts | 98 ++++++++++- packages/agents-extensions/test/aiSdk.test.ts | 1 + .../src/openaiChatCompletionsModel.ts | 12 +- 9 files changed, 345 insertions(+), 29 deletions(-) create mode 100644 .changeset/three-islands-pump.md diff --git a/.changeset/three-islands-pump.md b/.changeset/three-islands-pump.md new file mode 100644 index 00000000..077b47f6 --- /dev/null +++ b/.changeset/three-islands-pump.md @@ -0,0 +1,7 @@ +--- +'@openai/agents-extensions': patch +'@openai/agents-openai': patch +'@openai/agents-core': patch +--- + +feat: #678 Add a list of per-request usage data to Usage diff --git a/packages/agents-core/src/index.ts b/packages/agents-core/src/index.ts index 7a1dc10f..e43cda52 100644 --- a/packages/agents-core/src/index.ts +++ b/packages/agents-core/src/index.ts @@ -176,7 +176,7 @@ export type { StreamEventResponseStarted, StreamEventGenericItem, } from './types'; -export { Usage } from './usage'; +export { RequestUsage, Usage } from './usage'; export type { Session, SessionInputCallback } from './memory/session'; export { MemorySession } from './memory/memorySession'; diff --git a/packages/agents-core/src/runState.ts b/packages/agents-core/src/runState.ts index aee534c3..7627197c 100644 --- a/packages/agents-core/src/runState.ts +++ b/packages/agents-core/src/runState.ts @@ -72,11 +72,22 @@ const SerializedSpan: z.ZodType = serializedSpanBase.extend( }, ); +const requestUsageSchema = z.object({ + inputTokens: z.number(), + outputTokens: z.number(), + totalTokens: z.number(), + inputTokensDetails: z.record(z.string(), z.number()).optional(), + outputTokensDetails: z.record(z.string(), z.number()).optional(), +}); + const usageSchema = z.object({ requests: z.number(), inputTokens: z.number(), outputTokens: z.number(), totalTokens: z.number(), + inputTokensDetails: z.array(z.record(z.string(), z.number())).optional(), + outputTokensDetails: z.array(z.record(z.string(), z.number())).optional(), + requestUsageEntries: z.array(requestUsageSchema).optional(), }); const modelResponseSchema = z.object({ @@ -287,6 +298,13 @@ export class RunState> { * Run context tracking approvals, usage, and other metadata. */ public _context: RunContext; + + /** + * The usage aggregated for this run. This includes per-request breakdowns when available. + */ + get usage(): Usage { + return this._context.usage; + } /** * Tracks what tools each agent has used. */ @@ -440,6 +458,22 @@ export class RunState> { inputTokens: response.usage.inputTokens, outputTokens: response.usage.outputTokens, totalTokens: response.usage.totalTokens, + inputTokensDetails: response.usage.inputTokensDetails, + outputTokensDetails: response.usage.outputTokensDetails, + ...(response.usage.requestUsageEntries && + response.usage.requestUsageEntries.length > 0 + ? { + requestUsageEntries: response.usage.requestUsageEntries.map( + (entry) => ({ + inputTokens: entry.inputTokens, + outputTokens: entry.outputTokens, + totalTokens: entry.totalTokens, + inputTokensDetails: entry.inputTokensDetails, + outputTokensDetails: entry.outputTokensDetails, + }), + ), + } + : {}), }, output: response.output as any, responseId: response.responseId, @@ -683,11 +717,7 @@ export function deserializeSpan( export function deserializeModelResponse( serializedModelResponse: z.infer, ): ModelResponse { - const usage = new Usage(); - usage.requests = serializedModelResponse.usage.requests; - usage.inputTokens = serializedModelResponse.usage.inputTokens; - usage.outputTokens = serializedModelResponse.usage.outputTokens; - usage.totalTokens = serializedModelResponse.usage.totalTokens; + const usage = new Usage(serializedModelResponse.usage); return { usage, diff --git a/packages/agents-core/src/types/protocol.ts b/packages/agents-core/src/types/protocol.ts index e7131749..8d292b97 100644 --- a/packages/agents-core/src/types/protocol.ts +++ b/packages/agents-core/src/types/protocol.ts @@ -743,8 +743,7 @@ export type ModelItem = z.infer; // Meta data types // ---------------------------- -export const UsageData = z.object({ - requests: z.number().optional(), +export const RequestUsageData = z.object({ inputTokens: z.number(), outputTokens: z.number(), totalTokens: z.number(), @@ -752,6 +751,28 @@ export const UsageData = z.object({ outputTokensDetails: z.record(z.string(), z.number()).optional(), }); +export type RequestUsageData = z.infer; + +export const UsageData = z.object({ + requests: z.number().optional(), + inputTokens: z.number(), + outputTokens: z.number(), + totalTokens: z.number(), + inputTokensDetails: z + .union([ + z.record(z.string(), z.number()), + z.array(z.record(z.string(), z.number())), + ]) + .optional(), + outputTokensDetails: z + .union([ + z.record(z.string(), z.number()), + z.array(z.record(z.string(), z.number())), + ]) + .optional(), + requestUsageEntries: z.array(RequestUsageData).optional(), +}); + export type UsageData = z.infer; // ---------------------------- diff --git a/packages/agents-core/src/usage.ts b/packages/agents-core/src/usage.ts index e9894d2c..a72fc05b 100644 --- a/packages/agents-core/src/usage.ts +++ b/packages/agents-core/src/usage.ts @@ -1,14 +1,80 @@ -import { UsageData } from './types/protocol'; +import { RequestUsageData, UsageData } from './types/protocol'; -type UsageInput = Partial< - UsageData & { +type RequestUsageInput = Partial< + RequestUsageData & { input_tokens: number; output_tokens: number; total_tokens: number; input_tokens_details: object; output_tokens_details: object; } -> & { requests?: number }; +>; + +type UsageInput = Partial< + UsageData & { + input_tokens: number; + output_tokens: number; + total_tokens: number; + input_tokens_details: + | Record + | Array> + | object; + output_tokens_details: + | Record + | Array> + | object; + request_usage_entries: RequestUsageInput[]; + } +> & { requests?: number; requestUsageEntries?: RequestUsageInput[] }; + +/** + * Usage details for a single API request. + */ +export class RequestUsage { + /** + * The number of input tokens used for this request. + */ + public inputTokens: number; + + /** + * The number of output tokens used for this request. + */ + public outputTokens: number; + + /** + * The total number of tokens sent and received for this request. + */ + public totalTokens: number; + + /** + * Details about the input tokens used for this request. + */ + public inputTokensDetails: Record; + + /** + * Details about the output tokens used for this request. + */ + public outputTokensDetails: Record; + + constructor(input?: RequestUsageInput) { + this.inputTokens = input?.inputTokens ?? input?.input_tokens ?? 0; + this.outputTokens = input?.outputTokens ?? input?.output_tokens ?? 0; + this.totalTokens = + input?.totalTokens ?? + input?.total_tokens ?? + this.inputTokens + this.outputTokens; + const inputTokensDetails = + input?.inputTokensDetails ?? input?.input_tokens_details; + this.inputTokensDetails = inputTokensDetails + ? (inputTokensDetails as Record) + : {}; + const outputTokensDetails = + input?.outputTokensDetails ?? input?.output_tokens_details; + this.outputTokensDetails = outputTokensDetails + ? (outputTokensDetails as Record) + : {}; + } +} /** * Tracks token usage and request counts for an agent run. @@ -44,6 +110,11 @@ export class Usage { */ public outputTokensDetails: Array> = []; + /** + * List of per-request usage entries for detailed cost calculations. + */ + public requestUsageEntries: RequestUsage[] | undefined; + constructor(input?: UsageInput) { if (typeof input === 'undefined') { this.requests = 0; @@ -52,29 +123,58 @@ export class Usage { this.totalTokens = 0; this.inputTokensDetails = []; this.outputTokensDetails = []; + this.requestUsageEntries = undefined; } else { this.requests = input?.requests ?? 1; this.inputTokens = input?.inputTokens ?? input?.input_tokens ?? 0; this.outputTokens = input?.outputTokens ?? input?.output_tokens ?? 0; - this.totalTokens = input?.totalTokens ?? input?.total_tokens ?? 0; + this.totalTokens = + input?.totalTokens ?? + input?.total_tokens ?? + this.inputTokens + this.outputTokens; const inputTokensDetails = input?.inputTokensDetails ?? input?.input_tokens_details; - this.inputTokensDetails = inputTokensDetails - ? [inputTokensDetails as Record] - : []; + if (Array.isArray(inputTokensDetails)) { + this.inputTokensDetails = inputTokensDetails as Array< + Record + >; + } else { + this.inputTokensDetails = inputTokensDetails + ? [inputTokensDetails as Record] + : []; + } const outputTokensDetails = input?.outputTokensDetails ?? input?.output_tokens_details; - this.outputTokensDetails = outputTokensDetails - ? [outputTokensDetails as Record] - : []; + if (Array.isArray(outputTokensDetails)) { + this.outputTokensDetails = outputTokensDetails as Array< + Record + >; + } else { + this.outputTokensDetails = outputTokensDetails + ? [outputTokensDetails as Record] + : []; + } + + const requestUsageEntries = + input?.requestUsageEntries ?? input?.request_usage_entries; + const normalizedRequestUsageEntries = Array.isArray(requestUsageEntries) + ? requestUsageEntries.map((entry) => + entry instanceof RequestUsage ? entry : new RequestUsage(entry), + ) + : undefined; + this.requestUsageEntries = + normalizedRequestUsageEntries && + normalizedRequestUsageEntries.length > 0 + ? normalizedRequestUsageEntries + : undefined; } } add(newUsage: Usage) { - this.requests += newUsage.requests; - this.inputTokens += newUsage.inputTokens; - this.outputTokens += newUsage.outputTokens; - this.totalTokens += newUsage.totalTokens; + this.requests += newUsage.requests ?? 0; + this.inputTokens += newUsage.inputTokens ?? 0; + this.outputTokens += newUsage.outputTokens ?? 0; + this.totalTokens += newUsage.totalTokens ?? 0; if (newUsage.inputTokensDetails) { // The type does not allow undefined, but it could happen runtime this.inputTokensDetails.push(...newUsage.inputTokensDetails); @@ -83,7 +183,30 @@ export class Usage { // The type does not allow undefined, but it could happen runtime this.outputTokensDetails.push(...newUsage.outputTokensDetails); } + + if ( + Array.isArray(newUsage.requestUsageEntries) && + newUsage.requestUsageEntries.length > 0 + ) { + this.requestUsageEntries ??= []; + this.requestUsageEntries.push( + ...newUsage.requestUsageEntries.map((entry) => + entry instanceof RequestUsage ? entry : new RequestUsage(entry), + ), + ); + } else if (newUsage.requests === 1 && newUsage.totalTokens > 0) { + this.requestUsageEntries ??= []; + this.requestUsageEntries.push( + new RequestUsage({ + inputTokens: newUsage.inputTokens, + outputTokens: newUsage.outputTokens, + totalTokens: newUsage.totalTokens, + inputTokensDetails: newUsage.inputTokensDetails?.[0], + outputTokensDetails: newUsage.outputTokensDetails?.[0], + }), + ); + } } } -export { UsageData }; +export { RequestUsageData, UsageData }; diff --git a/packages/agents-core/test/run.test.ts b/packages/agents-core/test/run.test.ts index 51eb6d2e..31b75125 100644 --- a/packages/agents-core/test/run.test.ts +++ b/packages/agents-core/test/run.test.ts @@ -88,6 +88,40 @@ describe('Runner.run', () => { expectTypeOf(result.finalOutput).toEqualTypeOf(); }); + it('exposes aggregated usage on run results', async () => { + const model = new FakeModel([ + { + output: [fakeModelMessage('hi there')], + usage: new Usage({ + requests: 1, + inputTokens: 2, + outputTokens: 3, + totalTokens: 5, + }), + responseId: 'usage-res', + }, + ]); + const agent = new Agent({ + name: 'UsageAgent', + model, + }); + + const result = await run(agent, 'ping'); + + expect(result.state.usage.inputTokens).toBe(2); + expect(result.state.usage.outputTokens).toBe(3); + expect(result.state.usage.totalTokens).toBe(5); + expect(result.state.usage.requestUsageEntries).toEqual([ + { + inputTokens: 2, + outputTokens: 3, + totalTokens: 5, + inputTokensDetails: {}, + outputTokensDetails: {}, + }, + ]); + }); + it('sholuld handle structured output', async () => { const fakeModel = new FakeModel([ { diff --git a/packages/agents-core/test/usage.test.ts b/packages/agents-core/test/usage.test.ts index dc215a93..f3a1b001 100644 --- a/packages/agents-core/test/usage.test.ts +++ b/packages/agents-core/test/usage.test.ts @@ -1,6 +1,6 @@ -import { describe, it, expect } from 'vitest'; +import { describe, expect, it } from 'vitest'; -import { Usage } from '../src/usage'; +import { RequestUsage, Usage } from '../src/usage'; describe('Usage', () => { it('initialises with default values', () => { @@ -10,9 +10,10 @@ describe('Usage', () => { expect(usage.inputTokens).toBe(0); expect(usage.outputTokens).toBe(0); expect(usage.totalTokens).toBe(0); + expect(usage.requestUsageEntries).toBeUndefined(); }); - it('can be constructed from a ResponseUsage‑like object', () => { + it('can be constructed from a ResponseUsage-like object', () => { const usage = new Usage({ requests: 3, inputTokens: 10, @@ -24,6 +25,7 @@ describe('Usage', () => { expect(usage.inputTokens).toBe(10); expect(usage.outputTokens).toBe(5); expect(usage.totalTokens).toBe(15); + expect(usage.requestUsageEntries).toBeUndefined(); }); it('falls back to snake_case fields', () => { @@ -34,6 +36,15 @@ describe('Usage', () => { total_tokens: 10, input_tokens_details: { foo: 1 }, output_tokens_details: { bar: 2 }, + request_usage_entries: [ + { + input_tokens: 7, + output_tokens: 3, + total_tokens: 10, + input_tokens_details: { foo: 1 }, + output_tokens_details: { bar: 2 }, + }, + ], }); expect(usage.requests).toBe(2); @@ -42,6 +53,15 @@ describe('Usage', () => { expect(usage.totalTokens).toBe(10); expect(usage.inputTokensDetails).toEqual([{ foo: 1 }]); expect(usage.outputTokensDetails).toEqual([{ bar: 2 }]); + expect(usage.requestUsageEntries).toEqual([ + new RequestUsage({ + inputTokens: 7, + outputTokens: 3, + totalTokens: 10, + inputTokensDetails: { foo: 1 }, + outputTokensDetails: { bar: 2 }, + }), + ]); }); it('adds other Usage instances correctly', () => { @@ -63,6 +83,7 @@ describe('Usage', () => { expect(usageA.inputTokens).toBe(4); // 1 + 3 expect(usageA.outputTokens).toBe(5); // 1 + 4 expect(usageA.totalTokens).toBe(9); // 2 + 7 + expect(usageA.requestUsageEntries).toBeUndefined(); }); it('the add method accepts an empty object', () => { @@ -70,5 +91,76 @@ describe('Usage', () => { usage.add({} as Usage); expect(usage.inputTokensDetails).toEqual([]); expect(usage.outputTokensDetails).toEqual([]); + expect(usage.requestUsageEntries).toBeUndefined(); + expect(usage.requests).toBe(1); + }); + + it('adds a request usage entry for single request usage', () => { + const aggregated = new Usage(); + aggregated.add( + new Usage({ + requests: 1, + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + inputTokensDetails: { cached_tokens: 2 }, + outputTokensDetails: { reasoning_tokens: 3 }, + }), + ); + + expect(aggregated.requestUsageEntries).toEqual([ + { + inputTokens: 10, + outputTokens: 5, + totalTokens: 15, + inputTokensDetails: { cached_tokens: 2 }, + outputTokensDetails: { reasoning_tokens: 3 }, + }, + ]); + }); + + it('ignores zero-token single requests when tracking request usage', () => { + const aggregated = new Usage(); + aggregated.add( + new Usage({ + requests: 1, + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + inputTokensDetails: { cached_tokens: 0 }, + outputTokensDetails: { reasoning_tokens: 0 }, + }), + ); + + expect(aggregated.requestUsageEntries).toBeUndefined(); + }); + + it('merges existing request usage entries when present', () => { + const aggregated = new Usage(); + const withEntries = new Usage({ + requests: 1, + inputTokens: 5, + outputTokens: 6, + totalTokens: 11, + requestUsageEntries: [ + new RequestUsage({ + inputTokens: 5, + outputTokens: 6, + totalTokens: 11, + }), + ], + }); + + aggregated.add(withEntries); + + expect(aggregated.requestUsageEntries).toEqual([ + { + inputTokens: 5, + outputTokens: 6, + totalTokens: 11, + inputTokensDetails: {}, + outputTokensDetails: {}, + }, + ]); }); }); diff --git a/packages/agents-extensions/test/aiSdk.test.ts b/packages/agents-extensions/test/aiSdk.test.ts index 2d26a4f3..897517ea 100644 --- a/packages/agents-extensions/test/aiSdk.test.ts +++ b/packages/agents-extensions/test/aiSdk.test.ts @@ -842,6 +842,7 @@ describe('AiSdkModel.getResponse', () => { totalTokens: 0, inputTokensDetails: [], outputTokensDetails: [], + requestUsageEntries: undefined, }); }); }); diff --git a/packages/agents-openai/src/openaiChatCompletionsModel.ts b/packages/agents-openai/src/openaiChatCompletionsModel.ts index f445a4d0..34d77fcc 100644 --- a/packages/agents-openai/src/openaiChatCompletionsModel.ts +++ b/packages/agents-openai/src/openaiChatCompletionsModel.ts @@ -222,8 +222,16 @@ export class OpenAIChatCompletionsModel implements Model { prompt_tokens: event.response.usage.inputTokens, completion_tokens: event.response.usage.outputTokens, total_tokens: event.response.usage.totalTokens, - prompt_tokens_details: event.response.usage.inputTokensDetails, - completion_tokens_details: event.response.usage.outputTokensDetails, + prompt_tokens_details: Array.isArray( + event.response.usage.inputTokensDetails, + ) + ? event.response.usage.inputTokensDetails[0] + : event.response.usage.inputTokensDetails, + completion_tokens_details: Array.isArray( + event.response.usage.outputTokensDetails, + ) + ? event.response.usage.outputTokensDetails[0] + : event.response.usage.outputTokensDetails, }; } yield event;