diff --git a/instrumentation/openai/.rubocop.yml b/instrumentation/openai/.rubocop.yml new file mode 100644 index 000000000..1248a2f82 --- /dev/null +++ b/instrumentation/openai/.rubocop.yml @@ -0,0 +1 @@ +inherit_from: ../../.rubocop.yml diff --git a/instrumentation/openai/.yardopts b/instrumentation/openai/.yardopts new file mode 100644 index 000000000..65b6490ff --- /dev/null +++ b/instrumentation/openai/.yardopts @@ -0,0 +1,9 @@ +--no-private +--title=OpenTelemetry OpenAI Instrumentation +--markup=markdown +--main=README.md +./lib/opentelemetry/instrumentation/**/*.rb +./lib/opentelemetry/instrumentation.rb +- +README.md +CHANGELOG.md diff --git a/instrumentation/openai/Appraisals b/instrumentation/openai/Appraisals new file mode 100644 index 000000000..8c24e16b1 --- /dev/null +++ b/instrumentation/openai/Appraisals @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +## TODO: Include the supported version to be tested here. +## Example: +# appraise 'rack-2.1' do +# gem 'rack', '~> 2.1.2' +# end + +# appraise 'rack-2.0' do +# gem 'rack', '2.0.8' +# end diff --git a/instrumentation/openai/CHANGELOG.md b/instrumentation/openai/CHANGELOG.md new file mode 100644 index 000000000..9cea593b2 --- /dev/null +++ b/instrumentation/openai/CHANGELOG.md @@ -0,0 +1 @@ +# Release History: opentelemetry-instrumentation-openai diff --git a/instrumentation/openai/Gemfile b/instrumentation/openai/Gemfile new file mode 100644 index 000000000..5e5c469a7 --- /dev/null +++ b/instrumentation/openai/Gemfile @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +source 'https://rubygems.org' + +gemspec + +group :test do + gem 'opentelemetry-instrumentation-base', path: '../base' + gem 'appraisal', '~> 2.5' + gem 'bundler', '~> 2.4' + gem 'minitest', '~> 5.0' + gem 'openai', '~> 0.36.0' + gem 'opentelemetry-sdk', '~> 1.0' + gem 'opentelemetry-test-helpers', '~> 0.3' + gem 'rake', '~> 13.0' + gem 'rubocop', '~> 1.81.1' + gem 'rubocop-performance', '~> 1.26.0' + gem 'simplecov', '~> 0.17.1' + gem 'webmock', '~> 3.24' + gem 'yard', '~> 0.9' +end diff --git a/instrumentation/openai/LICENSE b/instrumentation/openai/LICENSE new file mode 100644 index 000000000..1ef7dad2c --- /dev/null +++ b/instrumentation/openai/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright The OpenTelemetry Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/instrumentation/openai/README.md b/instrumentation/openai/README.md new file mode 100644 index 000000000..4512ac649 --- /dev/null +++ b/instrumentation/openai/README.md @@ -0,0 +1,49 @@ +# OpenTelemetry OpenAI Instrumentation + +Todo: Add a description. + +## How do I get started? + +Install the gem using: + +```console +gem install opentelemetry-instrumentation-openai +``` + +Or, if you use [bundler][bundler-home], include `opentelemetry-instrumentation-openai` in your `Gemfile`. + +## Usage + +To use the instrumentation, call `use` with the name of the instrumentation: + +```ruby +OpenTelemetry::SDK.configure do |c| + c.use 'OpenTelemetry::Instrumentation::OpenAI' +end +``` + +Alternatively, you can also call `use_all` to install all the available instrumentation. + +```ruby +OpenTelemetry::SDK.configure do |c| + c.use_all +end +``` + +## How can I get involved? + +The `opentelemetry-instrumentation-openai` gem source is [on github][repo-github], along with related gems including `opentelemetry-api` and `opentelemetry-sdk`. + +The OpenTelemetry Ruby gems are maintained by the OpenTelemetry Ruby special interest group (SIG). You can get involved by joining us on our [GitHub Discussions][discussions-url], [Slack Channel][slack-channel] or attending our weekly meeting. See the [meeting calendar][community-meetings] for dates and times. For more information on this and other language SIGs, see the OpenTelemetry [community page][ruby-sig]. + +## License + +The `opentelemetry-instrumentation-openai` gem is distributed under the Apache 2.0 license. See [LICENSE][license-github] for more information. + +[bundler-home]: https://bundler.io +[repo-github]: https://github.com/open-telemetry/opentelemetry-ruby +[license-github]: https://github.com/open-telemetry/opentelemetry-ruby-contrib/blob/main/LICENSE +[ruby-sig]: https://github.com/open-telemetry/community#ruby-sig +[community-meetings]: https://github.com/open-telemetry/community#community-meetings +[slack-channel]: https://cloud-native.slack.com/archives/C01NWKKMKMY +[discussions-url]: https://github.com/open-telemetry/opentelemetry-ruby/discussions diff --git a/instrumentation/openai/Rakefile b/instrumentation/openai/Rakefile new file mode 100644 index 000000000..1a64ba842 --- /dev/null +++ b/instrumentation/openai/Rakefile @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'bundler/gem_tasks' +require 'rake/testtask' +require 'yard' +require 'rubocop/rake_task' + +RuboCop::RakeTask.new + +Rake::TestTask.new :test do |t| + t.libs << 'test' + t.libs << 'lib' + t.test_files = FileList['test/**/*_test.rb'] +end + +YARD::Rake::YardocTask.new do |t| + t.stats_options = ['--list-undoc'] +end + +if RUBY_ENGINE == 'truffleruby' + task default: %i[test] +else + task default: %i[test rubocop yard] +end diff --git a/instrumentation/openai/lib/opentelemetry-instrumentation-openai.rb b/instrumentation/openai/lib/opentelemetry-instrumentation-openai.rb new file mode 100644 index 000000000..c034f140f --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry-instrumentation-openai.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'opentelemetry/instrumentation' diff --git a/instrumentation/openai/lib/opentelemetry/instrumentation.rb b/instrumentation/openai/lib/opentelemetry/instrumentation.rb new file mode 100644 index 000000000..3d25f83a3 --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry/instrumentation.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# OpenTelemetry is an open source observability framework, providing a +# general-purpose API, SDK, and related tools required for the instrumentation +# of cloud-native software, frameworks, and libraries. +# +# The OpenTelemetry module provides global accessors for telemetry objects. +# See the documentation for the `opentelemetry-api` gem for details. +module OpenTelemetry + # Instrumentation should be able to handle the case when the library is not installed on a user's system. + module Instrumentation + end +end + +require_relative 'instrumentation/openai' diff --git a/instrumentation/openai/lib/opentelemetry/instrumentation/openai.rb b/instrumentation/openai/lib/opentelemetry/instrumentation/openai.rb new file mode 100644 index 000000000..8947585df --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry/instrumentation/openai.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'opentelemetry' +require 'opentelemetry-instrumentation-base' + +module OpenTelemetry + module Instrumentation + # Contains the OpenTelemetry instrumentation for the openai gem + module OpenAI + end + end +end + +require_relative 'openai/instrumentation' +require_relative 'openai/version' diff --git a/instrumentation/openai/lib/opentelemetry/instrumentation/openai/instrumentation.rb b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/instrumentation.rb new file mode 100644 index 000000000..97fe1b3c3 --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/instrumentation.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Instrumentation + module OpenAI + # The Instrumentation class contains logic to detect and install the openai instrumentation + class Instrumentation < OpenTelemetry::Instrumentation::Base + MINIMUM_VERSION = Gem::Version.new('0.35.2') + ALLOWED_OPERATION = %w[chat completions embeddings].freeze + + install do |_config| + require_dependencies + determine_the_content_mode + patch_client + end + + present do + defined?(::OpenAI) + end + + compatible do + gem_version >= MINIMUM_VERSION + end + + option :capture_content, default: false, validate: :boolean + option :allowed_operation, default: ALLOWED_OPERATION, validate: :array + + private + + def gem_version + ::OpenAI::VERSION + end + + def determine_the_content_mode + should_capture_content = ENV['OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT'].to_s.downcase == 'true' + config[:capture_content] = should_capture_content + end + + def require_dependencies + require_relative 'patches/client' + end + + def patch_client + ::OpenAI::Client.prepend(Patches::Client) + end + end + end + end +end diff --git a/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/client.rb b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/client.rb new file mode 100644 index 000000000..b43404605 --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/client.rb @@ -0,0 +1,261 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'operation_name' +require_relative 'stream_wrapper' +require_relative 'utils' + +module OpenTelemetry + module Instrumentation + module OpenAI + module Patches + # OpenAIClient Patch + # rubocop:disable Metrics/ModuleLength + module Client + include OperationName + include Utils + + def request(req) + operation_name = determine_operation_name(req) + + # Only instrument implemented/tested OpenAI operation + return super unless config[:allowed_operation].include? operation_name + + model = (req[:body][:model] || req[:body]['model']).to_s if req[:body].is_a? Hash + span_name = model.empty? ? operation_name : "#{operation_name} #{model}" + attributes = extract_request_attributes(req, operation_name, model) + + # For streaming, start span manually so it stays open during iteration + # Stream mode return OpenAI::Internal::Stream[OpenAI::Models::Chat::ChatCompletionChunk] + if req[:stream] + span = tracer.start_span(span_name, attributes: attributes, kind: :client) + log_request_content(span, req) if config[:capture_content] + + response = super + return StreamWrapper.new(response, span, config[:capture_content]) + end + + # Non-streaming path + tracer.in_span( + span_name, + attributes: attributes, + kind: :client + ) do |span| + # Log request details if content capture is enabled and + log_request_content(span, req) if config[:capture_content] + + response = super + handle_response(span, response, req) + + response + rescue StandardError => e + handle_span_exception(span, e) + raise + end + end + + private + + def tracer + OpenAI::Instrumentation.instance.tracer + end + + def config + OpenAI::Instrumentation.instance.config + end + + # Extract comprehensive request attributes following semantic conventions + def extract_request_attributes(req, operation_name, model) + uri = begin + URI.parse(req[:url]) + rescue StandardError + nil + end + + request_attributes = { + 'gen_ai.operation.name' => operation_name, + 'gen_ai.provider.name' => 'openai', + 'gen_ai.request.model' => model, + 'server.address' => uri&.host || 'api.openai.com', + 'server.port' => uri&.port || 443, + 'http.request.method' => req[:method].to_s.upcase, + 'url.path' => req[:path], + 'gen_ai.output.type' => get_output_type(operation_name) + }.compact + + # Extract attributes from request body based on operation name + merge_body_attributes!(request_attributes, req[:body], operation_name) + + request_attributes + end + + # Since only chat and embedding is allowed, so we will only expect 'text' as output + def get_output_type(operation_name) + case operation_name + when 'chat' + 'text' + when 'images.generate', 'images.edit', 'images.variation' + 'image' + when 'audio.transcription', 'audio.translation', 'audio.speech' + 'speech' + else + 'json' + end + end + + # Merge body attributes based on operation type + def merge_body_attributes!(attributes, body, operation_name) + return unless body.is_a?(Hash) + + case operation_name + when 'chat', 'completions' + merge_chat_attributes!(attributes, body) + when 'embeddings' + merge_embeddings_attributes!(attributes, body) + end + end + + # Merge chat/completion specific attributes + def merge_chat_attributes!(attributes, body) + n_count = body[:n] + stop_sequences = if body[:stop].is_a?(Array) + body[:stop] + else + (body[:stop] ? [body[:stop]] : nil) + end + service_tier = body[:service_tier]&.to_s + + chat_attributes = { + 'gen_ai.request.temperature' => body[:temperature], + 'gen_ai.request.max_tokens' => body[:max_tokens] || body[:max_completion_tokens], + 'gen_ai.request.top_p' => body[:top_p], + 'gen_ai.request.frequency_penalty' => body[:frequency_penalty], + 'gen_ai.request.presence_penalty' => body[:presence_penalty], + 'gen_ai.request.seed' => body[:seed], + 'gen_ai.request.stop_sequences' => stop_sequences, + 'gen_ai.request.choice.count' => n_count && n_count != 1 ? n_count : nil, + 'openai.request.service_tier' => service_tier && service_tier != 'auto' ? service_tier : nil + }.compact + + attributes.merge!(chat_attributes) + end + + # Merge embeddings specific attributes + def merge_embeddings_attributes!(attributes, body) + encoding_formats = body[:encoding_format] ? [body[:encoding_format].to_s] : nil + + embeddings_attributes = { + 'gen_ai.request.encoding_formats' => encoding_formats + }.compact + + attributes.merge!(embeddings_attributes) + end + + # Log request content for debugging/monitoring + def log_request_content(span, req) + body = req[:body] + return unless body.is_a?(Hash) + + if body[:messages].is_a?(Array) + body[:messages].each do |message| + event = message_to_log_event(message, capture_content: true) + log_structured_event(event) + end + end + + if body[:input] + input_text = body[:input].is_a?(Array) ? body[:input].join(', ') : body[:input].to_s + event = { + event_name: 'gen_ai.user.message', + attributes: { 'gen_ai.provider.name' => 'openai' }, + body: { content: input_text } + } + log_structured_event(event) + end + + return unless body[:prompt] + + prompt_text = body[:prompt].is_a?(Array) ? body[:prompt].join(', ') : body[:prompt].to_s + event = { + event_name: 'gen_ai.user.message', + attributes: { 'gen_ai.provider.name' => 'openai' }, + body: { content: prompt_text } + } + log_structured_event(event) + end + + # Handle different response types and extract telemetry data + def handle_response(span, result, req) + return unless span.recording? + + # Set basic response attributes (only for non-streaming responses with these methods) + response_attributes = { + 'gen_ai.response.model' => result.respond_to?(:model) ? result.model : nil, + 'gen_ai.response.id' => result.respond_to?(:id) ? result.id : nil, + 'openai.response.service_tier' => result.respond_to?(:service_tier) ? result.service_tier&.to_s : nil, + 'openai.response.system_fingerprint' => result.respond_to?(:system_fingerprint) ? result.system_fingerprint : nil + }.compact + span.add_attributes(response_attributes) + + # Handle usage/token information + set_usage_attributes(span, result.usage) if result.respond_to?(:usage) && result.usage + + # Handle different completion responses + if result.respond_to?(:choices) && result.choices&.any? + handle_chat_completion_response(span, result) + elsif result.respond_to?(:data) && result.data&.any? + handle_embeddings_response(span, result) if result.data.first.respond_to?(:embedding) + end + end + + # Handle chat completion response + def handle_chat_completion_response(span, result) + finish_reasons = result.choices.map { |x| x.finish_reason.to_s } + span.set_attribute('gen_ai.response.finish_reasons', finish_reasons) if finish_reasons.any? + + return unless config[:capture_content] + + result.choices.each do |choice| + event = choice_to_log_event(choice, capture_content: true) + log_structured_event(event) + end + end + + # Handle embeddings response + def handle_embeddings_response(span, result) + embedding_dimensions = result.data.first.respond_to?(:embedding) ? result.data.first.embedding&.size : nil + + attributes = { + 'gen_ai.embeddings.dimension.count' => embedding_dimensions + }.compact + + span.add_attributes(attributes) + end + + # Set token usage attributes + def set_usage_attributes(span, usage) + usage_attributes = { + 'gen_ai.usage.input_tokens' => usage.respond_to?(:prompt_tokens) ? usage.prompt_tokens : nil, + 'gen_ai.usage.output_tokens' => usage.respond_to?(:completion_tokens) ? usage.completion_tokens : nil, + 'gen_ai.usage.total_tokens' => usage.respond_to?(:total_tokens) ? usage.total_tokens : nil + }.compact + + span.add_attributes(usage_attributes) + end + + # Handle span exception + def handle_span_exception(span, error) + span.set_attribute('error.type', error.class.name) + span.record_exception(error) + span.status = OpenTelemetry::Trace::Status.error(error.message) + span.finish + end + end + # rubocop:enable Metrics/ModuleLength + end + end + end +end diff --git a/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/operation_name.rb b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/operation_name.rb new file mode 100644 index 000000000..c1e567e2c --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/operation_name.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Instrumentation + module OpenAI + module Patches + # Determine the operation name from the request path + module OperationName + def determine_operation_name(req) + path = req[:path].to_s + + case path + when %r{^chat/completions} + 'chat' + when /^embeddings/ + 'embeddings' + when /^completions/ + 'completions' + when %r{^images/generations} + 'images.generate' + when %r{^images/edits} + 'images.edit' + when %r{^images/variations} + 'images.variation' + when %r{^audio/transcriptions} + 'audio.transcription' + when %r{^audio/translations} + 'audio.translation' + when %r{^audio/speech} + 'audio.speech' + when /^models/ + 'models' + when /^files/ + 'files' + when %r{^fine_tuning/jobs} + 'fine_tuning.jobs' + when %r{^fine_tuning/alpha/graders/run} + 'fine_tuning.graders.run' + when %r{^fine_tuning/alpha/graders/validate} + 'fine_tuning.graders.validate' + when /^fine_tuning/ + 'fine_tuning' + when /^moderations/ + 'moderations' + when /^batches/ + 'batches' + when /^uploads/ + 'uploads' + when /^vector_stores/ + 'vector_stores' + when /^assistants/ + 'assistants' + when %r{^threads/runs} + 'threads.runs' + when /^threads/ + 'threads' + when /^conversations/ + 'conversations' + when %r{^responses/input_tokens} + 'responses.input_tokens' + when /^responses/ + 'responses' + when /^containers/ + 'containers' + when /^evals/ + 'evals' + when /^videos/ + 'videos' + when %r{^chatkit/sessions} + 'chatkit.sessions' + when %r{^chatkit/threads} + 'chatkit.threads' + when %r{^realtime/client_secrets} + 'realtime.client_secrets' + else + 'openai.request' + end + end + end + end + end + end +end diff --git a/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/stream_wrapper.rb b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/stream_wrapper.rb new file mode 100644 index 000000000..975385a71 --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/stream_wrapper.rb @@ -0,0 +1,216 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'utils' + +module OpenTelemetry + module Instrumentation + module OpenAI + module Patches + # Stream wrapper for chat completion streaming + class StreamWrapper + include Enumerable + include Utils + + attr_reader :stream, :span, :capture_content + + def initialize(stream, span, capture_content) + @stream = stream + @span = span + @capture_content = capture_content + @response_id = nil + @response_model = nil + @service_tier = nil + @finish_reasons = [] + @prompt_tokens = 0 + @completion_tokens = 0 + @choice_buffers = [] + @span_started = true + end + + def each(&) + @stream.each do |event| + process_event(event) + yield(event) if block_given? + end + rescue StandardError => e + handle_error(e) + raise + ensure + cleanup + end + + private + + # @param chunk [OpenAI::Models::Chat::ChatCompletionChunk] + def process_event(chunk) + response_metadata(chunk) + build_streaming_response(chunk) + usage(chunk) + end + + def response_metadata(chunk) + @response_model ||= chunk.model if chunk.respond_to?(:model) + @response_id ||= chunk.id if chunk.respond_to?(:id) + @service_tier ||= chunk.service_tier if chunk.respond_to?(:service_tier) + end + + def build_streaming_response(chunk) + return unless chunk.respond_to?(:choices) && chunk.choices + + chunk.choices.each do |choice| + next unless choice.respond_to?(:delta) && choice.delta + + # Ensure we have enough choice buffers + index = choice.respond_to?(:index) ? choice.index : 0 + @choice_buffers << ChoiceBuffer.new(@choice_buffers.size) while @choice_buffers.size <= index + + buffer = @choice_buffers[index] + buffer.finish_reason = choice.finish_reason if choice.respond_to?(:finish_reason) && choice.finish_reason + + delta = choice.delta + buffer.append_content(delta.content) if delta.respond_to?(:content) && delta.content + buffer.append_tool_calls(delta.tool_calls) if delta.respond_to?(:tool_calls) && delta.tool_calls + end + end + + def usage(chunk) + return unless chunk.respond_to?(:usage) && chunk.usage + + usage = chunk.usage + @completion_tokens = usage.completion_tokens if usage.respond_to?(:completion_tokens) + @prompt_tokens = usage.prompt_tokens if usage.respond_to?(:prompt_tokens) + end + + def cleanup + return unless @span_started + + # Set final attributes only if span is still recording + if @span.recording? + finish_reasons = @choice_buffers.map { |x| x.finish_reason.to_s } + attributes = { + 'gen_ai.response.model' => @response_model, + 'gen_ai.response.id' => @response_id, + 'gen_ai.usage.input_tokens' => @prompt_tokens.positive? ? @prompt_tokens : nil, + 'gen_ai.usage.output_tokens' => @completion_tokens.positive? ? @completion_tokens : nil, + 'gen_ai.response.finish_reasons' => finish_reasons.any? ? finish_reasons : nil, + 'openai.response.service_tier' => @service_tier.to_s + }.compact + @span.add_attributes(attributes) + end + + # Emit structured log events for each choice (not span events) + if @capture_content + @choice_buffers.each do |buffer| + event = buffer.to_log_event + log_structured_event(event) + end + end + ensure + @span.finish + @span_started = false + end + + def handle_error(error) + @span.set_attribute('error.type', error.class.name) + @span.record_exception(error) + @span.status = OpenTelemetry::Trace::Status.error(error.message) + end + + # Buffer for accumulating streaming choice data + class ChoiceBuffer + attr_accessor :finish_reason + attr_reader :index, :text_content, :tool_call_buffers + + def initialize(index) + @index = index + @text_content = [] + @tool_call_buffers = [] + @finish_reason = nil + end + + def append_content(content) + @text_content << content if content + end + + def append_tool_calls(tool_calls) + tool_calls.each do |tool_call| + # Find or create tool call buffer + buffer = @tool_call_buffers.find { |b| b.index == tool_call.index } if tool_call.respond_to?(:index) + + if buffer.nil? + tc_index = tool_call.respond_to?(:index) ? tool_call.index : @tool_call_buffers.size + buffer = ToolCallBuffer.new(tc_index) + @tool_call_buffers << buffer + end + + buffer.append(tool_call) + end + end + + def to_log_event + body = { + index: @index, + finish_reason: @finish_reason&.to_s || 'error', + message: { + role: 'assistant' + } + } + + body[:message][:content] = @text_content.join if @text_content.any? + + if @tool_call_buffers.any? + tool_calls = @tool_call_buffers.map(&:to_hash) + body[:message][:tool_calls] = tool_calls + end + + { + event_name: 'gen_ai.choice', + attributes: { + 'gen_ai.provider.name' => 'openai' + }, + body: body + } + end + end + + # Buffer for accumulating tool call data + class ToolCallBuffer + attr_reader :index + + def initialize(index) + @index = index + @tool_call_id = nil + @function_name = nil + @arguments = [] + end + + def append(tool_call) + @tool_call_id ||= tool_call.id if tool_call.respond_to?(:id) + + return unless tool_call.respond_to?(:function) && tool_call.function + + function = tool_call.function + @function_name ||= function.name if function.respond_to?(:name) + @arguments << function.arguments if function.respond_to?(:arguments) && function.arguments + end + + def to_hash + { + id: @tool_call_id, + type: 'function', + function: { + name: @function_name, + arguments: @arguments.join + } + } + end + end + end + end + end + end +end diff --git a/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/utils.rb b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/utils.rb new file mode 100644 index 000000000..9133255a9 --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/patches/utils.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'json' +require 'logger' + +module OpenTelemetry + module Instrumentation + module OpenAI + module Patches + # Utils + module Utils + def get_property_value(obj, property_name) + if obj.is_a?(Hash) + obj[property_name] || obj[property_name.to_sym] + else + obj.respond_to?(property_name) ? obj.public_send(property_name) : nil + end + end + + def extract_tool_calls(item, capture_content) + tool_calls = get_property_value(item, :tool_calls) + return nil unless tool_calls + + calls = [] + tool_calls.each do |tool_call| + tool_call_dict = {} + + call_id = get_property_value(tool_call, :id) + tool_call_dict[:id] = call_id if call_id + + tool_type = get_property_value(tool_call, :type) + tool_call_dict[:type] = tool_type.to_s if tool_type + + func = get_property_value(tool_call, :function) + if func + tool_call_dict[:function] = {} + + name = get_property_value(func, :name) + tool_call_dict[:function][:name] = name if name + + arguments = get_property_value(func, :arguments) + if capture_content && arguments + arguments = arguments.to_s.delete("\n") if arguments.is_a?(String) + tool_call_dict[:function][:arguments] = arguments + end + end + + calls << tool_call_dict + end + calls + end + + def message_to_log_event(message, capture_content: true) + role = get_property_value(message, :role)&.to_s + content = get_property_value(message, :content) + + body = {} + body[:content] = content.to_s if capture_content && content + + if role == 'assistant' + tool_calls = extract_tool_calls(message, capture_content) + body = { tool_calls: tool_calls } if tool_calls + elsif role == 'tool' + tool_call_id = get_property_value(message, :tool_call_id) + body[:id] = tool_call_id if tool_call_id + end + + { + event_name: "gen_ai.#{role}.message", + attributes: { + 'gen_ai.provider.name' => 'openai' + }, + body: body.empty? ? nil : body + } + end + + def choice_to_log_event(choice, capture_content: true) + index = get_property_value(choice, :index) || 0 + finish_reason = get_property_value(choice, :finish_reason)&.to_s || 'error' + + body = { + index: index, + finish_reason: finish_reason + } + + message_obj = get_property_value(choice, :message) + if message_obj + message = {} + role = get_property_value(message_obj, :role) + message[:role] = role.to_s if role + + tool_calls = extract_tool_calls(message_obj, capture_content) + message[:tool_calls] = tool_calls if tool_calls + + content = get_property_value(message_obj, :content) + message[:content] = content.to_s if capture_content && content + + body[:message] = message + end + + { + event_name: 'gen_ai.choice', + attributes: { + 'gen_ai.provider.name' => 'openai' + }, + body: body + } + end + + def log_structured_event(event) + log_message = { + event: event[:event_name], + attributes: event[:attributes], + body: event[:body] + }.compact + + OpenTelemetry.logger.info(log_message.to_json) + end + end + end + end + end +end diff --git a/instrumentation/openai/lib/opentelemetry/instrumentation/openai/version.rb b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/version.rb new file mode 100644 index 000000000..3b1425122 --- /dev/null +++ b/instrumentation/openai/lib/opentelemetry/instrumentation/openai/version.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Instrumentation + module OpenAI + VERSION = '0.0.0' + end + end +end diff --git a/instrumentation/openai/opentelemetry-instrumentation-openai.gemspec b/instrumentation/openai/opentelemetry-instrumentation-openai.gemspec new file mode 100644 index 000000000..fd4d792e9 --- /dev/null +++ b/instrumentation/openai/opentelemetry-instrumentation-openai.gemspec @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'opentelemetry/instrumentation/openai/version' + +Gem::Specification.new do |spec| + spec.name = 'opentelemetry-instrumentation-openai' + spec.version = OpenTelemetry::Instrumentation::OpenAI::VERSION + spec.authors = ['OpenTelemetry Authors'] + spec.email = ['cncf-opentelemetry-contributors@lists.cncf.io'] + + spec.summary = 'OpenAI instrumentation for the OpenTelemetry framework' + spec.description = 'OpenAI instrumentation for the OpenTelemetry framework' + spec.homepage = 'https://github.com/open-telemetry/opentelemetry-ruby-contrib' + spec.license = 'Apache-2.0' + + spec.files = Dir.glob('lib/**/*.rb') + + Dir.glob('*.md') + + ['LICENSE', '.yardopts'] + spec.require_paths = ['lib'] + spec.required_ruby_version = '>= 3.2' + + spec.add_dependency 'opentelemetry-instrumentation-base', '~> 0.25.0' + + if spec.respond_to?(:metadata) + spec.metadata['changelog_uri'] = "https://rubydoc.info/gems/#{spec.name}/#{spec.version}/file/CHANGELOG.md" + spec.metadata['source_code_uri'] = 'https://github.com/open-telemetry/opentelemetry-ruby-contrib/tree/main/instrumentation/openai' + spec.metadata['bug_tracker_uri'] = 'https://github.com/open-telemetry/opentelemetry-ruby-contrib/issues' + spec.metadata['documentation_uri'] = "https://rubydoc.info/gems/#{spec.name}/#{spec.version}" + end +end diff --git a/instrumentation/openai/test/opentelemetry/instrumentation/openai/instrumentation_test.rb b/instrumentation/openai/test/opentelemetry/instrumentation/openai/instrumentation_test.rb new file mode 100644 index 000000000..9190cad2a --- /dev/null +++ b/instrumentation/openai/test/opentelemetry/instrumentation/openai/instrumentation_test.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +require_relative '../../../../lib/opentelemetry/instrumentation/openai' + +describe OpenTelemetry::Instrumentation::OpenAI do + let(:instrumentation) { OpenTelemetry::Instrumentation::OpenAI::Instrumentation.instance } + + it 'has #name' do + _(instrumentation.name).must_equal 'OpenTelemetry::Instrumentation::OpenAI' + end + + it 'has #version' do + _(instrumentation.version).wont_be_nil + _(instrumentation.version).wont_be_empty + end + + describe '#install' do + it 'accepts argument' do + _(instrumentation.install({})).must_equal(true) + instrumentation.instance_variable_set(:@installed, false) + end + end +end diff --git a/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/client_test.rb b/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/client_test.rb new file mode 100644 index 000000000..640a5749a --- /dev/null +++ b/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/client_test.rb @@ -0,0 +1,384 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +require_relative '../../../../../lib/opentelemetry/instrumentation/openai' +require_relative '../../../../../lib/opentelemetry/instrumentation/openai/patches/client' + +describe OpenTelemetry::Instrumentation::OpenAI::Patches::Client do + let(:instrumentation) { OpenTelemetry::Instrumentation::OpenAI::Instrumentation.instance } + let(:exporter) { EXPORTER } + let(:spans) { exporter.finished_spans } + let(:client_span) { spans.first } + + before do + exporter.reset + instrumentation.install + end + + after do + instrumentation.instance_variable_set(:@installed, false) + end + + describe 'chat completions via client.request' do + let(:model) { 'gpt-4' } + let(:messages) { [{ role: 'user', content: 'Hello!' }] } + let(:response_body) do + { + id: 'chatcmpl-123', + object: 'chat.completion', + created: 1_677_652_288, + model: 'gpt-4-0613', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'Hello! How can I assist you today?' + }, + finish_reason: 'stop' + } + ], + usage: { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30 + } + } + end + + before do + stub_request(:post, 'https://api.openai.com/v1/chat/completions') + .to_return(status: 200, body: response_body.to_json, headers: { 'Content-Type' => 'application/json' }) + end + + it 'creates span with basic attributes for chat completions request' do + client = OpenAI::Client.new(api_key: 'test-token') + client.chat.completions.create( + model: model, + messages: messages + ) + + _(client_span).wont_be_nil + _(client_span.name).must_include 'chat' + _(client_span.kind).must_equal :client + + _(client_span.attributes['gen_ai.operation.name']).must_equal 'chat' + _(client_span.attributes['gen_ai.provider.name']).must_equal 'openai' + _(client_span.attributes['gen_ai.request.model']).must_equal model + _(client_span.attributes['server.address']).must_equal 'api.openai.com' + _(client_span.attributes['server.port']).must_equal 443 + _(client_span.attributes['http.request.method']).must_equal 'POST' + _(client_span.attributes['url.path']).must_equal 'chat/completions' + _(client_span.attributes['gen_ai.response.model']).must_equal 'gpt-4-0613' + _(client_span.attributes['gen_ai.response.id']).must_equal 'chatcmpl-123' + _(client_span.attributes['gen_ai.response.finish_reasons']).must_equal ['stop'] + _(client_span.attributes['gen_ai.usage.input_tokens']).must_equal 10 + _(client_span.attributes['gen_ai.usage.output_tokens']).must_equal 20 + _(client_span.attributes['gen_ai.usage.total_tokens']).must_equal 30 + end + + it 'sets optional chat completion parameters' do + client = OpenAI::Client.new(api_key: 'test-token') + client.chat.completions.create( + model: model, + messages: messages, + temperature: 0.7, + max_tokens: 100, + top_p: 0.9, + frequency_penalty: 0.5, + presence_penalty: 0.3, + seed: 42 + ) + + _(client_span.attributes['gen_ai.request.temperature']).must_equal 0.7 + _(client_span.attributes['gen_ai.request.max_tokens']).must_equal 100 + _(client_span.attributes['gen_ai.request.top_p']).must_equal 0.9 + _(client_span.attributes['gen_ai.request.frequency_penalty']).must_equal 0.5 + _(client_span.attributes['gen_ai.request.presence_penalty']).must_equal 0.3 + _(client_span.attributes['gen_ai.request.seed']).must_equal 42 + end + + it 'captures message content when enabled' do + # Content capture logs to logger, not span events + # This test verifies the span is created successfully + instrumentation.instance_variable_set(:@installed, false) + instrumentation.install + instrumentation.config[:capture_content] = true + + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output, level: Logger::INFO) + + client = OpenAI::Client.new(api_key: 'test-token') + client.chat.completions.create( + model: model, + messages: messages + ) + + OpenTelemetry.logger = original_logger + + _(client_span).wont_be_nil + logged_message = logger_output.string + + _(logged_message).must_include 'gen_ai.user.message' + _(logged_message).must_include 'Hello!' + _(logged_message).must_include 'gen_ai.choice' + _(logged_message).must_include 'Hello! How can I assist you today?' + _(logged_message).must_include 'stop' + _(logged_message).must_include 'gen_ai.provider.name' + _(logged_message).must_include 'openai' + end + end + + describe 'embeddings via client.request' do + let(:model) { 'text-embedding-ada-002' } + let(:input_text) { 'The quick brown fox jumps over the lazy dog.' } + let(:response_body) do + { + object: 'list', + data: [ + { + object: 'embedding', + embedding: Array.new(1536) { rand }, + index: 0 + } + ], + model: 'text-embedding-ada-002-v2', + usage: { + prompt_tokens: 10, + total_tokens: 10 + } + } + end + + before do + stub_request(:post, 'https://api.openai.com/v1/embeddings') + .to_return(status: 200, body: response_body.to_json, headers: { 'Content-Type' => 'application/json' }) + end + + it 'creates a span for embeddings request' do + client = OpenAI::Client.new(api_key: 'test-token') + client.embeddings.create( + model: model, + input: input_text + ) + + _(client_span).wont_be_nil + _(client_span.name).must_include 'embeddings' + _(client_span.kind).must_equal :client + + _(client_span.attributes['gen_ai.operation.name']).must_equal 'embeddings' + _(client_span.attributes['gen_ai.provider.name']).must_equal 'openai' + _(client_span.attributes['gen_ai.request.model']).must_equal model + _(client_span.attributes['server.address']).must_equal 'api.openai.com' + _(client_span.attributes['server.port']).must_equal 443 + _(client_span.attributes['http.request.method']).must_equal 'POST' + _(client_span.attributes['url.path']).must_equal 'embeddings' + _(client_span.attributes['gen_ai.output.type']).must_equal 'json' + _(client_span.attributes['gen_ai.response.model']).must_equal 'text-embedding-ada-002-v2' + _(client_span.attributes['gen_ai.embeddings.dimension.count']).must_equal 1536 + _(client_span.attributes['gen_ai.usage.input_tokens']).must_equal 10 + _(client_span.attributes['gen_ai.usage.total_tokens']).must_equal 10 + end + + it 'captures embedding input content when enabled' do + instrumentation.instance_variable_set(:@installed, false) + instrumentation.install + instrumentation.config[:capture_content] = true + + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output, level: Logger::INFO) + + client = OpenAI::Client.new(api_key: 'test-token') + client.embeddings.create( + model: model, + input: input_text + ) + + OpenTelemetry.logger = original_logger + + _(client_span).wont_be_nil + logged_message = logger_output.string + _(logged_message).must_include 'gen_ai.user.message' + _(logged_message).must_include 'gen_ai.provider.name' + _(logged_message).must_include 'openai' + _(logged_message).must_include 'content' + _(logged_message).must_include 'The quick brown fox jumps over the lazy dog.' + end + end + + describe 'error handling' do + before do + stub_request(:post, 'https://api.openai.com/v1/chat/completions') + .to_return(status: 500, body: { error: { message: 'Internal Server Error' } }.to_json) + end + + it 'records exception and sets error status' do + client = OpenAI::Client.new(api_key: 'test-token') + + begin + client.chat.completions.create( + model: 'gpt-4', + messages: [{ role: 'user', content: 'Hello!' }] + ) + rescue StandardError + # Expected to raise + end + + _(client_span).wont_be_nil + _(client_span.status.code).must_equal OpenTelemetry::Trace::Status::ERROR + _(client_span.status.description).must_include 'status=>500' + _(client_span.attributes['gen_ai.operation.name']).must_equal 'chat' + _(client_span.attributes['gen_ai.provider.name']).must_equal 'openai' + _(client_span.attributes['gen_ai.request.model']).must_equal 'gpt-4' + _(client_span.attributes['server.address']).must_equal 'api.openai.com' + _(client_span.attributes['server.port']).must_equal 443 + _(client_span.attributes['http.request.method']).must_equal 'POST' + _(client_span.attributes['url.path']).must_equal 'chat/completions' + _(client_span.attributes['gen_ai.output.type']).must_equal 'text' + _(client_span.attributes['error.type']).must_equal 'OpenAI::Errors::InternalServerError' + + exception_event = client_span.events.find { |event| event.name == 'exception' } + _(exception_event).wont_be_nil + _(exception_event.attributes['exception.type']).must_equal 'OpenAI::Errors::InternalServerError' + _(exception_event.attributes['exception.message']).must_include 'status=>500' + end + end + + # Images generation is not in the default allowed_operation list + # These tests are skipped. To enable, add 'images.generate' to allowed_operation config + describe 'images generation via client.request (skipped - not in allowed_operation)' do + let(:response_body) do + { + created: 1_677_652_288, + data: [ + { + url: 'https://example.com/image1.png' + }, + { + url: 'https://example.com/image2.png' + } + ] + } + end + + before do + stub_request(:post, 'https://api.openai.com/v1/images/generations') + .to_return(status: 200, body: response_body.to_json, headers: { 'Content-Type' => 'application/json' }) + end + + it 'skip span creation for images generation' do + client = OpenAI::Client.new(api_key: 'test-token') + + client.images.generate_stream_raw( + prompt: 'A futuristic cityscape at night', + model: 'gpt-image-1', + n: 3, # Generate 3 different images + partial_images: 4, + size: '1536x1024', # Landscape + output_format: :png, + output_compression: 80 + ) + + _(client_span).must_be_nil + end + end + + describe 'completions (legacy) via client.request' do + let(:model) { 'gpt-3.5-turbo-instruct' } + let(:prompt) { 'Once upon a time' } + let(:response_body) do + { + id: 'cmpl-123', + object: 'text_completion', + created: 1_677_652_288, + model: 'gpt-3.5-turbo-instruct', + choices: [ + { + text: ' there was a kingdom far away.', + index: 0, + finish_reason: 'stop' + } + ], + usage: { + prompt_tokens: 5, + completion_tokens: 10, + total_tokens: 15 + } + } + end + + before do + stub_request(:post, 'https://api.openai.com/v1/completions') + .to_return(status: 200, body: response_body.to_json, headers: { 'Content-Type' => 'application/json' }) + end + + it 'creates a span for completions request' do + client = OpenAI::Client.new(api_key: 'test-token') + test_model = model + test_prompt = prompt + + client.instance_eval do + request( + method: :post, + path: 'completions', + body: { + model: test_model, + prompt: test_prompt + }, + model: OpenAI::Internal::Type::Unknown + ) + end + + _(client_span).wont_be_nil + _(client_span.attributes['gen_ai.operation.name']).must_equal 'completions' + _(client_span.attributes['gen_ai.provider.name']).must_equal 'openai' + _(client_span.attributes['gen_ai.request.model']).must_equal model + _(client_span.attributes['server.address']).must_equal 'api.openai.com' + _(client_span.attributes['server.port']).must_equal 443 + _(client_span.attributes['http.request.method']).must_equal 'POST' + _(client_span.attributes['url.path']).must_equal 'completions' + _(client_span.attributes['gen_ai.output.type']).must_equal 'json' + end + + it 'captures prompt content when enabled' do + instrumentation.instance_variable_set(:@installed, false) + instrumentation.install + instrumentation.config[:capture_content] = true + + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output, level: Logger::INFO) + + client = OpenAI::Client.new(api_key: 'test-token') + test_model = model + test_prompt = prompt + + client.instance_eval do + request( + method: :post, + path: 'completions', + body: { + model: test_model, + prompt: test_prompt + }, + model: OpenAI::Internal::Type::Unknown + ) + end + + OpenTelemetry.logger = original_logger + + _(client_span).wont_be_nil + logged_message = logger_output.string + _(logged_message).must_include 'gen_ai.user.message' + _(logged_message).must_include 'openai' + _(logged_message).must_include 'Once upon a time' + end + end +end diff --git a/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/operation_name_test.rb b/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/operation_name_test.rb new file mode 100644 index 000000000..b9dc6cb24 --- /dev/null +++ b/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/operation_name_test.rb @@ -0,0 +1,189 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +require_relative '../../../../../lib/opentelemetry/instrumentation/openai/patches/operation_name' + +describe OpenTelemetry::Instrumentation::OpenAI::Patches::OperationName do + let(:operation_name_module) do + Class.new do + include OpenTelemetry::Instrumentation::OpenAI::Patches::OperationName + end.new + end + + describe '#determine_operation_name' do + it 'returns chat for chat completions path' do + req = { path: 'chat/completions' } + _(operation_name_module.determine_operation_name(req)).must_equal 'chat' + end + + it 'returns embeddings for embeddings path' do + req = { path: 'embeddings' } + _(operation_name_module.determine_operation_name(req)).must_equal 'embeddings' + end + + it 'returns completions for completions path' do + req = { path: 'completions' } + _(operation_name_module.determine_operation_name(req)).must_equal 'completions' + end + + it 'returns images.generate for images generations path' do + req = { path: 'images/generations' } + _(operation_name_module.determine_operation_name(req)).must_equal 'images.generate' + end + + it 'returns images.edit for images edits path' do + req = { path: 'images/edits' } + _(operation_name_module.determine_operation_name(req)).must_equal 'images.edit' + end + + it 'returns images.variation for images variations path' do + req = { path: 'images/variations' } + _(operation_name_module.determine_operation_name(req)).must_equal 'images.variation' + end + + it 'returns audio.transcription for audio transcriptions path' do + req = { path: 'audio/transcriptions' } + _(operation_name_module.determine_operation_name(req)).must_equal 'audio.transcription' + end + + it 'returns audio.translation for audio translations path' do + req = { path: 'audio/translations' } + _(operation_name_module.determine_operation_name(req)).must_equal 'audio.translation' + end + + it 'returns audio.speech for audio speech path' do + req = { path: 'audio/speech' } + _(operation_name_module.determine_operation_name(req)).must_equal 'audio.speech' + end + + it 'returns models for models path' do + req = { path: 'models' } + _(operation_name_module.determine_operation_name(req)).must_equal 'models' + end + + it 'returns files for files path' do + req = { path: 'files' } + _(operation_name_module.determine_operation_name(req)).must_equal 'files' + end + + it 'returns fine_tuning.jobs for fine tuning jobs path' do + req = { path: 'fine_tuning/jobs' } + _(operation_name_module.determine_operation_name(req)).must_equal 'fine_tuning.jobs' + end + + it 'returns fine_tuning.graders.run for fine tuning graders run path' do + req = { path: 'fine_tuning/alpha/graders/run' } + _(operation_name_module.determine_operation_name(req)).must_equal 'fine_tuning.graders.run' + end + + it 'returns fine_tuning.graders.validate for fine tuning graders validate path' do + req = { path: 'fine_tuning/alpha/graders/validate' } + _(operation_name_module.determine_operation_name(req)).must_equal 'fine_tuning.graders.validate' + end + + it 'returns fine_tuning for generic fine tuning path' do + req = { path: 'fine_tuning' } + _(operation_name_module.determine_operation_name(req)).must_equal 'fine_tuning' + end + + it 'returns moderations for moderations path' do + req = { path: 'moderations' } + _(operation_name_module.determine_operation_name(req)).must_equal 'moderations' + end + + it 'returns batches for batches path' do + req = { path: 'batches' } + _(operation_name_module.determine_operation_name(req)).must_equal 'batches' + end + + it 'returns uploads for uploads path' do + req = { path: 'uploads' } + _(operation_name_module.determine_operation_name(req)).must_equal 'uploads' + end + + it 'returns vector_stores for vector stores path' do + req = { path: 'vector_stores' } + _(operation_name_module.determine_operation_name(req)).must_equal 'vector_stores' + end + + it 'returns assistants for assistants path' do + req = { path: 'assistants' } + _(operation_name_module.determine_operation_name(req)).must_equal 'assistants' + end + + it 'returns threads.runs for threads runs path' do + req = { path: 'threads/runs' } + _(operation_name_module.determine_operation_name(req)).must_equal 'threads.runs' + end + + it 'returns threads for threads path' do + req = { path: 'threads' } + _(operation_name_module.determine_operation_name(req)).must_equal 'threads' + end + + it 'returns conversations for conversations path' do + req = { path: 'conversations' } + _(operation_name_module.determine_operation_name(req)).must_equal 'conversations' + end + + it 'returns responses.input_tokens for responses input tokens path' do + req = { path: 'responses/input_tokens' } + _(operation_name_module.determine_operation_name(req)).must_equal 'responses.input_tokens' + end + + it 'returns responses for responses path' do + req = { path: 'responses' } + _(operation_name_module.determine_operation_name(req)).must_equal 'responses' + end + + it 'returns containers for containers path' do + req = { path: 'containers' } + _(operation_name_module.determine_operation_name(req)).must_equal 'containers' + end + + it 'returns evals for evals path' do + req = { path: 'evals' } + _(operation_name_module.determine_operation_name(req)).must_equal 'evals' + end + + it 'returns videos for videos path' do + req = { path: 'videos' } + _(operation_name_module.determine_operation_name(req)).must_equal 'videos' + end + + it 'returns chatkit.sessions for chatkit sessions path' do + req = { path: 'chatkit/sessions' } + _(operation_name_module.determine_operation_name(req)).must_equal 'chatkit.sessions' + end + + it 'returns chatkit.threads for chatkit threads path' do + req = { path: 'chatkit/threads' } + _(operation_name_module.determine_operation_name(req)).must_equal 'chatkit.threads' + end + + it 'returns realtime.client_secrets for realtime client secrets path' do + req = { path: 'realtime/client_secrets' } + _(operation_name_module.determine_operation_name(req)).must_equal 'realtime.client_secrets' + end + + it 'returns openai.request for unknown path' do + req = { path: 'unknown/endpoint' } + _(operation_name_module.determine_operation_name(req)).must_equal 'openai.request' + end + + it 'handles nil path gracefully' do + req = { path: nil } + _(operation_name_module.determine_operation_name(req)).must_equal 'openai.request' + end + + it 'handles path as symbol' do + req = { path: :'chat/completions' } + _(operation_name_module.determine_operation_name(req)).must_equal 'chat' + end + end +end diff --git a/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/stream_wrapper_test.rb b/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/stream_wrapper_test.rb new file mode 100644 index 000000000..e65296ecc --- /dev/null +++ b/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/stream_wrapper_test.rb @@ -0,0 +1,553 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +require_relative '../../../../../lib/opentelemetry/instrumentation/openai' +require_relative '../../../../../lib/opentelemetry/instrumentation/openai/patches/stream_wrapper' + +describe OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper do + let(:instrumentation) { OpenTelemetry::Instrumentation::OpenAI::Instrumentation.instance } + let(:exporter) { EXPORTER } + let(:spans) { exporter.finished_spans } + let(:tracer) { instrumentation.tracer } + + before do + exporter.reset + instrumentation.install(capture_content: true) + end + + after do + instrumentation.instance_variable_set(:@installed, false) + end + + describe 'streaming chat completion' do + # NOTE: StreamWrapper wraps OpenAI::Internal::Stream[ChatCompletionChunk] which yields chunks directly. + # This is different from client.chat.completions.stream() which yields events with .type and .chunk. + # The instrumentation patches client.request() which returns the raw stream before event wrapping. + + it 'wraps stream and collects response data' do + span = tracer.start_root_span('test_span', kind: :client) + + # Mock streaming chunks that simulate OpenAI::Models::Chat::ChatCompletionChunk structure + # These are yielded directly by OpenAI::Internal::Stream when iterating + chunks = [ + # First chunk with role + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content, :role).new('', :assistant), + nil + ) + ] + ), + # Content chunks + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new('1'), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(' '), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new('2'), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(' '), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new('3'), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(' '), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new('4'), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(' '), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new('5'), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new("\n"), + nil + ) + ] + ), + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new('Done'), + nil + ) + ] + ), + # Final chunk with finish reason + Struct.new(:id, :model, :service_tier, :choices).new( + 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez', + 'gpt-5-nano-2025-08-07', + :default, + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(nil), + :stop + ) + ] + ) + ] + + # Simulate OpenAI::Internal::Stream by using an Enumerator that yields chunks + # In production, this would be: response = client.request(..., stream: true) + # which returns OpenAI::Internal::Stream[ChatCompletionChunk] that yields chunks when iterated + stream = chunks.each + wrapper = OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper.new( + stream, + span, + true + ) + + collected_chunks = wrapper.map do |chunk| + chunk + end + + _(collected_chunks.length).must_equal chunks.length + _(span.attributes['gen_ai.response.id']).must_equal 'chatcmpl-Ce2zyewKHuOsD0esxDe6lp5ZqINez' + _(span.attributes['gen_ai.response.model']).must_equal 'gpt-5-nano-2025-08-07' + _(span.attributes['gen_ai.response.finish_reasons']).must_equal ['stop'] + _(span.attributes['openai.response.service_tier']).must_equal 'default' + end + + it 'accumulates streaming content correctly' do + span = tracer.start_root_span('test_span', kind: :client) + + chunks = [ + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content, :role).new('Hello', :assistant), + nil + ) + ] + ), + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(' world'), + nil + ) + ] + ), + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new('!'), + :stop + ) + ] + ) + ] + + stream = chunks.each + wrapper = OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper.new( + stream, + span, + true + ) + + # Capture logger output to check logged events + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output) + + wrapper.each { |_chunk| } + + OpenTelemetry.logger = original_logger + + logged_message = logger_output.string + _(logged_message).must_include 'gen_ai.choice' + _(logged_message).must_include 'openai' + _(logged_message).must_include 'stop' + _(logged_message).must_include 'Hello world!' + end + + it 'handles streaming with usage information' do + span = tracer.start_root_span('test_span', kind: :client) + + chunks = [ + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content, :role).new('Hello', :assistant), + nil + ) + ] + ), + Struct.new(:id, :model, :choices, :usage).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(nil), + :stop + ) + ], + Struct.new(:prompt_tokens, :completion_tokens).new(10, 5) + ) + ] + + stream = chunks.each + wrapper = OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper.new( + stream, + span, + false + ) + + wrapper.each { |_chunk| } + + _(span.attributes['gen_ai.usage.input_tokens']).must_equal 10 + _(span.attributes['gen_ai.usage.output_tokens']).must_equal 5 + end + + it 'handles streaming with tool calls' do + span = tracer.start_root_span('test_span', kind: :client) + + chunks = [ + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:role, :tool_calls).new( + :assistant, + [ + Struct.new(:index, :id, :function).new( + 0, + 'call_123', + Struct.new(:name, :arguments).new('get_weather', '{"loc') + ) + ] + ), + nil + ) + ] + ), + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:tool_calls).new( + [ + Struct.new(:index, :function).new( + 0, + Struct.new(:arguments).new('ation":"NYC"}') + ) + ] + ), + nil + ) + ] + ), + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(nil), + :tool_calls + ) + ] + ) + ] + + stream = chunks.each + wrapper = OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper.new( + stream, + span, + true + ) + + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output) + + wrapper.each { |_chunk| } + + OpenTelemetry.logger = original_logger + + logged_message = logger_output.string + _(logged_message).must_include 'tool_calls' + _(logged_message).must_include 'get_weather' + _(logged_message).must_include 'location' + _(logged_message).must_include 'NYC' + end + + it 'handles multiple choices in streaming' do + span = tracer.start_root_span('test_span', kind: :client) + + chunks = [ + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content, :role).new('Choice 1', :assistant), + nil + ), + Struct.new(:index, :delta, :finish_reason).new( + 1, + Struct.new(:content, :role).new('Choice 2', :assistant), + nil + ) + ] + ), + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content).new(nil), + :stop + ), + Struct.new(:index, :delta, :finish_reason).new( + 1, + Struct.new(:content).new(nil), + :stop + ) + ] + ) + ] + + stream = chunks.each + wrapper = OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper.new( + stream, + span, + true + ) + + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output) + + wrapper.each { |_chunk| } + + OpenTelemetry.logger = original_logger + + _(span.attributes['gen_ai.response.finish_reasons']).must_equal %w[stop stop] + logged_message = logger_output.string + _(logged_message).must_include 'Choice 1' + _(logged_message).must_include 'Choice 2' + end + + it 'handles errors during streaming' do + span = tracer.start_root_span('test_span', kind: :client) + + error_stream = Enumerator.new do |yielder| + yielder << Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content, :role).new('Hello', :assistant), + nil + ) + ] + ) + raise StandardError, 'Stream error' + end + + wrapper = OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper.new( + error_stream, + span, + false + ) + + assert_raises(StandardError) do + wrapper.each { |_chunk| } + end + + _(spans.length).must_equal 1 + _(spans.first.name).must_equal 'test_span' + _(span.attributes['error.type']).must_equal 'StandardError' + _(span.status.code).must_equal OpenTelemetry::Trace::Status::ERROR + end + + it 'finishes span in ensure block even if error occurs' do + span = tracer.start_root_span('test_span', kind: :client) + + error_stream = Enumerator.new do + raise StandardError, 'Test error' + end + + wrapper = OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper.new( + error_stream, + span, + false + ) + + assert_raises(StandardError) do + wrapper.each { |_chunk| } + end + + # Span should be finished even with error + _(spans.length).must_equal 1 + _(spans.first.name).must_equal 'test_span' + _(spans.first.attributes['error.type']).must_equal 'StandardError' + end + + it 'does not log content when capture_content is false' do + span = tracer.start_root_span('test_span', kind: :client) + + chunks = [ + Struct.new(:id, :model, :choices).new( + 'chatcmpl-123', + 'gpt-4', + [ + Struct.new(:index, :delta, :finish_reason).new( + 0, + Struct.new(:content, :role).new('Secret content', :assistant), + :stop + ) + ] + ) + ] + + stream = chunks.each + wrapper = OpenTelemetry::Instrumentation::OpenAI::Patches::StreamWrapper.new( + stream, + span, + false # capture_content disabled + ) + + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output) + + wrapper.each { |_chunk| } + + OpenTelemetry.logger = original_logger + + logged_message = logger_output.string + _(logged_message).must_be_empty + end + end +end diff --git a/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/utils_test.rb b/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/utils_test.rb new file mode 100644 index 000000000..c2a45fe65 --- /dev/null +++ b/instrumentation/openai/test/opentelemetry/instrumentation/openai/patches/utils_test.rb @@ -0,0 +1,346 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' +require 'json' + +require_relative '../../../../../lib/opentelemetry/instrumentation/openai/patches/utils' + +describe OpenTelemetry::Instrumentation::OpenAI::Patches::Utils do + let(:utils_class) do + Class.new do + include OpenTelemetry::Instrumentation::OpenAI::Patches::Utils + end.new + end + + describe '#get_property_value' do + it 'retrieves value from hash with string key' do + obj = { 'name' => 'test' } + _(utils_class.get_property_value(obj, 'name')).must_equal 'test' + end + + it 'retrieves value from hash with symbol key' do + obj = { name: 'test' } + _(utils_class.get_property_value(obj, 'name')).must_equal 'test' + end + + it 'retrieves value from object with method' do + obj = Struct.new(:name).new('test') + _(utils_class.get_property_value(obj, :name)).must_equal 'test' + end + + it 'returns nil for non-existent property in hash' do + obj = { name: 'test' } + _(utils_class.get_property_value(obj, 'missing')).must_be_nil + end + + it 'returns nil for non-existent method in object' do + obj = Struct.new(:name).new('test') + _(utils_class.get_property_value(obj, :missing)).must_be_nil + end + end + + describe '#extract_tool_calls' do + it 'returns nil when tool_calls is not present' do + item = { role: 'assistant', content: 'Hello' } + _(utils_class.extract_tool_calls(item, true)).must_be_nil + end + + it 'extracts tool call with id and type' do + item = { + tool_calls: [ + { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location":"NYC"}' + } + } + ] + } + + result = utils_class.extract_tool_calls(item, true) + _(result).wont_be_nil + _(result.length).must_equal 1 + _(result[0][:id]).must_equal 'call_123' + _(result[0][:type]).must_equal 'function' + _(result[0][:function][:name]).must_equal 'get_weather' + _(result[0][:function][:arguments]).must_equal '{"location":"NYC"}' + end + + it 'strips newlines from arguments when capture_content is true' do + item = { + tool_calls: [ + { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + arguments: "{\n \"location\": \"NYC\"\n}" + } + } + ] + } + + result = utils_class.extract_tool_calls(item, true) + _(result[0][:function][:arguments]).must_equal '{ "location": "NYC"}' + end + + it 'omits arguments when capture_content is false' do + item = { + tool_calls: [ + { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location":"NYC"}' + } + } + ] + } + + result = utils_class.extract_tool_calls(item, false) + _(result[0][:function][:arguments]).must_be_nil + end + + it 'handles multiple tool calls' do + item = { + tool_calls: [ + { + id: 'call_123', + type: 'function', + function: { name: 'get_weather' } + }, + { + id: 'call_456', + type: 'function', + function: { name: 'get_time' } + } + ] + } + + result = utils_class.extract_tool_calls(item, true) + _(result.length).must_equal 2 + _(result[0][:id]).must_equal 'call_123' + _(result[1][:id]).must_equal 'call_456' + end + + it 'handles tool_calls as objects with respond_to' do + function_obj = Struct.new(:name, :arguments).new('get_weather', '{"location":"NYC"}') + tool_call_obj = Struct.new(:id, :type, :function).new('call_123', :function, function_obj) + item = Struct.new(:tool_calls).new([tool_call_obj]) + + result = utils_class.extract_tool_calls(item, true) + _(result).wont_be_nil + _(result[0][:id]).must_equal 'call_123' + _(result[0][:function][:name]).must_equal 'get_weather' + end + end + + describe '#message_to_log_event' do + it 'creates log event for user message with content' do + message = { role: 'user', content: 'Hello, how are you?' } + event = utils_class.message_to_log_event(message, capture_content: true) + + _(event[:event_name]).must_equal 'gen_ai.user.message' + _(event[:attributes]['gen_ai.provider.name']).must_equal 'openai' + _(event[:body][:content]).must_equal 'Hello, how are you?' + end + + it 'creates log event for system message' do + message = { role: 'system', content: 'You are a helpful assistant.' } + event = utils_class.message_to_log_event(message, capture_content: true) + + _(event[:event_name]).must_equal 'gen_ai.system.message' + _(event[:body][:content]).must_equal 'You are a helpful assistant.' + end + + it 'creates log event for assistant message with tool calls' do + message = { + role: 'assistant', + tool_calls: [ + { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location":"NYC"}' + } + } + ] + } + + event = utils_class.message_to_log_event(message, capture_content: true) + _(event[:event_name]).must_equal 'gen_ai.assistant.message' + _(event[:body][:tool_calls]).wont_be_nil + _(event[:body][:tool_calls][0][:id]).must_equal 'call_123' + end + + it 'creates log event for tool message' do + message = { role: 'tool', tool_call_id: 'call_123', content: 'Weather is sunny' } + event = utils_class.message_to_log_event(message, capture_content: true) + + _(event[:event_name]).must_equal 'gen_ai.tool.message' + _(event[:body][:id]).must_equal 'call_123' + _(event[:body][:content]).must_equal 'Weather is sunny' + end + + it 'omits content when capture_content is false' do + message = { role: 'user', content: 'Hello' } + event = utils_class.message_to_log_event(message, capture_content: false) + + _(event[:body]).must_be_nil + end + + it 'handles role as symbol' do + message = { role: :assistant, content: 'Hello' } + event = utils_class.message_to_log_event(message, capture_content: true) + + _(event[:event_name]).must_equal 'gen_ai.assistant.message' + end + end + + describe '#choice_to_log_event' do + it 'creates log event for choice with message content' do + choice = { + index: 0, + finish_reason: 'stop', + message: { + role: 'assistant', + content: 'Hello, how can I help you?' + } + } + + event = utils_class.choice_to_log_event(choice, capture_content: true) + _(event[:event_name]).must_equal 'gen_ai.choice' + _(event[:attributes]['gen_ai.provider.name']).must_equal 'openai' + _(event[:body][:index]).must_equal 0 + _(event[:body][:finish_reason]).must_equal 'stop' + _(event[:body][:message][:role]).must_equal 'assistant' + _(event[:body][:message][:content]).must_equal 'Hello, how can I help you?' + end + + it 'creates log event for choice with tool calls' do + choice = { + index: 0, + finish_reason: 'tool_calls', + message: { + role: 'assistant', + tool_calls: [ + { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location":"NYC"}' + } + } + ] + } + } + + event = utils_class.choice_to_log_event(choice, capture_content: true) + _(event[:body][:finish_reason]).must_equal 'tool_calls' + _(event[:body][:message][:tool_calls]).wont_be_nil + _(event[:body][:message][:tool_calls][0][:id]).must_equal 'call_123' + end + + it 'defaults finish_reason to error when missing' do + choice = { + index: 0, + message: { role: 'assistant', content: 'Hello' } + } + + event = utils_class.choice_to_log_event(choice, capture_content: true) + _(event[:body][:finish_reason]).must_equal 'error' + end + + it 'defaults index to 0 when missing' do + choice = { + finish_reason: 'stop', + message: { role: 'assistant', content: 'Hello' } + } + + event = utils_class.choice_to_log_event(choice, capture_content: true) + _(event[:body][:index]).must_equal 0 + end + + it 'omits content when capture_content is false' do + choice = { + index: 0, + finish_reason: 'stop', + message: { + role: 'assistant', + content: 'Hello' + } + } + + event = utils_class.choice_to_log_event(choice, capture_content: false) + _(event[:event_name]).must_equal 'gen_ai.choice' + _(event[:body][:index]).must_equal 0 + _(event[:body][:finish_reason]).must_equal 'stop' + _(event[:body][:message][:role]).must_equal 'assistant' + assert_equal(event[:attributes], { 'gen_ai.provider.name' => 'openai' }) + _(event[:body][:message][:content]).must_be_nil + end + + it 'handles choice as object with respond_to methods' do + message_obj = Struct.new(:role, :content).new('assistant', 'Hello') + choice_obj = Struct.new(:index, :finish_reason, :message).new(0, 'stop', message_obj) + + event = utils_class.choice_to_log_event(choice_obj, capture_content: true) + _(event[:body][:index]).must_equal 0 + _(event[:body][:finish_reason]).must_equal 'stop' + _(event[:body][:message][:content]).must_equal 'Hello' + end + end + + describe '#log_structured_event' do + it 'logs event as JSON to OpenTelemetry logger' do + event = { + event_name: 'gen_ai.user.message', + attributes: { 'gen_ai.provider.name' => 'openai' }, + body: { content: 'Hello' } + } + + # Capture logger output + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output) + + utils_class.log_structured_event(event) + + OpenTelemetry.logger = original_logger + + logged_message = logger_output.string + _(logged_message).must_include 'gen_ai.user.message' + _(logged_message).must_include 'openai' + _(logged_message).must_include 'Hello' + end + + it 'handles events with nil body' do + event = { + event_name: 'gen_ai.user.message', + attributes: { 'gen_ai.provider.name' => 'openai' }, + body: nil + } + + logger_output = StringIO.new + original_logger = OpenTelemetry.logger + OpenTelemetry.logger = Logger.new(logger_output) + + utils_class.log_structured_event(event) + + OpenTelemetry.logger = original_logger + + logged_message = logger_output.string + _(logged_message).must_include 'gen_ai.user.message' + _(logged_message).wont_include 'body' + end + end +end diff --git a/instrumentation/openai/test/test_helper.rb b/instrumentation/openai/test/test_helper.rb new file mode 100644 index 000000000..dfef1e31b --- /dev/null +++ b/instrumentation/openai/test/test_helper.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'bundler/setup' +Bundler.require(:default, :development, :test) + +require 'minitest/autorun' +require 'webmock/minitest' +require 'openai' + +# global opentelemetry-sdk setup: +EXPORTER = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new +span_processor = OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(EXPORTER) + +OpenTelemetry::SDK.configure do |c| + c.error_handler = ->(exception:, message:) { raise(exception || message) } + c.logger = Logger.new($stderr, level: ENV.fetch('OTEL_LOG_LEVEL', 'fatal').to_sym) + c.add_span_processor span_processor +end