From bfb7c637ccf7fd44c09201990042bc872f7f740c Mon Sep 17 00:00:00 2001 From: Radoslav Kirilov Date: Mon, 6 Oct 2025 16:44:55 +0300 Subject: [PATCH 1/7] feat(sidekiq): scheduled jobs improvements - `scheduled` operation and add links When a Job is pushed via `perform_in` or friends, it goes through the following life-cycle: 1. First the Job is pushed with `at` attribute in Redis from `Sidekiq::Client` - this will now create `<...> scheduled` span (before it was `<...> publish` 2. Then the Job is handled from `Sidekiq::Scheduled#enqueue` and pushed without `at` attribute in Redis from `Sidekiq::Client` - this will keep creating `<...> publish` span 3. Then the Job is handled by the Worker as usual Additionally when using `:propagation_style = :link` and `:trace_poller_enqueue = true` a handy set of links are added between the 3 actors above. --- .../middlewares/client/tracer_middleware.rb | 17 ++++++++++++++--- .../middlewares/server/tracer_middleware.rb | 3 +++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb index c15f70047e..cd2ea06e03 100644 --- a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb +++ b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb @@ -24,14 +24,25 @@ def call(_worker_class, job, _queue, _redis_pool) } attributes[SemanticConventions::Trace::PEER_SERVICE] = instrumentation_config[:peer_service] if instrumentation_config[:peer_service] + scheduled_at = job['at'] + op = scheduled_at.nil? ? "publish" : "scheduled" + span_name = case instrumentation_config[:span_naming] - when :job_class then "#{job['wrapped']&.to_s || job['class']} publish" - else "#{job['queue']} publish" + when :job_class then "#{job['wrapped']&.to_s || job['class']} #{op}" + else "#{job['queue']} #{op}" end - tracer.in_span(span_name, attributes: attributes, kind: :producer) do |span| + # In case this is Scheduled job, there is already context injected, so link to that context + # NOTE: :propagation_style = :child is not supported as it is quite tricky when :trace_poller_enqueue = true + extracted_context = OpenTelemetry.propagation.extract(job, context: OpenTelemetry::Context::ROOT) + links = [] + span_context = OpenTelemetry::Trace.current_span(extracted_context).context + links << OpenTelemetry::Trace::Link.new(span_context) if instrumentation_config[:propagation_style] == :link && span_context.valid? + + tracer.in_span(span_name, attributes: attributes, links: links, kind: :producer) do |span| OpenTelemetry.propagation.inject(job) span.add_event('created_at', timestamp: time_from_timestamp(job['created_at'])) + span.add_event('scheduled_at', timestamp: time_from_timestamp(scheduled_at)) unless scheduled_at.nil? yield end end diff --git a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb index 47c3d7dce7..0a882e6e91 100644 --- a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb +++ b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb @@ -33,11 +33,13 @@ def call(_worker, msg, _queue) extracted_context = OpenTelemetry.propagation.extract(msg) created_at = time_from_timestamp(msg['created_at']) enqueued_at = time_from_timestamp(msg['created_at']) + scheduled_at = time_from_timestamp(msg['at']) unless msg['at'].nil? OpenTelemetry::Context.with_current(extracted_context) do if instrumentation_config[:propagation_style] == :child tracer.in_span(span_name, attributes: attributes, kind: :consumer) do |span| span.add_event('created_at', timestamp: created_at) span.add_event('enqueued_at', timestamp: enqueued_at) + span.add_event('scheduled_at', timestamp: scheduled_at) unless scheduled_at.nil? yield end else @@ -48,6 +50,7 @@ def call(_worker, msg, _queue) OpenTelemetry::Trace.with_span(span) do span.add_event('created_at', timestamp: created_at) span.add_event('enqueued_at', timestamp: enqueued_at) + span.add_event('scheduled_at', timestamp: scheduled_at) unless scheduled_at.nil? yield rescue Exception => e # rubocop:disable Lint/RescueException span.record_exception(e) From dc004078e6240590bb5da1262c33130146c96ffa Mon Sep 17 00:00:00 2001 From: Radoslav Kirilov Date: Mon, 6 Oct 2025 17:36:57 +0300 Subject: [PATCH 2/7] fix: use `enqueued_at` for `enqueued_at` broken with https://github.com/open-telemetry/opentelemetry-ruby-contrib/pull/1444 --- .../sidekiq/middlewares/server/tracer_middleware.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb index 0a882e6e91..c28f7b8803 100644 --- a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb +++ b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb @@ -32,7 +32,7 @@ def call(_worker, msg, _queue) extracted_context = OpenTelemetry.propagation.extract(msg) created_at = time_from_timestamp(msg['created_at']) - enqueued_at = time_from_timestamp(msg['created_at']) + enqueued_at = time_from_timestamp(msg['enqueued_at']) scheduled_at = time_from_timestamp(msg['at']) unless msg['at'].nil? OpenTelemetry::Context.with_current(extracted_context) do if instrumentation_config[:propagation_style] == :child From 6a3983f9946be52e2a85254fb5429e6f063b9f20 Mon Sep 17 00:00:00 2001 From: Radoslav Kirilov Date: Tue, 7 Oct 2025 20:08:19 +0300 Subject: [PATCH 3/7] fix: tests work with empty array of links --- .../sidekiq/middlewares/server/tracer_middleware_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb index 5c770bb78c..90820a79fa 100644 --- a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb +++ b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb @@ -133,7 +133,7 @@ _(root_span.kind).must_equal :producer # process span is linked to the root enqueuing job - child_span1 = spans.find { |s| s.links && s.links.first.span_context.span_id == root_span.span_id } + child_span1 = spans.find { |s| s.links&.first&.span_context&.span_id == root_span.span_id } _(child_span1.name).must_equal 'default process' _(child_span1.kind).must_equal :consumer @@ -143,7 +143,7 @@ _(child_span2.kind).must_equal :producer # last process job is linked back to the process job that enqueued it - child_span3 = spans.find { |s| s.links && s.links.first.span_context.span_id == child_span2.span_id } + child_span3 = spans.find { |s| s.links&.first&.span_context&.span_id == child_span2.span_id } _(child_span3.name).must_equal 'default process' _(child_span3.kind).must_equal :consumer end From 4ed7a704d3ae948d0c963fb0dd14ac9bbf127cbd Mon Sep 17 00:00:00 2001 From: Radoslav Kirilov Date: Wed, 8 Oct 2025 18:33:05 +0300 Subject: [PATCH 4/7] fix: support ruby 3.1 fixes https://github.com/open-telemetry/opentelemetry-ruby-contrib/actions/runs/18347031681/job/52256395831?pr=1717 --- .../sidekiq/middlewares/server/tracer_middleware_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb index 90820a79fa..20cf0626e0 100644 --- a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb +++ b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb @@ -133,7 +133,7 @@ _(root_span.kind).must_equal :producer # process span is linked to the root enqueuing job - child_span1 = spans.find { |s| s.links&.first&.span_context&.span_id == root_span.span_id } + child_span1 = spans.find { |s| s.links && s.links.first && s.links.first.span_context.span_id == root_span.span_id } _(child_span1.name).must_equal 'default process' _(child_span1.kind).must_equal :consumer @@ -143,7 +143,7 @@ _(child_span2.kind).must_equal :producer # last process job is linked back to the process job that enqueued it - child_span3 = spans.find { |s| s.links&.first&.span_context&.span_id == child_span2.span_id } + child_span3 = spans.find { |s| s.links && s.links.first && s.links.first.span_context.span_id == child_span2.span_id } _(child_span3.name).must_equal 'default process' _(child_span3.kind).must_equal :consumer end From 37deaa10c62618161ba042c9c9105e43df860487 Mon Sep 17 00:00:00 2001 From: Radoslav Kirilov Date: Thu, 9 Oct 2025 01:59:18 +0300 Subject: [PATCH 5/7] Revert "fix: support ruby 3.1" This reverts commit 4ed7a704d3ae948d0c963fb0dd14ac9bbf127cbd. --- .../sidekiq/middlewares/server/tracer_middleware_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb index 20cf0626e0..90820a79fa 100644 --- a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb +++ b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb @@ -133,7 +133,7 @@ _(root_span.kind).must_equal :producer # process span is linked to the root enqueuing job - child_span1 = spans.find { |s| s.links && s.links.first && s.links.first.span_context.span_id == root_span.span_id } + child_span1 = spans.find { |s| s.links&.first&.span_context&.span_id == root_span.span_id } _(child_span1.name).must_equal 'default process' _(child_span1.kind).must_equal :consumer @@ -143,7 +143,7 @@ _(child_span2.kind).must_equal :producer # last process job is linked back to the process job that enqueued it - child_span3 = spans.find { |s| s.links && s.links.first && s.links.first.span_context.span_id == child_span2.span_id } + child_span3 = spans.find { |s| s.links&.first&.span_context&.span_id == child_span2.span_id } _(child_span3.name).must_equal 'default process' _(child_span3.kind).must_equal :consumer end From 730ca36dc467f4466da397c353d3f3f33173e483 Mon Sep 17 00:00:00 2001 From: Radoslav Kirilov Date: Thu, 9 Oct 2025 01:59:44 +0300 Subject: [PATCH 6/7] fix: lint issues use single quote --- .../sidekiq/middlewares/client/tracer_middleware.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb index cd2ea06e03..156bd6166f 100644 --- a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb +++ b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb @@ -25,7 +25,7 @@ def call(_worker_class, job, _queue, _redis_pool) attributes[SemanticConventions::Trace::PEER_SERVICE] = instrumentation_config[:peer_service] if instrumentation_config[:peer_service] scheduled_at = job['at'] - op = scheduled_at.nil? ? "publish" : "scheduled" + op = scheduled_at.nil? ? 'publish' : 'scheduled' span_name = case instrumentation_config[:span_naming] when :job_class then "#{job['wrapped']&.to_s || job['class']} #{op}" From 8055c89be6f5656ef209927ca2cc7b1c74333b76 Mon Sep 17 00:00:00 2001 From: Radoslav Kirilov Date: Thu, 9 Oct 2025 02:03:16 +0300 Subject: [PATCH 7/7] fix: lint issues use less safe navigation --- .../sidekiq/middlewares/server/tracer_middleware_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb index 90820a79fa..0ea036ee38 100644 --- a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb +++ b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb @@ -133,7 +133,7 @@ _(root_span.kind).must_equal :producer # process span is linked to the root enqueuing job - child_span1 = spans.find { |s| s.links&.first&.span_context&.span_id == root_span.span_id } + child_span1 = spans.find { |s| s.links&.first && s.links.first.span_context.span_id == root_span.span_id } _(child_span1.name).must_equal 'default process' _(child_span1.kind).must_equal :consumer @@ -143,7 +143,7 @@ _(child_span2.kind).must_equal :producer # last process job is linked back to the process job that enqueued it - child_span3 = spans.find { |s| s.links&.first&.span_context&.span_id == child_span2.span_id } + child_span3 = spans.find { |s| s.links&.first && s.links.first.span_context.span_id == child_span2.span_id } _(child_span3.name).must_equal 'default process' _(child_span3.kind).must_equal :consumer end