Ruby Splunk inegration Add Server-Timing headers for RUM

application.rb

# path: config/application.rb
# frozen_string_literal: true

require_relative "boot"

require "rails/all"
# Load RUM trace headers middleware for Splunk Observability Cloud.
require_relative "../lib/middleware/rum_trace_headers"
# Load functions for log correlation with OpenTelemetry traces.
require_relative "../../lib/otel/logging"

# Require the gems listed in Gemfile, including any gems
# you've limited to :test, :development, or :production.
Bundler.require(*Rails.groups)

module App
  class Application < Rails::Application
    # Add RUM trace headers middleware for Splunk Observability Cloud.
    # Adds OpenTelemetry trace IDs to Server-Timing header for RUM correlation.
    config.middleware.use Middleware::RumTraceHeaders

    # Add trace correlation to Rails logs following OpenTelemetry semantic conventions.
    Rails.application.config.log_tags = [
      ->(_request) { Otel::Logging.format_correlation }
    ]
    
    # Initialize configuration defaults for originally generated Rails version.
    config.load_defaults 8.1

    # Please, add to the `ignore` list any other `lib` subdirectories that do
    # not contain `.rb` files, or that should not be reloaded or eager loaded.
    # Common ones are `templates`, `generators`, or `middleware`, for example.
    config.autoload_lib(ignore: %w[assets tasks])

    # Configuration for the application, engines, and railties goes here.
    #
    # These settings can be overridden in specific environments using the files
    # in config/environments, which are processed later.
    #
    # config.time_zone = "Central Time (US & Canada)"
    # config.eager_load_paths << Rails.root.join("extras")
  end
end

logging.rb

# frozen_string_literal: true

# Functions for log correlation with OpenTelemetry traces.
# Implements OpenTelemetry semantic conventions for log correlation by adding
# trace_id and span_id to log messages, enabling correlation between logs and traces.
#
# This follows OpenTelemetry best practices for log correlation:
# https://opentelemetry.io/docs/specs/semconv/general/logs/#log-correlation
#
# Backported from deprecated splunk-otel-ruby gem (End of Support: March 15, 2025).
# Source: https://github.com/signalfx/splunk-otel-ruby/blob/main/lib/splunk/otel/logging.rb
module Otel
  module Logging
    # Returns log formatted trace context that can be added to log messages.
    # Format: "service.name=<service> trace_id=<trace_id> span_id=<span_id>"
    # or just "service.name=<service>" if no active span exists.
    #
    # This enables correlating logs with traces in observability platforms
    # that support OpenTelemetry (e.g., Splunk Observability Cloud, Datadog, etc.).
    def format_correlation
      resource_attributes = OpenTelemetry.tracer_provider.resource.attribute_enumerator.to_h
      service_name = resource_attributes["service.name"] || "unknown"
      span = OpenTelemetry::Trace.current_span

      if span == OpenTelemetry::Trace::Span::INVALID
        "service.name=#{service_name}"
      else
        %W[service.name=#{service_name} trace_id=#{span.context.hex_trace_id}
          span_id=#{span.context.hex_span_id}].join(" ")
      end
    rescue
      "service.name=unknown"
    end

    module_function :format_correlation
  end
end

rum_trace_headers.rb

# path: "lib/middleware/rum_trace_headers.rb"
# frozen_string_literal: true

module Middleware
  # RumTraceHeadersMiddleware propagates OpenTelemetry trace context
  # to response headers for RUM (Real User Monitoring) support.
  #
  # This middleware adds traceparent information to the Server-Timing header
  # and exposes it via CORS headers, enabling frontend applications to correlate
  # client-side performance metrics with server-side traces in Splunk Observability Cloud.
  #
  # This middleware was manually backported from the splunk-otel-ruby gem
  # (https://github.com/signalfx/splunk-otel-ruby) because the gem reached
  # End of Support on March 15, 2025 and is now deprecated. We maintain this
  # functionality directly in our application to continue supporting RUM trace
  # correlation without depending on the deprecated gem.
  #
  # Source code references:
  # - RUM headers logic: https://github.com/signalfx/splunk-otel-ruby/blob/main/lib/splunk/otel/common.rb
  # - Middleware implementation: https://github.com/signalfx/splunk-otel-ruby/blob/main/lib/splunk/otel/instrumentation/rack/middleware.rb
  #
  # For more information about Ruby instrumentation with Splunk Observability Cloud, see:
  # https://help.splunk.com/en/splunk-observability-cloud/manage-data/instrument-back-end-services/instrument-back-end-applications-to-send-spans-to-splunk-apm./instrument-a-ruby-application/instrument-your-ruby-application
  #
  # The middleware is enabled when SPLUNK_TRACE_RESPONSE_HEADER_ENABLED environment variable is set to "true".
  #
  class RumTraceHeaders
    CORS_EXPOSE_HEADER = "Access-Control-Expose-Headers"
    SERVER_TIMING_HEADER = "Server-Timing"

    def initialize(app)
      @app = app
    end

    def call(env)
      status, headers, body = @app.call(env)

      headers = add_rum_headers(headers) if trace_response_header_enabled?

      [status, headers, body]
    end

    private

    def trace_response_header_enabled?
      value = ENV.fetch("SPLUNK_TRACE_RESPONSE_HEADER_ENABLED", "true")
      %w[false no f 0].exclude?(value.strip.downcase)
    end

    def add_rum_headers(headers)
      return headers unless defined?(OpenTelemetry)

      span = OpenTelemetry::Trace.current_span

      return headers if span == OpenTelemetry::Trace::Span::INVALID

      version = "00"
      trace_id = span.context.hex_trace_id
      span_id = span.context.hex_span_id
      flags = span.context.trace_flags.sampled? ? "01" : "00"

      trace_parent = [version, trace_id, span_id, flags]
      trace_parent_value = "traceparent;desc=\"#{trace_parent.join("-")}\""

      headers[SERVER_TIMING_HEADER] = if (headers[SERVER_TIMING_HEADER] || "").empty?
        trace_parent_value
      else
        "#{headers[SERVER_TIMING_HEADER]}, #{trace_parent_value}"
      end

      headers[CORS_EXPOSE_HEADER] = if (headers[CORS_EXPOSE_HEADER] || "").empty?
        SERVER_TIMING_HEADER
      else
        "#{headers[CORS_EXPOSE_HEADER]}, #{SERVER_TIMING_HEADER}"
      end

      headers
    end
  end
end