OpenTelemetry Prototype Draft

build-support/build_opentelemetry.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+
+# Copyright (c) YugabyteDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.  See the License for the specific language governing permissions and limitations
+# under the License.
+
+# Build OpenTelemetry C++ SDK with static libraries for YugabyteDB
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+YB_SRC_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Configuration
+OTEL_VERSION="v1.24.0"
+# Default to installing in thirdparty directory (no sudo required)
+OTEL_INSTALL_PREFIX="${OTEL_INSTALL_PREFIX:-${YB_SRC_ROOT}/thirdparty/installed/opentelemetry}"
+BUILD_DIR="/tmp/opentelemetry-cpp-build"
+NUM_JOBS="${NUM_JOBS:-$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)}"
+
+echo "================================================================"
+echo "Building OpenTelemetry C++ SDK ${OTEL_VERSION}"
+echo "================================================================"
+echo "Install prefix: ${OTEL_INSTALL_PREFIX}"
+echo "Build directory: ${BUILD_DIR}"
+echo "Parallel jobs: ${NUM_JOBS}"
+echo ""
+
+# Check for required tools
+for tool in git cmake make; do
+  if ! command -v "$tool" &> /dev/null; then
+    echo "ERROR: $tool is required but not installed"
+    exit 1
+  fi
+done
+
+# Clean up old build directory
+if [[ -d "${BUILD_DIR}" ]]; then
+  echo "Removing old build directory..."
+  rm -rf "${BUILD_DIR}"
+fi
+
+# Clone OpenTelemetry C++ SDK
+echo "Cloning OpenTelemetry C++ SDK..."
+git clone --recurse-submodules --depth 1 --branch "${OTEL_VERSION}" \
+  https://github.com/open-telemetry/opentelemetry-cpp.git "${BUILD_DIR}"
+
+cd "${BUILD_DIR}"
+
+# Create build directory
+mkdir -p build
+cd build
+
+echo ""
+echo "Configuring OpenTelemetry build..."
+echo ""
+
+# Configure with CMake
+# - Static libraries only (BUILD_SHARED_LIBS=OFF)
+# - OTLP HTTP exporter (avoids gRPC/protobuf conflicts with YugabyteDB)
+# - No examples, tests, or benchmarks
+# - Use libc++ to match YugabyteDB's ABI (required for Linux builds)
+
+# On Linux, use clang with libc++ to match YugabyteDB's ABI
+# macOS uses libc++ by default with Apple clang
+CMAKE_EXTRA_FLAGS=""
+if [[ "$(uname)" != "Darwin" ]]; then
+  # Find clang in YB's LLVM installation
+  CLANG_PATH=""
+  CLANGXX_PATH=""
+
+  # Check /opt/yb-build/llvm for YB's clang installation
+  for llvm_dir in /opt/yb-build/llvm/yb-llvm-*/; do
+    if [[ -x "${llvm_dir}bin/clang++" ]]; then
+      CLANG_PATH="${llvm_dir}bin/clang"
+      CLANGXX_PATH="${llvm_dir}bin/clang++"
+      break
+    fi
+  done
+
+  # Fallback to system clang if not found
+  if [[ -z "${CLANGXX_PATH}" ]]; then
+    if command -v clang++ &> /dev/null; then
+      CLANG_PATH="clang"
+      CLANGXX_PATH="clang++"
+    fi
+  fi
+
+  if [[ -n "${CLANGXX_PATH}" ]]; then
+    echo "Using clang: ${CLANGXX_PATH}"
+    CMAKE_EXTRA_FLAGS="-DCMAKE_C_COMPILER=${CLANG_PATH} -DCMAKE_CXX_COMPILER=${CLANGXX_PATH} -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_EXE_LINKER_FLAGS=-stdlib=libc++ -DCMAKE_SHARED_LINKER_FLAGS=-stdlib=libc++"
+  else
+    echo "WARNING: clang++ not found, using default compiler (may cause ABI issues)"
+  fi
+fi
+
+# shellcheck disable=SC2086
+cmake .. \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX="${OTEL_INSTALL_PREFIX}" \
+  -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+  -DCMAKE_CXX_STANDARD=17 \
+  -DBUILD_SHARED_LIBS=OFF \
+  -DWITH_OTLP_GRPC=OFF \
+  -DWITH_OTLP_HTTP=OFF \
+  -DWITH_HTTP_CLIENT_CURL=OFF \
+  -DWITH_PROMETHEUS=OFF \
+  -DWITH_ZIPKIN=OFF \
+  -DWITH_JAEGER=OFF \
+  -DWITH_ELASTICSEARCH=OFF \
+  -DWITH_EXAMPLES=OFF \
+  -DWITH_LOGS_PREVIEW=OFF \
+  -DWITH_METRICS_PREVIEW=OFF \
+  -DBUILD_TESTING=OFF \
+  -DWITH_BENCHMARK=OFF \
+  ${CMAKE_EXTRA_FLAGS}
+
+echo ""
+echo "Building OpenTelemetry (this may take a few minutes)..."
+echo ""
+
+# Build
+make -j"${NUM_JOBS}"
+
+echo ""
+echo "Installing OpenTelemetry to ${OTEL_INSTALL_PREFIX}..."
+echo ""
+
+# Create install prefix directory if it doesn't exist
+mkdir -p "${OTEL_INSTALL_PREFIX}"
+
+# Install (no sudo - use a writable prefix or set OTEL_INSTALL_PREFIX)
+if [[ -w "${OTEL_INSTALL_PREFIX}" ]]; then
+  make install
+else
+  echo "ERROR: Install directory ${OTEL_INSTALL_PREFIX} is not writable"
+  echo "Either:"
+  echo "  1. Set OTEL_INSTALL_PREFIX to a writable location, or"
+  echo "  2. Create the directory with appropriate permissions first"
+  exit 1
+fi
+
+# Verify installation
+echo ""
+echo "Verifying installation..."
+if [[ -f "${OTEL_INSTALL_PREFIX}/include/opentelemetry/version.h" ]]; then
+  echo "✓ Headers installed"
+else
+  echo "✗ Headers not found"
+  exit 1
+fi
+
+if [[ -f "${OTEL_INSTALL_PREFIX}/lib/libopentelemetry_trace.a" ]]; then
+  echo "✓ Static libraries installed"
+else
+  echo "✗ Static libraries not found"
+  exit 1
+fi
+
+# List installed libraries
+echo ""
+echo "Installed static libraries:"
+ls -lh "${OTEL_INSTALL_PREFIX}/lib/"*.a | awk '{print "  " $9 " (" $5 ")"}'
+
+# Clean up build directory
+echo ""
+echo "Removing build directory ${BUILD_DIR}..."
+rm -rf "${BUILD_DIR}"
+echo "Build directory removed"
+
+echo ""
+echo "================================================================"
+echo "OpenTelemetry C++ SDK installed successfully!"
+echo "================================================================"
+echo ""
+echo "Installation directory: ${OTEL_INSTALL_PREFIX}"
+echo ""
+echo "Next steps:"
+echo "  1. Rebuild YugabyteDB: ./yb_build.sh release"
+echo "  2. CMake will automatically detect and use OpenTelemetry"
+echo ""
+echo "To uninstall:"
+echo "  rm -rf ${OTEL_INSTALL_PREFIX}"
+echo ""
+

dev.yml

@@ -0,0 +1,44 @@
+name: yugabyte-db
+
+up:
+  - packages:
+      - llvm@16
+      - cmake
+  - go: 1.24.6
+  - python: 3.12.7
+
+  - custom:
+      name: Verify Clang 16
+      met?: /opt/homebrew/opt/llvm@16/bin/clang --version | grep -q "clang version 16"
+      meet: echo "Clang 16 installed via llvm@16"
+
+#  - custom:
+#      name: Verify CMake >= 3.31
+#      met?: cmake --version | grep -E "cmake version (3\.3[1-9]|3\.[4-9][0-9]|[4-9]\.[0-9]+)"
+#      meet: echo "CMake version check passed"
+
+  # Setup Yugabyte build dir
+  - custom:
+      name: Setup build directory
+      met?: test -d /opt/yb-build && [ "$(stat -f '%Su' /opt/yb-build 2>/dev/null || echo '')" = "$USER" ]
+      meet: sudo mkdir -p /opt/yb-build && sudo chown "$USER" /opt/yb-build
+
+env:
+  # Point to Clang 16
+  CC: /opt/homebrew/opt/llvm@16/bin/clang
+  CXX: /opt/homebrew/opt/llvm@16/bin/clang++
+  LDFLAGS: "-L/opt/homebrew/opt/llvm@16/lib"
+  CPPFLAGS: "-I/opt/homebrew/opt/llvm@16/include"
+
+commands:
+  build:
+    desc: Build Yugabyte 
+    run: ./yb_build.sh release
+
+#  test:
+#    desc: Run tests
+#    run: ./build/your-test-binary
+
+#  clean:
+#    desc: Clean build artifacts
+#    run: rm -rf build

src/postgres/src/backend/access/transam/xact.c

@@ -2450,6 +2450,7 @@ CommitTransaction(void)
 		 * Postgres transaction can be aborted at this point without an issue
 		 * in case of YBCCommitTransaction failure.
 		 */
+		YBCOtelCommitStart();
 		YBCCommitTransaction();
 		if (increment_pg_txns)
 			YbIncrementPgTxnsCommitted();
@@ -2486,6 +2487,7 @@ CommitTransaction(void)
 	}
 
 	TRACE_POSTGRESQL_TRANSACTION_COMMIT(MyProc->lxid);
+	YBCOtelCommitDone();
 
 	/*
 	 * Let others know about no transaction in progress by me. Note that this
@@ -3044,6 +3046,7 @@ AbortTransaction(void)
 		XLogSetAsyncXactLSN(XactLastRecEnd);
 	}
 
+	YBCOtelAbortStart();
 	TRACE_POSTGRESQL_TRANSACTION_ABORT(MyProc->lxid);
 
 	/*
@@ -3095,6 +3098,7 @@ AbortTransaction(void)
 	}
 
 	YBCAbortTransaction();
+	YBCOtelAbortDone();
 
 	/* Reset the value of the sticky connection */
 	s->ybUncommittedStickyObjectCount = 0;

src/postgres/src/backend/tcop/postgres.c

@@ -692,6 +692,7 @@ pg_parse_query(const char *query_string)
 	List	   *raw_parsetree_list;
 
 	TRACE_POSTGRESQL_QUERY_PARSE_START(query_string);
+	YBCOtelParseStart();
 
 	if (log_parser_stats)
 		ResetUsage();
@@ -720,6 +721,7 @@ pg_parse_query(const char *query_string)
 	 * here.
 	 */
 
+	YBCOtelParseDone();
 	TRACE_POSTGRESQL_QUERY_PARSE_DONE(query_string);
 
 	return raw_parsetree_list;
@@ -766,6 +768,7 @@ pg_analyze_and_rewrite_fixedparams(RawStmt *parsetree,
 	List	   *querytree_list;
 
 	TRACE_POSTGRESQL_QUERY_REWRITE_START(query_string);
+	YBCOtelRewriteStart();
 
 	/*
 	 * (1) Perform parse analysis.
@@ -784,6 +787,7 @@ pg_analyze_and_rewrite_fixedparams(RawStmt *parsetree,
 	 */
 	querytree_list = pg_rewrite_query(query);
 
+	YBCOtelRewriteDone();
 	TRACE_POSTGRESQL_QUERY_REWRITE_DONE(query_string);
 
 	return querytree_list;
@@ -805,6 +809,7 @@ pg_analyze_and_rewrite_varparams(RawStmt *parsetree,
 	List	   *querytree_list;
 
 	TRACE_POSTGRESQL_QUERY_REWRITE_START(query_string);
+	YBCOtelRewriteStart();
 
 	/*
 	 * (1) Perform parse analysis.
@@ -837,6 +842,7 @@ pg_analyze_and_rewrite_varparams(RawStmt *parsetree,
 	 */
 	querytree_list = pg_rewrite_query(query);
 
+	YBCOtelRewriteDone();
 	TRACE_POSTGRESQL_QUERY_REWRITE_DONE(query_string);
 
 	return querytree_list;
@@ -859,6 +865,7 @@ pg_analyze_and_rewrite_withcb(RawStmt *parsetree,
 	List	   *querytree_list;
 
 	TRACE_POSTGRESQL_QUERY_REWRITE_START(query_string);
+	YBCOtelRewriteStart();
 
 	/*
 	 * (1) Perform parse analysis.
@@ -877,6 +884,7 @@ pg_analyze_and_rewrite_withcb(RawStmt *parsetree,
 	 */
 	querytree_list = pg_rewrite_query(query);
 
+	YBCOtelRewriteDone();
 	TRACE_POSTGRESQL_QUERY_REWRITE_DONE(query_string);
 
 	return querytree_list;
@@ -994,6 +1002,7 @@ pg_plan_query(Query *querytree, const char *query_string, int cursorOptions,
 	Assert(ActiveSnapshotSet());
 
 	TRACE_POSTGRESQL_QUERY_PLAN_START();
+	YBCOtelPlanStart();
 
 	if (log_planner_stats)
 		ResetUsage();
@@ -1053,6 +1062,7 @@ pg_plan_query(Query *querytree, const char *query_string, int cursorOptions,
 	if (Debug_print_plan)
 		elog_node_display(LOG, "plan", plan, Debug_pretty_print);
 
+	YBCOtelPlanDone();
 	TRACE_POSTGRESQL_QUERY_PLAN_DONE();
 
 	return plan;
@@ -1127,12 +1137,16 @@ exec_simple_query(const char *query_string)
 	 */
 	debug_query_string = query_string;
 
+	/* Parse traceparent from query comment for OTEL tracing */
+	YbSetTraceparentFromQuery(query_string);
+
 	/* Use YbParseCommandTag to suppress error warnings. */
 	command_tag = YbParseCommandTag(query_string);
 	redacted_query_string = YbRedactPasswordIfExists(query_string, command_tag);
 	pgstat_report_activity(STATE_RUNNING, redacted_query_string);
 
 	TRACE_POSTGRESQL_QUERY_START(query_string);
+	YBCOtelQueryStart(query_string);
 
 	/*
 	 * We use save_log_statement_stats so ShowUsage doesn't report incorrect
@@ -1479,6 +1493,7 @@ exec_simple_query(const char *query_string)
 	if (save_log_statement_stats)
 		ShowUsage("QUERY STATISTICS");
 
+	YBCOtelQueryDone();
 	TRACE_POSTGRESQL_QUERY_DONE(query_string);
 
 	debug_query_string = NULL;

src/postgres/src/backend/tcop/pquery.c

@@ -32,6 +32,7 @@
 #include "executor/ybModifyTable.h"
 #include "optimizer/ybplan.h"
 #include "pg_yb_utils.h"
+#include "yb/yql/pggate/ybc_pggate.h"
 
 
 /*
@@ -713,6 +714,7 @@ PortalRun(Portal portal, long count, bool isTopLevel, bool run_once,
 	AssertArg(PortalIsValid(portal));
 
 	TRACE_POSTGRESQL_QUERY_EXECUTE_START();
+	YBCOtelExecuteStart();
 
 	/* Initialize empty completion data */
 	if (qc)
@@ -865,6 +867,7 @@ PortalRun(Portal portal, long count, bool isTopLevel, bool run_once,
 	if (log_executor_stats && portal->strategy != PORTAL_MULTI_QUERY)
 		ShowUsage("EXECUTOR STATISTICS");
 
+	YBCOtelExecuteDone();
 	TRACE_POSTGRESQL_QUERY_EXECUTE_DONE();
 
 	return result;