From 435ff40bbe5aec425e4b2808be80fc8a7b2c3843 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Fri, 8 May 2026 13:34:20 +0000 Subject: [PATCH 1/7] feat(jdbc): implement OTel SDK caching and thread-safe context propagation --- .../google-cloud-bigquery-jdbc/pom.xml | 13 ++++ .../bigquery/jdbc/BigQueryConnection.java | 22 ++++-- .../jdbc/BigQueryJdbcOAuthUtility.java | 2 +- .../jdbc/BigQueryJdbcOpenTelemetry.java | 75 +++++++++++++++++-- .../bigquery/jdbc/BigQueryJdbcUrlUtility.java | 10 +++ .../cloud/bigquery/jdbc/DataSource.java | 24 ++++++ .../jdbc/BigQueryArrowStructTest.java | 4 +- .../jdbc/BigQueryJdbcOpenTelemetryTest.java | 29 ++++++- 8 files changed, 161 insertions(+), 18 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery-jdbc/pom.xml b/java-bigquery/google-cloud-bigquery-jdbc/pom.xml index f0b19c988214..d3d32af701ab 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/pom.xml +++ b/java-bigquery/google-cloud-bigquery-jdbc/pom.xml @@ -301,6 +301,19 @@ io.opentelemetry opentelemetry-context + + + io.opentelemetry + opentelemetry-sdk + + + io.opentelemetry + opentelemetry-exporter-otlp + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure + diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index e515a0b14701..61ff7a0b60c0 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -88,6 +88,8 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { int transactionIsolation; List sqlWarnings; String catalog; + String gcpTelemetryCredentials; + String gcpTelemetryProjectId; int holdability; long retryTimeoutInSeconds; Duration retryTimeoutDuration; @@ -169,6 +171,8 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { this.labels = ds.getLabels() != null ? ds.getLabels() : new java.util.HashMap<>(); this.maxBytesBilled = ds.getMaximumBytesBilled(); + this.gcpTelemetryCredentials = ds.getGcpTelemetryCredentials(); + this.gcpTelemetryProjectId = ds.getGcpTelemetryProjectId(); this.retryTimeoutInSeconds = ds.getTimeout(); this.retryTimeoutDuration = Duration.ofMillis(retryTimeoutInSeconds * 1000L); this.retryInitialDelayInSeconds = ds.getRetryInitialDelay(); @@ -1000,14 +1004,18 @@ private BigQuery getBigQueryConnection() { OpenTelemetry openTelemetry = BigQueryJdbcOpenTelemetry.getOpenTelemetry( - this.enableGcpTraceExporter, this.enableGcpLogExporter, this.customOpenTelemetry); + this.enableGcpTraceExporter, + this.enableGcpLogExporter, + this.customOpenTelemetry, + this.gcpTelemetryCredentials, + this.gcpTelemetryProjectId); - if (this.enableGcpLogExporter || this.customOpenTelemetry != null) { + if (Boolean.TRUE.equals(this.enableGcpLogExporter) || this.customOpenTelemetry != null) { BigQueryJdbcOpenTelemetry.registerConnection( this.connectionId, openTelemetry, null, this.enableGcpLogExporter); } - if (this.enableGcpTraceExporter || this.customOpenTelemetry != null) { + if (Boolean.TRUE.equals(this.enableGcpTraceExporter) || this.customOpenTelemetry != null) { this.tracer = BigQueryJdbcOpenTelemetry.getTracer(openTelemetry); bigQueryOptions.setOpenTelemetryTracer(this.tracer); } @@ -1062,8 +1070,12 @@ private BigQueryReadClient getBigQueryReadClientConnection() throws IOException OpenTelemetry openTelemetry = BigQueryJdbcOpenTelemetry.getOpenTelemetry( - this.enableGcpTraceExporter, this.enableGcpLogExporter, this.customOpenTelemetry); - if (this.enableGcpTraceExporter || this.customOpenTelemetry != null) { + this.enableGcpTraceExporter, + this.enableGcpLogExporter, + this.customOpenTelemetry, + this.gcpTelemetryCredentials, + this.gcpTelemetryProjectId); + if (Boolean.TRUE.equals(this.enableGcpTraceExporter) || this.customOpenTelemetry != null) { bigQueryReadSettings.setOpenTelemetryTracerProvider(openTelemetry.getTracerProvider()); } diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOAuthUtility.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOAuthUtility.java index 7e05e7d74618..47c3dc58505e 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOAuthUtility.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOAuthUtility.java @@ -307,7 +307,7 @@ private static boolean isFileExists(String filename) { } } - private static boolean isJson(byte[] value) { + static boolean isJson(byte[] value) { try { // This is done this way to ensure strict Json parsing // https://github.com/google/gson/issues/1208#issuecomment-2120764686 diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index 3216b25d1557..a108868333c7 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -19,7 +19,12 @@ import com.google.cloud.logging.Logging; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import java.nio.charset.StandardCharsets; import java.util.Collection; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Handler; import java.util.logging.Logger; @@ -29,6 +34,19 @@ public class BigQueryJdbcOpenTelemetry { static final String INSTRUMENTATION_SCOPE_NAME = "com.google.cloud.bigquery.jdbc"; static final String BIGQUERY_NAMESPACE = "com.google.cloud.bigquery"; public static final String CONNECTION_ID_BAGGAGE_KEY = "jdbc.connection_id"; + private static final String OTEL_TRACES_EXPORTER = "otel.traces.exporter"; + private static final String OTEL_EXPORTER_OTLP_ENDPOINT = "otel.exporter.otlp.endpoint"; + private static final String OTEL_LOGS_EXPORTER = "otel.logs.exporter"; + private static final String OTEL_METRICS_EXPORTER = "otel.metrics.exporter"; + private static final String GOOGLE_CLOUD_PROJECT = "google.cloud.project"; + private static final String CREDENTIALS_JSON = "google.cloud.credentials.json"; + private static final String CREDENTIALS_PATH = "google.cloud.credentials.path"; + private static final String OTLP_ENDPOINT_VALUE = "https://telemetry.googleapis.com:443"; + private static final String EXPORTER_NONE = "none"; + private static final String EXPORTER_OTLP = "otlp"; + + private static final ConcurrentHashMap sdkCache = + new ConcurrentHashMap<>(); static class TelemetryConfig { final OpenTelemetry openTelemetry; @@ -36,10 +54,10 @@ static class TelemetryConfig { final boolean useDirectGcpLogging; TelemetryConfig( - OpenTelemetry openTelemetry, Logging loggingClient, boolean useDirectGcpLogging) { + OpenTelemetry openTelemetry, Logging loggingClient, Boolean useDirectGcpLogging) { this.openTelemetry = openTelemetry; this.loggingClient = loggingClient; - this.useDirectGcpLogging = useDirectGcpLogging; + this.useDirectGcpLogging = useDirectGcpLogging != null ? useDirectGcpLogging : false; } } @@ -70,7 +88,7 @@ public static void registerConnection( String connectionId, OpenTelemetry openTelemetry, Logging loggingClient, - boolean useDirectGcpLogging) { + Boolean useDirectGcpLogging) { connectionConfigs.put( connectionId, new TelemetryConfig(openTelemetry, loggingClient, useDirectGcpLogging)); } @@ -94,14 +112,59 @@ public static Collection getRegisteredConfigs() { public static OpenTelemetry getOpenTelemetry( boolean enableGcpTraceExporter, boolean enableGcpLogExporter, - OpenTelemetry customOpenTelemetry) { + OpenTelemetry customOpenTelemetry, + String gcpTelemetryCredentials, + String gcpTelemetryProjectId) { if (customOpenTelemetry != null) { return customOpenTelemetry; } + // NOTE: Currently, tracing only fully supports Application Default Credentials (ADC). + // Once b/503721589 is completed, Service Account (SA) will work as well. + if (enableGcpTraceExporter || enableGcpLogExporter) { - // TODO(b/491238299): Initialize and return GCP-specific auto-configured SDK - return OpenTelemetry.noop(); + String key = + (gcpTelemetryProjectId != null ? gcpTelemetryProjectId : "") + + ":" + + (gcpTelemetryCredentials != null ? gcpTelemetryCredentials : ""); + return sdkCache.computeIfAbsent( + key, + k -> { + Map props = new HashMap<>(); + if (gcpTelemetryCredentials != null) { + byte[] credsBytes = gcpTelemetryCredentials.getBytes(StandardCharsets.UTF_8); + if (BigQueryJdbcOAuthUtility.isJson(credsBytes)) { + props.put(CREDENTIALS_JSON, gcpTelemetryCredentials); + } else { + props.put(CREDENTIALS_PATH, gcpTelemetryCredentials); + } + } + + if (enableGcpTraceExporter) { + props.put(OTEL_TRACES_EXPORTER, EXPORTER_OTLP); + props.put(OTEL_EXPORTER_OTLP_ENDPOINT, OTLP_ENDPOINT_VALUE); + } else { + props.put(OTEL_TRACES_EXPORTER, EXPORTER_NONE); + } + + // Logs are handled directly via GCP logging + props.put(OTEL_LOGS_EXPORTER, EXPORTER_NONE); + // Metrics are deferred to a future phase + props.put(OTEL_METRICS_EXPORTER, EXPORTER_NONE); + + if (gcpTelemetryProjectId != null) { + props.put(GOOGLE_CLOUD_PROJECT, gcpTelemetryProjectId); + } + + AutoConfiguredOpenTelemetrySdk autoConfigured = + AutoConfiguredOpenTelemetrySdk.builder().addPropertiesSupplier(() -> props).build(); + + OpenTelemetrySdk sdk = autoConfigured.getOpenTelemetrySdk(); + + Runtime.getRuntime().addShutdownHook(new Thread(sdk::close)); + + return sdk; + }); } return OpenTelemetry.noop(); diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java index 901edaa92dc1..f55b329c9286 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcUrlUtility.java @@ -98,6 +98,8 @@ protected boolean removeEldestEntry(Map.Entry> eldes static final String BIGQUERY_ENDPOINT_OVERRIDE_PROPERTY_NAME = "BIGQUERY"; static final String STS_ENDPOINT_OVERRIDE_PROPERTY_NAME = "STS"; static final String OAUTH_ACCESS_TOKEN_PROPERTY_NAME = "OAuthAccessToken"; + static final String GCP_TELEMETRY_PROJECT_ID_PROPERTY_NAME = "gcpTelemetryProjectId"; + static final String GCP_TELEMETRY_CREDENTIALS_PROPERTY_NAME = "gcpTelemetryCredentials"; static final String OAUTH_ACCESS_TOKEN_READONLY_PROPERTY_NAME = "OAuthAccessTokenReadonly"; static final String OAUTH_REFRESH_TOKEN_PROPERTY_NAME = "OAuthRefreshToken"; static final String OAUTH_CLIENT_ID_PROPERTY_NAME = "OAuthClientId"; @@ -628,6 +630,14 @@ protected boolean removeEldestEntry(Map.Entry> eldes .setDescription( "Enables or disables GCP OpenTelemetry Log exporter. Disabled by default.") .setDefaultValue(String.valueOf(DEFAULT_ENABLE_GCP_LOG_EXPORTER_VALUE)) + .build(), + BigQueryConnectionProperty.newBuilder() + .setName(GCP_TELEMETRY_CREDENTIALS_PROPERTY_NAME) + .setDescription("Path or raw JSON of credentials for OTel exporter.") + .build(), + BigQueryConnectionProperty.newBuilder() + .setName(GCP_TELEMETRY_PROJECT_ID_PROPERTY_NAME) + .setDescription("GCP Project ID for OTel exporter.") .build()))); private static final List NETWORK_PROPERTIES = diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java index 9f646d9889ac..a52bd19486b5 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/DataSource.java @@ -58,6 +58,8 @@ public class DataSource implements javax.sql.DataSource { private String logLevel; private Boolean enableSession; private String logPath; + private String gcpTelemetryProjectId; + private String gcpTelemetryCredentials; private Integer oAuthType; private String oAuthServiceAcctEmail; private String oAuthPvtKeyPath; @@ -134,6 +136,12 @@ public class DataSource implements javax.sql.DataSource { .put(BigQueryJdbcUrlUtility.PROJECT_ID_PROPERTY_NAME, DataSource::setProjectId) .put(BigQueryJdbcUrlUtility.DEFAULT_DATASET_PROPERTY_NAME, DataSource::setDefaultDataset) .put(BigQueryJdbcUrlUtility.LOCATION_PROPERTY_NAME, DataSource::setLocation) + .put( + BigQueryJdbcUrlUtility.GCP_TELEMETRY_PROJECT_ID_PROPERTY_NAME, + DataSource::setGcpTelemetryProjectId) + .put( + BigQueryJdbcUrlUtility.GCP_TELEMETRY_CREDENTIALS_PROPERTY_NAME, + DataSource::setGcpTelemetryCredentials) .put( BigQueryJdbcUrlUtility.ENABLE_HTAPI_PROPERTY_NAME, (ds, val) -> @@ -867,6 +875,22 @@ public void setLogPath(String logPath) { this.logPath = logPath; } + public String getGcpTelemetryProjectId() { + return gcpTelemetryProjectId; + } + + public void setGcpTelemetryProjectId(String gcpTelemetryProjectId) { + this.gcpTelemetryProjectId = gcpTelemetryProjectId; + } + + public String getGcpTelemetryCredentials() { + return gcpTelemetryCredentials; + } + + public void setGcpTelemetryCredentials(String gcpTelemetryCredentials) { + this.gcpTelemetryCredentials = gcpTelemetryCredentials; + } + public String getUniverseDomain() { return universeDomain != null ? universeDomain diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowStructTest.java b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowStructTest.java index 3f6b528c87db..84d264cf3925 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowStructTest.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowStructTest.java @@ -59,7 +59,6 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.arrow.vector.util.JsonStringArrayList; import org.apache.arrow.vector.util.JsonStringHashMap; import org.apache.arrow.vector.util.Text; @@ -138,8 +137,7 @@ BIGNUMERIC, new BigDecimal("11.2657"), new BigDecimal("33.4657")), LocalDateTime.parse("2023-03-30T11:15:19.820227")), arrowArraySchemaAndValue( GEOGRAPHY, new Text("POINT(-122 47)"), new Text("POINT(-122 48)")), - arrowArraySchemaAndValue( - BYTES, Stream.of("one", "two").map(String::getBytes).toArray(byte[][]::new))); + arrowArraySchemaAndValue(BYTES, "one".getBytes(), "two".getBytes())); List orderedSchemas = schemaAndValues.stream().map(Tuple::x).collect(Collectors.toList()); diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java index 24f77abdd071..4510d3168e83 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java @@ -29,7 +29,8 @@ public class BigQueryJdbcOpenTelemetryTest { public void testGetOpenTelemetry_withCustomSdk_returnsCustom() { OpenTelemetry mockCustomOtel = mock(OpenTelemetry.class); - OpenTelemetry result = BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, mockCustomOtel); + OpenTelemetry result = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, mockCustomOtel, null, null); assertThat(result).isSameInstanceAs(mockCustomOtel); } @@ -39,14 +40,16 @@ public void testGetOpenTelemetry_withCustomSdkAndFlags_returnsCustom() { OpenTelemetry mockCustomOtel = mock(OpenTelemetry.class); // Custom SDK always takes precedence over individual flags - OpenTelemetry result = BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, true, mockCustomOtel); + OpenTelemetry result = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, true, mockCustomOtel, null, null); assertThat(result).isSameInstanceAs(mockCustomOtel); } @Test public void testGetOpenTelemetry_noFlags_returnsNoop() { - OpenTelemetry result = BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, null); + OpenTelemetry result = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, false, null, null, null); assertThat(result).isSameInstanceAs(OpenTelemetry.noop()); } @@ -57,4 +60,24 @@ public void testGetTracer_respectsScopeName() { assertThat(result).isNotNull(); } + + @Test + public void testGetOpenTelemetry_cachesSdkInstances() { + OpenTelemetry result1 = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + OpenTelemetry result2 = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + + assertThat(result1).isSameInstanceAs(result2); + } + + @Test + public void testGetOpenTelemetry_createsNewInstanceForDifferentKey() { + OpenTelemetry result1 = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + OpenTelemetry result2 = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project2"); + + assertThat(result1).isNotSameInstanceAs(result2); + } } From eb13db4355066a50f1111367a5cee60d8eb6cf78 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Fri, 8 May 2026 13:40:07 +0000 Subject: [PATCH 2/7] chore: fix lint --- .../google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index a108868333c7..32c637ac8c8d 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -45,7 +45,7 @@ public class BigQueryJdbcOpenTelemetry { private static final String EXPORTER_NONE = "none"; private static final String EXPORTER_OTLP = "otlp"; - private static final ConcurrentHashMap sdkCache = + private static final ConcurrentHashMap sdkCache = new ConcurrentHashMap<>(); static class TelemetryConfig { From 0c3229abe522a0f2fb580ba2c1bb5b54c6d84fbf Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Fri, 8 May 2026 14:33:22 +0000 Subject: [PATCH 3/7] chore: add global shutdown hook --- .../jdbc/BigQueryJdbcOpenTelemetry.java | 28 +++++++++++++++++-- .../jdbc/BigQueryJdbcOpenTelemetryTest.java | 10 +++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index 32c637ac8c8d..4b6fb8ca817c 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -17,6 +17,7 @@ package com.google.cloud.bigquery.jdbc; import com.google.cloud.logging.Logging; +import com.google.common.hash.Hashing; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.sdk.OpenTelemetrySdk; @@ -68,6 +69,14 @@ private BigQueryJdbcOpenTelemetry() {} static { ensureGlobalHandlerAttached(); + Runtime.getRuntime() + .addShutdownHook( + new Thread( + () -> { + for (OpenTelemetrySdk sdk : sdkCache.values()) { + sdk.close(); + } + })); } public static void ensureGlobalHandlerAttached() { @@ -105,6 +114,17 @@ public static Collection getRegisteredConfigs() { return connectionConfigs.values(); } + private static String getCredentialsIdentifier(String credentials) { + if (credentials == null) { + return ""; + } + byte[] credsBytes = credentials.getBytes(StandardCharsets.UTF_8); + if (BigQueryJdbcOAuthUtility.isJson(credsBytes)) { + return Hashing.sha256().hashString(credentials, StandardCharsets.UTF_8).toString(); + } + return credentials; + } + /** * Initializes or returns the OpenTelemetry instance based on hybrid logic. Prefer * customOpenTelemetry if provided; fallback to an auto-configured GCP exporter if requested. @@ -126,7 +146,11 @@ public static OpenTelemetry getOpenTelemetry( String key = (gcpTelemetryProjectId != null ? gcpTelemetryProjectId : "") + ":" - + (gcpTelemetryCredentials != null ? gcpTelemetryCredentials : ""); + + getCredentialsIdentifier(gcpTelemetryCredentials) + + ":" + + enableGcpTraceExporter + + ":" + + enableGcpLogExporter; return sdkCache.computeIfAbsent( key, k -> { @@ -161,8 +185,6 @@ public static OpenTelemetry getOpenTelemetry( OpenTelemetrySdk sdk = autoConfigured.getOpenTelemetrySdk(); - Runtime.getRuntime().addShutdownHook(new Thread(sdk::close)); - return sdk; }); } diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java index 4510d3168e83..0608358888e1 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java @@ -80,4 +80,14 @@ public void testGetOpenTelemetry_createsNewInstanceForDifferentKey() { assertThat(result1).isNotSameInstanceAs(result2); } + + @Test + public void testGetOpenTelemetry_createsNewInstanceForDifferentFlags() { + OpenTelemetry result1 = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + OpenTelemetry result2 = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, null, null, "project1"); + + assertThat(result1).isNotSameInstanceAs(result2); + } } From 440924f4b8c5ccce8e1f8622645870323935ed12 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Fri, 8 May 2026 14:57:44 +0000 Subject: [PATCH 4/7] chore: implement custom class for sdkCache --- .../bigquery/jdbc/BigQueryConnection.java | 8 ++-- .../jdbc/BigQueryJdbcOpenTelemetry.java | 48 +++++++++++++++---- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index 61ff7a0b60c0..6f87d21b1b4a 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -1004,8 +1004,8 @@ private BigQuery getBigQueryConnection() { OpenTelemetry openTelemetry = BigQueryJdbcOpenTelemetry.getOpenTelemetry( - this.enableGcpTraceExporter, - this.enableGcpLogExporter, + Boolean.TRUE.equals(this.enableGcpTraceExporter), + Boolean.TRUE.equals(this.enableGcpLogExporter), this.customOpenTelemetry, this.gcpTelemetryCredentials, this.gcpTelemetryProjectId); @@ -1070,8 +1070,8 @@ private BigQueryReadClient getBigQueryReadClientConnection() throws IOException OpenTelemetry openTelemetry = BigQueryJdbcOpenTelemetry.getOpenTelemetry( - this.enableGcpTraceExporter, - this.enableGcpLogExporter, + Boolean.TRUE.equals(this.enableGcpTraceExporter), + Boolean.TRUE.equals(this.enableGcpLogExporter), this.customOpenTelemetry, this.gcpTelemetryCredentials, this.gcpTelemetryProjectId); diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index 4b6fb8ca817c..cf1639d3f3a5 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Handler; import java.util.logging.Logger; @@ -46,7 +47,38 @@ public class BigQueryJdbcOpenTelemetry { private static final String EXPORTER_NONE = "none"; private static final String EXPORTER_OTLP = "otlp"; - private static final ConcurrentHashMap sdkCache = + private static final class SdkCacheKey { + private final String projectId; + private final String credentialsHashOrPath; + private final boolean enableTrace; + private final boolean enableLog; + + SdkCacheKey( + String projectId, String credentialsHashOrPath, boolean enableTrace, boolean enableLog) { + this.projectId = projectId; + this.credentialsHashOrPath = credentialsHashOrPath; + this.enableTrace = enableTrace; + this.enableLog = enableLog; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SdkCacheKey that = (SdkCacheKey) o; + return enableTrace == that.enableTrace + && enableLog == that.enableLog + && Objects.equals(projectId, that.projectId) + && Objects.equals(credentialsHashOrPath, that.credentialsHashOrPath); + } + + @Override + public int hashCode() { + return Objects.hash(projectId, credentialsHashOrPath, enableTrace, enableLog); + } + } + + private static final ConcurrentHashMap sdkCache = new ConcurrentHashMap<>(); static class TelemetryConfig { @@ -143,14 +175,12 @@ public static OpenTelemetry getOpenTelemetry( // Once b/503721589 is completed, Service Account (SA) will work as well. if (enableGcpTraceExporter || enableGcpLogExporter) { - String key = - (gcpTelemetryProjectId != null ? gcpTelemetryProjectId : "") - + ":" - + getCredentialsIdentifier(gcpTelemetryCredentials) - + ":" - + enableGcpTraceExporter - + ":" - + enableGcpLogExporter; + SdkCacheKey key = + new SdkCacheKey( + gcpTelemetryProjectId, + getCredentialsIdentifier(gcpTelemetryCredentials), + enableGcpTraceExporter, + enableGcpLogExporter); return sdkCache.computeIfAbsent( key, k -> { From 0aae0143c71798530bd7f96be042b7e46bb939e4 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Fri, 8 May 2026 16:36:24 +0000 Subject: [PATCH 5/7] chore: address pr feedback --- .../google-cloud-bigquery-jdbc/pom.xml | 2 +- .../jdbc/BigQueryJdbcOpenTelemetry.java | 21 +++++++++++-------- .../jdbc/BigQueryJdbcOpenTelemetryTest.java | 14 +++++++++++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery-jdbc/pom.xml b/java-bigquery/google-cloud-bigquery-jdbc/pom.xml index d3d32af701ab..be812c986723 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/pom.xml +++ b/java-bigquery/google-cloud-bigquery-jdbc/pom.xml @@ -185,7 +185,7 @@ com.google.cloud google-cloud-logging - 3.33.0-SNAPSHOT + 3.32.0 com.google.http-client diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index cf1639d3f3a5..fa748c606865 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -46,19 +46,18 @@ public class BigQueryJdbcOpenTelemetry { private static final String OTLP_ENDPOINT_VALUE = "https://telemetry.googleapis.com:443"; private static final String EXPORTER_NONE = "none"; private static final String EXPORTER_OTLP = "otlp"; + private static final BigQueryJdbcCustomLogger LOG = + new BigQueryJdbcCustomLogger("BigQueryJdbcOpenTelemetry"); private static final class SdkCacheKey { private final String projectId; private final String credentialsHashOrPath; private final boolean enableTrace; - private final boolean enableLog; - SdkCacheKey( - String projectId, String credentialsHashOrPath, boolean enableTrace, boolean enableLog) { + SdkCacheKey(String projectId, String credentialsHashOrPath, boolean enableTrace) { this.projectId = projectId; this.credentialsHashOrPath = credentialsHashOrPath; this.enableTrace = enableTrace; - this.enableLog = enableLog; } @Override @@ -67,14 +66,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; SdkCacheKey that = (SdkCacheKey) o; return enableTrace == that.enableTrace - && enableLog == that.enableLog && Objects.equals(projectId, that.projectId) && Objects.equals(credentialsHashOrPath, that.credentialsHashOrPath); } @Override public int hashCode() { - return Objects.hash(projectId, credentialsHashOrPath, enableTrace, enableLog); + return Objects.hash(projectId, credentialsHashOrPath, enableTrace); } } @@ -106,7 +104,13 @@ private BigQueryJdbcOpenTelemetry() {} new Thread( () -> { for (OpenTelemetrySdk sdk : sdkCache.values()) { - sdk.close(); + try { + sdk.close(); + } catch (Exception e) { + // Ignore failures during shutdown to ensure all SDKs are attempted to be + // closed. Logging is avoided here because the logging system might have + // already been shut down by the JVM. + } } })); } @@ -179,8 +183,7 @@ public static OpenTelemetry getOpenTelemetry( new SdkCacheKey( gcpTelemetryProjectId, getCredentialsIdentifier(gcpTelemetryCredentials), - enableGcpTraceExporter, - enableGcpLogExporter); + enableGcpTraceExporter); return sdkCache.computeIfAbsent( key, k -> { diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java index 0608358888e1..a7abdb479e4c 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetryTest.java @@ -82,12 +82,22 @@ public void testGetOpenTelemetry_createsNewInstanceForDifferentKey() { } @Test - public void testGetOpenTelemetry_createsNewInstanceForDifferentFlags() { + public void testGetOpenTelemetry_createsNewInstanceForDifferentTraceFlag() { OpenTelemetry result1 = - BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, true, null, null, "project1"); OpenTelemetry result2 = BigQueryJdbcOpenTelemetry.getOpenTelemetry(false, true, null, null, "project1"); assertThat(result1).isNotSameInstanceAs(result2); } + + @Test + public void testGetOpenTelemetry_ignoresEnableLogFlagInCacheKey() { + OpenTelemetry result1 = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, true, null, null, "project1"); + OpenTelemetry result2 = + BigQueryJdbcOpenTelemetry.getOpenTelemetry(true, false, null, null, "project1"); + + assertThat(result1).isSameInstanceAs(result2); + } } From 9e46b7688180bf69056e21d43a90c756ca175e62 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Fri, 8 May 2026 17:15:23 +0000 Subject: [PATCH 6/7] chore: add projectId and cred fallback --- .../bigquery/jdbc/BigQueryConnection.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index 6f87d21b1b4a..917be139cbde 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -965,6 +965,32 @@ void removeStatement(Statement statement) { this.openStatements.remove(statement); } + private OpenTelemetry getOpenTelemetryInstance() { + String effectiveProjectId = + (this.gcpTelemetryProjectId != null) ? this.gcpTelemetryProjectId : this.catalog; + + String effectiveCredentials = this.gcpTelemetryCredentials; + String authTypeStr = this.overrideProperties.get("OAuthType"); + + if (effectiveCredentials == null && "0".equals(authTypeStr)) { + effectiveCredentials = this.overrideProperties.get("OAuthPvtKey"); + } + + if (Boolean.TRUE.equals(this.enableGcpTraceExporter) && effectiveCredentials == null) { + if (!"0".equals(authTypeStr) && !"3".equals(authTypeStr)) { + throw new BigQueryJdbcRuntimeException( + "Exporting traces to Google Cloud is only supported when using Application Default Credentials (ADC) or Service Account authentication."); + } + } + + return BigQueryJdbcOpenTelemetry.getOpenTelemetry( + Boolean.TRUE.equals(this.enableGcpTraceExporter), + Boolean.TRUE.equals(this.enableGcpLogExporter), + this.customOpenTelemetry, + effectiveCredentials, + effectiveProjectId); + } + private BigQuery getBigQueryConnection() { BigQueryOptions.Builder bigQueryOptions = BigQueryOptions.newBuilder(); if (this.retryTimeoutInSeconds > 0L @@ -1001,14 +1027,7 @@ private BigQuery getBigQueryConnection() { if (this.httpTransportOptions != null) { bigQueryOptions.setTransportOptions(this.httpTransportOptions); } - - OpenTelemetry openTelemetry = - BigQueryJdbcOpenTelemetry.getOpenTelemetry( - Boolean.TRUE.equals(this.enableGcpTraceExporter), - Boolean.TRUE.equals(this.enableGcpLogExporter), - this.customOpenTelemetry, - this.gcpTelemetryCredentials, - this.gcpTelemetryProjectId); + OpenTelemetry openTelemetry = getOpenTelemetryInstance(); if (Boolean.TRUE.equals(this.enableGcpLogExporter) || this.customOpenTelemetry != null) { BigQueryJdbcOpenTelemetry.registerConnection( @@ -1068,13 +1087,7 @@ private BigQueryReadClient getBigQueryReadClientConnection() throws IOException bigQueryReadSettings.setTransportChannelProvider(activeProvider); - OpenTelemetry openTelemetry = - BigQueryJdbcOpenTelemetry.getOpenTelemetry( - Boolean.TRUE.equals(this.enableGcpTraceExporter), - Boolean.TRUE.equals(this.enableGcpLogExporter), - this.customOpenTelemetry, - this.gcpTelemetryCredentials, - this.gcpTelemetryProjectId); + OpenTelemetry openTelemetry = getOpenTelemetryInstance(); if (Boolean.TRUE.equals(this.enableGcpTraceExporter) || this.customOpenTelemetry != null) { bigQueryReadSettings.setOpenTelemetryTracerProvider(openTelemetry.getTracerProvider()); } From d32ae4f73aa9abe65a87fe98e35094aeff6143d0 Mon Sep 17 00:00:00 2001 From: Keshav Dandeva Date: Tue, 12 May 2026 00:34:16 +0000 Subject: [PATCH 7/7] feat: add log exporter to gcp --- .../bigquery/jdbc/BigQueryConnection.java | 66 ++++++--- .../jdbc/BigQueryJdbcOpenTelemetry.java | 133 +++++++++++++----- .../jdbc/OpenTelemetryJulHandler.java | 10 +- 3 files changed, 146 insertions(+), 63 deletions(-) diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index 917be139cbde..1a4e2206765c 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -41,6 +41,7 @@ import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.cloud.bigquery.storage.v1.BigQueryWriteSettings; import com.google.cloud.http.HttpTransportOptions; +import com.google.cloud.logging.Logging; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Tracer; import java.io.IOException; @@ -147,6 +148,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { Boolean enableGcpTraceExporter; Boolean enableGcpLogExporter; OpenTelemetry customOpenTelemetry; + private OpenTelemetry openTelemetry; Tracer tracer = OpenTelemetry.noop().getTracer(BigQueryJdbcOpenTelemetry.INSTRUMENTATION_SCOPE_NAME); DatabaseMetaData databaseMetaData; @@ -281,6 +283,7 @@ public class BigQueryConnection extends BigQueryNoOpsConnection { this.enableGcpTraceExporter = ds.getEnableGcpTraceExporter(); this.enableGcpLogExporter = ds.getEnableGcpLogExporter(); this.customOpenTelemetry = ds.getCustomOpenTelemetry(); + this.openTelemetry = getOpenTelemetryInstance(); this.bigQuery = getBigQueryConnection(); } } @@ -966,29 +969,56 @@ void removeStatement(Statement statement) { } private OpenTelemetry getOpenTelemetryInstance() { + boolean isTraceEnabled = Boolean.TRUE.equals(this.enableGcpTraceExporter); + boolean isLogEnabled = Boolean.TRUE.equals(this.enableGcpLogExporter); + boolean hasCustomOtel = this.customOpenTelemetry != null; + String effectiveProjectId = (this.gcpTelemetryProjectId != null) ? this.gcpTelemetryProjectId : this.catalog; + String effectiveCredentials = resolveEffectiveCredentials(); + + validateTraceConfiguration(isTraceEnabled, effectiveCredentials); + + OpenTelemetry openTelemetry = + BigQueryJdbcOpenTelemetry.getOpenTelemetry( + isTraceEnabled, + isLogEnabled, + this.customOpenTelemetry, + effectiveCredentials, + effectiveProjectId); + + Logging localLoggingClient = null; + if (isLogEnabled && !hasCustomOtel) { + localLoggingClient = + BigQueryJdbcOpenTelemetry.createLoggingClient( + true, null, effectiveCredentials, effectiveProjectId, this.credentials); + } - String effectiveCredentials = this.gcpTelemetryCredentials; - String authTypeStr = this.overrideProperties.get("OAuthType"); + if (isLogEnabled || hasCustomOtel) { + BigQueryJdbcOpenTelemetry.registerConnection( + this.connectionId, openTelemetry, localLoggingClient, isLogEnabled); + } + + return openTelemetry; + } - if (effectiveCredentials == null && "0".equals(authTypeStr)) { - effectiveCredentials = this.overrideProperties.get("OAuthPvtKey"); + private String resolveEffectiveCredentials() { + String creds = this.gcpTelemetryCredentials; + String authTypeStr = this.overrideProperties.get("OAuthType"); + if (creds == null && "0".equals(authTypeStr)) { + return this.overrideProperties.get("OAuthPvtKey"); } + return creds; + } - if (Boolean.TRUE.equals(this.enableGcpTraceExporter) && effectiveCredentials == null) { + private void validateTraceConfiguration(boolean isTraceEnabled, String effectiveCredentials) { + if (isTraceEnabled && effectiveCredentials == null) { + String authTypeStr = this.overrideProperties.get("OAuthType"); if (!"0".equals(authTypeStr) && !"3".equals(authTypeStr)) { throw new BigQueryJdbcRuntimeException( "Exporting traces to Google Cloud is only supported when using Application Default Credentials (ADC) or Service Account authentication."); } } - - return BigQueryJdbcOpenTelemetry.getOpenTelemetry( - Boolean.TRUE.equals(this.enableGcpTraceExporter), - Boolean.TRUE.equals(this.enableGcpLogExporter), - this.customOpenTelemetry, - effectiveCredentials, - effectiveProjectId); } private BigQuery getBigQueryConnection() { @@ -1027,15 +1057,8 @@ private BigQuery getBigQueryConnection() { if (this.httpTransportOptions != null) { bigQueryOptions.setTransportOptions(this.httpTransportOptions); } - OpenTelemetry openTelemetry = getOpenTelemetryInstance(); - - if (Boolean.TRUE.equals(this.enableGcpLogExporter) || this.customOpenTelemetry != null) { - BigQueryJdbcOpenTelemetry.registerConnection( - this.connectionId, openTelemetry, null, this.enableGcpLogExporter); - } - if (Boolean.TRUE.equals(this.enableGcpTraceExporter) || this.customOpenTelemetry != null) { - this.tracer = BigQueryJdbcOpenTelemetry.getTracer(openTelemetry); + this.tracer = BigQueryJdbcOpenTelemetry.getTracer(this.openTelemetry); bigQueryOptions.setOpenTelemetryTracer(this.tracer); } @@ -1087,9 +1110,8 @@ private BigQueryReadClient getBigQueryReadClientConnection() throws IOException bigQueryReadSettings.setTransportChannelProvider(activeProvider); - OpenTelemetry openTelemetry = getOpenTelemetryInstance(); if (Boolean.TRUE.equals(this.enableGcpTraceExporter) || this.customOpenTelemetry != null) { - bigQueryReadSettings.setOpenTelemetryTracerProvider(openTelemetry.getTracerProvider()); + bigQueryReadSettings.setOpenTelemetryTracerProvider(this.openTelemetry.getTracerProvider()); } return BigQueryReadClient.create(bigQueryReadSettings.build()); diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index fa748c606865..e1a54f63c642 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -16,7 +16,10 @@ package com.google.cloud.bigquery.jdbc; +import com.google.auth.Credentials; +import com.google.cloud.bigquery.exception.BigQueryJdbcRuntimeException; import com.google.cloud.logging.Logging; +import com.google.cloud.logging.LoggingOptions; import com.google.common.hash.Hashing; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Tracer; @@ -139,7 +142,61 @@ public static void registerConnection( } public static void unregisterConnection(String connectionId) { - connectionConfigs.remove(connectionId); + TelemetryConfig config = connectionConfigs.remove(connectionId); + if (config != null && config.loggingClient != null) { + try { + config.loggingClient.close(); + } catch (Exception e) { + LOG.warning("Failed to close Logging client during unregister: %s", e.getMessage()); + } + } + } + + public static Logging createLoggingClient( + boolean enableGcpLogExporter, + OpenTelemetry customOpenTelemetry, + String effectiveCredentials, + String effectiveProjectId, + Credentials fallbackCredentials) { + + if (enableGcpLogExporter && customOpenTelemetry == null) { + try { + Credentials credentials; + if (effectiveCredentials != null) { + credentials = resolveCredentialsFromString(effectiveCredentials); + } else { + credentials = fallbackCredentials; + } + + LoggingOptions.Builder loggingOptionsBuilder = + LoggingOptions.newBuilder().setProjectId(effectiveProjectId); + if (credentials != null) { + loggingOptionsBuilder.setCredentials(credentials); + } + return loggingOptionsBuilder.build().getService(); + } catch (Exception e) { + throw new BigQueryJdbcRuntimeException("Failed to initialize Logging client", e); + } + } + return null; + } + + private static Credentials resolveCredentialsFromString(String credsString) { + Map authProperties = new java.util.HashMap<>(); + authProperties.put(BigQueryJdbcUrlUtility.OAUTH_TYPE_PROPERTY_NAME, "0"); // Service Account + + byte[] credsBytes = credsString.getBytes(StandardCharsets.UTF_8); + if (BigQueryJdbcOAuthUtility.isJson(credsBytes)) { + authProperties.put(BigQueryJdbcUrlUtility.OAUTH_PVT_KEY_PROPERTY_NAME, credsString); + } else { + authProperties.put(BigQueryJdbcUrlUtility.OAUTH_PVT_KEY_PATH_PROPERTY_NAME, credsString); + } + + return BigQueryJdbcOAuthUtility.getCredentials( + authProperties, + new java.util.HashMap<>(), + false, + BigQueryJdbcOpenTelemetry.class.getName()); } public static TelemetryConfig getConnectionConfig(String connectionId) { @@ -171,58 +228,58 @@ public static OpenTelemetry getOpenTelemetry( OpenTelemetry customOpenTelemetry, String gcpTelemetryCredentials, String gcpTelemetryProjectId) { + if (customOpenTelemetry != null) { return customOpenTelemetry; } // NOTE: Currently, tracing only fully supports Application Default Credentials (ADC). // Once b/503721589 is completed, Service Account (SA) will work as well. + if (!enableGcpTraceExporter && !enableGcpLogExporter) { + return OpenTelemetry.noop(); + } - if (enableGcpTraceExporter || enableGcpLogExporter) { - SdkCacheKey key = - new SdkCacheKey( - gcpTelemetryProjectId, - getCredentialsIdentifier(gcpTelemetryCredentials), - enableGcpTraceExporter); - return sdkCache.computeIfAbsent( - key, - k -> { - Map props = new HashMap<>(); - if (gcpTelemetryCredentials != null) { - byte[] credsBytes = gcpTelemetryCredentials.getBytes(StandardCharsets.UTF_8); - if (BigQueryJdbcOAuthUtility.isJson(credsBytes)) { - props.put(CREDENTIALS_JSON, gcpTelemetryCredentials); - } else { - props.put(CREDENTIALS_PATH, gcpTelemetryCredentials); - } - } - - if (enableGcpTraceExporter) { - props.put(OTEL_TRACES_EXPORTER, EXPORTER_OTLP); - props.put(OTEL_EXPORTER_OTLP_ENDPOINT, OTLP_ENDPOINT_VALUE); + SdkCacheKey key = + new SdkCacheKey( + gcpTelemetryProjectId, + getCredentialsIdentifier(gcpTelemetryCredentials), + enableGcpTraceExporter); + return sdkCache.computeIfAbsent( + key, + k -> { + Map props = new HashMap<>(); + if (gcpTelemetryCredentials != null) { + byte[] credsBytes = gcpTelemetryCredentials.getBytes(StandardCharsets.UTF_8); + if (BigQueryJdbcOAuthUtility.isJson(credsBytes)) { + props.put(CREDENTIALS_JSON, gcpTelemetryCredentials); } else { - props.put(OTEL_TRACES_EXPORTER, EXPORTER_NONE); + props.put(CREDENTIALS_PATH, gcpTelemetryCredentials); } + } - // Logs are handled directly via GCP logging - props.put(OTEL_LOGS_EXPORTER, EXPORTER_NONE); - // Metrics are deferred to a future phase - props.put(OTEL_METRICS_EXPORTER, EXPORTER_NONE); + if (enableGcpTraceExporter) { + props.put(OTEL_TRACES_EXPORTER, EXPORTER_OTLP); + props.put(OTEL_EXPORTER_OTLP_ENDPOINT, OTLP_ENDPOINT_VALUE); + } else { + props.put(OTEL_TRACES_EXPORTER, EXPORTER_NONE); + } - if (gcpTelemetryProjectId != null) { - props.put(GOOGLE_CLOUD_PROJECT, gcpTelemetryProjectId); - } + // Logs are handled directly via GCP logging + props.put(OTEL_LOGS_EXPORTER, EXPORTER_NONE); + // Metrics are deferred to a future phase + props.put(OTEL_METRICS_EXPORTER, EXPORTER_NONE); - AutoConfiguredOpenTelemetrySdk autoConfigured = - AutoConfiguredOpenTelemetrySdk.builder().addPropertiesSupplier(() -> props).build(); + if (gcpTelemetryProjectId != null) { + props.put(GOOGLE_CLOUD_PROJECT, gcpTelemetryProjectId); + } - OpenTelemetrySdk sdk = autoConfigured.getOpenTelemetrySdk(); + AutoConfiguredOpenTelemetrySdk autoConfigured = + AutoConfiguredOpenTelemetrySdk.builder().addPropertiesSupplier(() -> props).build(); - return sdk; - }); - } + OpenTelemetrySdk sdk = autoConfigured.getOpenTelemetrySdk(); - return OpenTelemetry.noop(); + return sdk; + }); } /** Gets a Tracer for the JDBC driver instrumentation scope. */ diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java index b6be12f972af..722487c14ea4 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java @@ -85,12 +85,16 @@ private void publishToGcp(LogRecord record, String connectionId, Logging logging String traceId = spanContext.isValid() ? spanContext.getTraceId() : null; String spanId = spanContext.isValid() ? spanContext.getSpanId() : null; - // TODO(b/491238299): May require refinement for structured logging or error handling + String logId = record.getLoggerName(); + if (logId == null) { + logId = BigQueryJdbcOpenTelemetry.INSTRUMENTATION_SCOPE_NAME; + } LogEntry.Builder builder = LogEntry.newBuilder(Payload.StringPayload.of(formatMessage(record))) .setSeverity(mapGcpSeverity(record.getLevel())) - .setTimestamp(record.getMillis()); + .setTimestamp(record.getMillis()) + .setLogName(logId); if (traceId != null) { builder.setTrace(traceId); @@ -181,6 +185,6 @@ public void flush() { @Override public void close() throws SecurityException { - // TODO(b/491238299): Implement with gcp exporter logic + flush(); } }