Skip to content

Commit 1b799ad

Browse files
DNVindhyaDNVindhya
authored
gcp-observability: Update logging fields for GA and use custom BatchingSettings (#9959)
This commit updates the following in gcp observability logging schema * `payload.status_code` will be of type `google.rpc.Code` instead of `uint32`. * names in enum `Address.TYPE` Use custom batching settings for [LoggingOptions](https://javadoc.io/doc/com.google.cloud/google-cloud-logging/latest/com/google/cloud/logging/LoggingOptions.html) Note: Upgraded `com.google.cloud:google-cloud-logging` from `3.6.1` to `3.14.5`.
1 parent c1ff4a8 commit 1b799ad

File tree

5 files changed

+45
-20
lines changed

5 files changed

+45
-20
lines changed

‎gcp-observability/build.gradle‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@ tasks.named("compileJava").configure {
2020
}
2121

2222
dependencies {
23-
def cloudLoggingVersion = '3.6.1'
23+
def cloudLoggingVersion = '3.14.5'
2424

2525
annotationProcessor libraries.auto.value
2626
api project(':grpc-api')
27-
27+
28+
// TODO(dnvindhya): Prefer using our own libraries, update the dependencies
29+
// in gradle/libs.versions instead
2830
implementation project(':grpc-protobuf'),
2931
project(':grpc-stub'),
3032
project(':grpc-alts'),
@@ -35,12 +37,10 @@ dependencies {
3537
libraries.opencensus.exporter.trace.stackdriver,
3638
project(':grpc-xds'), // Align grpc versions
3739
project(':grpc-services'), // Align grpc versions
38-
libraries.animalsniffer.annotations, // Prefer our version
39-
libraries.google.auth.credentials, // Prefer our version
40-
libraries.protobuf.java.util, // Prefer our version
41-
libraries.gson, // Prefer our version
42-
libraries.perfmark.api, // Prefer our version
43-
libraries.re2j, // Prefer our version
40+
('com.google.protobuf:protobuf-java:3.21.12'),
41+
('com.google.api.grpc:proto-google-common-protos:2.14.2'),
42+
('com.google.auth:google-auth-library-oauth2-http:1.16.0'),
43+
('io.opencensus:opencensus-api:0.31.1'),
4444
('com.google.guava:guava:31.1-jre')
4545

4646
runtimeOnly libraries.opencensus.impl

‎gcp-observability/src/main/java/io/grpc/gcp/observability/interceptors/LogHelper.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.google.common.base.Joiner;
2424
import com.google.protobuf.ByteString;
2525
import com.google.protobuf.Duration;
26+
import com.google.rpc.Code;
2627
import io.grpc.Attributes;
2728
import io.grpc.Deadline;
2829
import io.grpc.Grpc;
@@ -182,7 +183,7 @@ void logTrailer(
182183

183184
PayloadBuilderHelper<Payload.Builder> pair =
184185
createMetadataProto(metadata, maxHeaderBytes);
185-
pair.payloadBuilder.setStatusCode(status.getCode().value());
186+
pair.payloadBuilder.setStatusCode(Code.forNumber(status.getCode().value()));
186187
String statusDescription = status.getDescription();
187188
if (statusDescription != null) {
188189
pair.payloadBuilder.setStatusMessage(statusDescription);
@@ -404,10 +405,10 @@ static Address socketAddressToProto(SocketAddress address) {
404405
if (address instanceof InetSocketAddress) {
405406
InetAddress inetAddress = ((InetSocketAddress) address).getAddress();
406407
if (inetAddress instanceof Inet4Address) {
407-
builder.setType(Address.Type.TYPE_IPV4)
408+
builder.setType(Address.Type.IPV4)
408409
.setAddress(InetAddressUtil.toAddrString(inetAddress));
409410
} else if (inetAddress instanceof Inet6Address) {
410-
builder.setType(Address.Type.TYPE_IPV6)
411+
builder.setType(Address.Type.IPV6)
411412
.setAddress(InetAddressUtil.toAddrString(inetAddress));
412413
} else {
413414
logger.log(Level.SEVERE, "unknown type of InetSocketAddress: {}", address);
@@ -417,7 +418,7 @@ static Address socketAddressToProto(SocketAddress address) {
417418
} else if (address.getClass().getName().equals("io.netty.channel.unix.DomainSocketAddress")) {
418419
// To avoid a compiled time dependency on grpc-netty, we check against the
419420
// runtime class name.
420-
builder.setType(Address.Type.TYPE_UNIX)
421+
builder.setType(Address.Type.UNIX)
421422
.setAddress(address.toString());
422423
} else {
423424
builder.setType(Address.Type.TYPE_UNKNOWN).setAddress(address.toString());

‎gcp-observability/src/main/java/io/grpc/gcp/observability/logging/GcpLogSink.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,15 @@
1818

1919
import static com.google.common.base.Preconditions.checkNotNull;
2020

21+
import com.google.api.gax.batching.BatchingSettings;
22+
import com.google.api.gax.batching.FlowController;
2123
import com.google.cloud.MonitoredResource;
2224
import com.google.cloud.logging.LogEntry;
2325
import com.google.cloud.logging.Logging;
2426
import com.google.cloud.logging.LoggingOptions;
2527
import com.google.cloud.logging.Payload.JsonPayload;
2628
import com.google.cloud.logging.Severity;
29+
import com.google.cloud.logging.v2.stub.LoggingServiceV2StubSettings;
2730
import com.google.common.annotations.VisibleForTesting;
2831
import com.google.common.base.Strings;
2932
import com.google.common.collect.ImmutableMap;
@@ -41,6 +44,7 @@
4144
import java.util.Set;
4245
import java.util.logging.Level;
4346
import java.util.logging.Logger;
47+
import org.threeten.bp.Duration;
4448

4549
/**
4650
* Sink for Google Cloud Logging.
@@ -102,6 +106,7 @@ public void write(GrpcLogRecord logProto) {
102106
if (servicesToExclude.contains(logProto.getServiceName())) {
103107
return;
104108
}
109+
LogEntry grpcLogEntry = null;
105110
try {
106111
GrpcLogRecord.EventType eventType = logProto.getType();
107112
// TODO(DNVindhya): make sure all (int, long) values are not displayed as double
@@ -117,11 +122,18 @@ public void write(GrpcLogRecord logProto) {
117122
if (!customTags.isEmpty()) {
118123
grpcLogEntryBuilder.setLabels(customTags);
119124
}
120-
LogEntry grpcLogEntry = grpcLogEntryBuilder.build();
125+
grpcLogEntry = grpcLogEntryBuilder.build();
121126
synchronized (this) {
122127
logger.log(Level.FINEST, "Writing gRPC event : {0} to Cloud Logging", eventType);
123128
gcpLoggingClient.write(Collections.singleton(grpcLogEntry));
124129
}
130+
} catch (FlowController.FlowControlRuntimeException e) {
131+
String grpcLogEntryString = null;
132+
if (grpcLogEntry != null) {
133+
grpcLogEntryString = grpcLogEntry.toStructuredJsonString();
134+
}
135+
logger.log(Level.SEVERE, "Limit exceeded while writing log entry to cloud logging");
136+
logger.log(Level.SEVERE, "Log entry = ", grpcLogEntryString);
125137
} catch (Exception e) {
126138
logger.log(Level.SEVERE, "Caught exception while writing to Cloud Logging", e);
127139
}
@@ -132,6 +144,16 @@ Logging createLoggingClient() {
132144
if (!Strings.isNullOrEmpty(projectId)) {
133145
builder.setProjectId(projectId);
134146
}
147+
BatchingSettings loggingDefaultBatchingSettings = LoggingServiceV2StubSettings.newBuilder()
148+
.writeLogEntriesSettings().getBatchingSettings();
149+
// Custom batching settings
150+
BatchingSettings grpcLoggingVBatchingSettings = loggingDefaultBatchingSettings.toBuilder()
151+
.setDelayThreshold(Duration.ofSeconds(1L)).setFlowControlSettings(
152+
loggingDefaultBatchingSettings.getFlowControlSettings().toBuilder()
153+
.setMaxOutstandingRequestBytes(52428800L) //50 MiB
154+
.setLimitExceededBehavior(FlowController.LimitExceededBehavior.ThrowException)
155+
.build()).build();
156+
builder.setBatchingSettings(grpcLoggingVBatchingSettings);
135157
return builder.build().getService();
136158
}
137159

‎gcp-observability/src/main/proto/grpc/observabilitylog/v1/observabilitylog.proto‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package grpc.observabilitylog.v1;
2020

2121
import "google/protobuf/duration.proto";
2222
import "google/protobuf/timestamp.proto";
23+
import "google/rpc/code.proto";
2324

2425
option java_multiple_files = true;
2526
option java_package = "io.grpc.observabilitylog.v1";
@@ -97,7 +98,7 @@ message Payload {
9798
// the RPC timeout value
9899
google.protobuf.Duration timeout = 2;
99100
// The gRPC status code
100-
uint32 status_code = 3;
101+
google.rpc.Code status_code = 3;
101102
// The gRPC status message
102103
string status_message = 4;
103104
// The value of the grpc-status-details-bin metadata key, if any.
@@ -115,9 +116,9 @@ message Payload {
115116
message Address {
116117
enum Type {
117118
TYPE_UNKNOWN = 0;
118-
TYPE_IPV4 = 1; // in 1.2.3.4 form
119-
TYPE_IPV6 = 2; // IPv6 canonical form (RFC5952 section 4)
120-
TYPE_UNIX = 3; // UDS string
119+
IPV4 = 1; // in 1.2.3.4 form
120+
IPV6 = 2; // IPv6 canonical form (RFC5952 section 4)
121+
UNIX = 3; // UDS string
121122
}
122123
Type type = 1;
123124
string address = 2;

‎gcp-observability/src/test/java/io/grpc/gcp/observability/interceptors/LogHelperTest.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.google.protobuf.ByteString;
3030
import com.google.protobuf.Duration;
3131
import com.google.protobuf.util.Durations;
32+
import com.google.rpc.Code;
3233
import io.grpc.Attributes;
3334
import io.grpc.Grpc;
3435
import io.grpc.Metadata;
@@ -94,7 +95,7 @@ public void socketToProto_ipv4() throws Exception {
9495
assertThat(LogHelper.socketAddressToProto(socketAddress))
9596
.isEqualTo(Address
9697
.newBuilder()
97-
.setType(Address.Type.TYPE_IPV4)
98+
.setType(Address.Type.IPV4)
9899
.setAddress("127.0.0.1")
99100
.setIpPort(12345)
100101
.build());
@@ -109,7 +110,7 @@ public void socketToProto_ipv6() throws Exception {
109110
assertThat(LogHelper.socketAddressToProto(socketAddress))
110111
.isEqualTo(Address
111112
.newBuilder()
112-
.setType(Address.Type.TYPE_IPV6)
113+
.setType(Address.Type.IPV6)
113114
.setAddress("2001:db8::2:1") // RFC 5952 section 4: ipv6 canonical form required
114115
.setIpPort(12345)
115116
.build());
@@ -454,7 +455,7 @@ public void logTrailer() throws Exception {
454455
builder.setPeer(LogHelper.socketAddressToProto(peer));
455456
builder.setPayload(
456457
builder.getPayload().toBuilder()
457-
.setStatusCode(Status.INTERNAL.getCode().value())
458+
.setStatusCode(Code.forNumber(Status.INTERNAL.getCode().value()))
458459
.setStatusMessage("test description")
459460
.build());
460461
GrpcLogRecord base = builder.build();

0 commit comments

Comments
 (0)