Skip to content
This repository was archived by the owner on Apr 2, 2024. It is now read-only.

Commit 20d59ef

Browse files
alejandrodnmalejandrodnm
committed
Support for Jaeger tags with binary value
Since we are using the OTEL translator, when ingesting Jaeger traces all the tags with `VType=ValueType_BINARY` are being encoded into base64 and stored as strings. The context of the type of value is lost after this translation occurs, meaning that, when we return those tags in a Jaeger query we return the base64 string as `VType=ValueType_STRING`. To preserve the type of the value, the prefix `data:application/octet-stream; base64,` will be added to the base64 string before storing in the DB. When retrieving a trace we look for tags that are string and have this prefix, we remove it and return a `ValueType_BINARY` with the result of decoding the base64 string. The prefix was choosen from the `The "data" URL scheme` RFC https://www.rfc-editor.org/rfc/rfc2397#section-2 . Jaeger allows these type of tags for Spans, Process and Logs.
1 parent 4aa9a58 commit 20d59ef

File tree

5 files changed

+280
-0
lines changed

5 files changed

+280
-0
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ We use the following categories for changes:
3030
- Log throughput in the same line for samples, spans and metric metadata [#1643]
3131
- The `chunks_created` metrics was removed. [#1634]
3232
- Stop logging as an error grpc NotFound and Canceled status codes [#1645]
33+
- When querying for Jaeger tags with binary values the binary data will be
34+
returned instead of the base64 representation of the string [#1649].
3335

3436
### Fixed
3537
- Do not collect telemetry if `timescaledb.telemetry_level=off` [#1612]

‎pkg/jaeger/store/binary_tags.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
package store
2+
3+
import (
4+
"encoding/base64"
5+
"fmt"
6+
"strings"
7+
8+
"github.com/jaegertracing/jaeger/model"
9+
)
10+
11+
const MEDIA_TYPE_ENCODED_BINARY = "data:application/octet-stream; base64,"
12+
const MEDIA_TYPE_ENCODED_BINARY_LEN = len(MEDIA_TYPE_ENCODED_BINARY)
13+
14+
// decodeSpanBinaryTags decodes the tags with binary values that are present in
15+
// the binaryTags sets for the span, process and logs.
16+
//
17+
// When writing binary tags we encode the slice of bytes into a base64 string
18+
// representation and add the prefix `__ValueType_BINARY__`. Decoding implies
19+
// removing the prefix and decoding the base64 string.
20+
func decodeSpanBinaryTags(span *model.Span) {
21+
decodeBinaryTags(span.Tags)
22+
decodeBinaryTags(span.Process.Tags)
23+
for _, log := range span.Logs {
24+
decodeBinaryTags(log.Fields)
25+
}
26+
}
27+
28+
func decodeBinaryTags(tags []model.KeyValue) {
29+
for i, tag := range tags {
30+
if tag.GetVType() != model.ValueType_STRING {
31+
continue
32+
}
33+
34+
encoded := tag.VStr
35+
if !strings.HasPrefix(encoded, MEDIA_TYPE_ENCODED_BINARY) {
36+
continue
37+
}
38+
39+
vBin, err := decodeBinaryTagValue(encoded)
40+
41+
// If we can't decode it means that we didn't encode it in the
42+
// first place, so we should keep it as is.
43+
if err != nil {
44+
continue
45+
}
46+
tags[i] = model.KeyValue{
47+
Key: tag.Key,
48+
VType: model.ValueType_BINARY,
49+
VBinary: vBin,
50+
}
51+
}
52+
}
53+
54+
func decodeBinaryTagValue(encoded string) ([]byte, error) {
55+
v := encoded[MEDIA_TYPE_ENCODED_BINARY_LEN:]
56+
return base64.StdEncoding.DecodeString(v)
57+
}
58+
59+
func encodeBinaryTagToStr(tag model.KeyValue) model.KeyValue {
60+
value := fmt.Sprintf("%s%s", MEDIA_TYPE_ENCODED_BINARY, base64.StdEncoding.EncodeToString(tag.GetVBinary()))
61+
return model.KeyValue{
62+
Key: tag.Key,
63+
VType: model.ValueType_STRING,
64+
VStr: value,
65+
}
66+
}
67+
68+
func encodeBinaryTags(span *model.Span) {
69+
for i, tag := range span.Tags {
70+
if !isBinaryTag(tag) {
71+
continue
72+
}
73+
span.Tags[i] = encodeBinaryTagToStr(tag)
74+
}
75+
76+
for _, log := range span.Logs {
77+
for i, tag := range log.Fields {
78+
if !isBinaryTag(tag) {
79+
continue
80+
}
81+
log.Fields[i] = encodeBinaryTagToStr(tag)
82+
}
83+
}
84+
85+
for i, tag := range span.Process.Tags {
86+
if !isBinaryTag(tag) {
87+
continue
88+
}
89+
span.Process.Tags[i] = encodeBinaryTagToStr(tag)
90+
}
91+
}
92+
93+
func isBinaryTag(tag model.KeyValue) bool {
94+
return tag.GetVType() == model.ValueType_BINARY
95+
}

‎pkg/jaeger/store/binary_tags_test.go

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
package store
2+
3+
import (
4+
"encoding/base64"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/jaegertracing/jaeger/model"
9+
"github.com/stretchr/testify/assert"
10+
)
11+
12+
var binaryValue1 = []byte{66, 105, 110, 97, 114, 121}
13+
var binaryValue2 = []byte{66, 105, 110, 97, 114, 121}
14+
15+
func keyValuesFixture(prefix string) []model.KeyValue {
16+
return []model.KeyValue{
17+
{
18+
Key: fmt.Sprintf("%s-binary1-data", prefix),
19+
VBinary: binaryValue1,
20+
VType: model.ValueType_BINARY,
21+
},
22+
{
23+
Key: fmt.Sprintf("%s-string-data", prefix),
24+
VStr: "My string",
25+
VType: model.ValueType_STRING,
26+
},
27+
{
28+
Key: fmt.Sprintf("%s-int64-data", prefix),
29+
VInt64: 42,
30+
VType: model.ValueType_INT64,
31+
},
32+
{
33+
Key: fmt.Sprintf("%s-binary2-data", prefix),
34+
VBinary: binaryValue2,
35+
VType: model.ValueType_BINARY,
36+
},
37+
{
38+
Key: fmt.Sprintf("%s-float64-data", prefix),
39+
VFloat64: 42.42,
40+
VType: model.ValueType_FLOAT64,
41+
},
42+
{
43+
Key: fmt.Sprintf("%s-bool-data", prefix),
44+
VBool: true,
45+
VType: model.ValueType_BOOL,
46+
},
47+
}
48+
}
49+
50+
func getExpectedStrV(key string, binaryValue []byte) model.KeyValue {
51+
return model.KeyValue{
52+
Key: key,
53+
VStr: fmt.Sprintf("%s%s", MEDIA_TYPE_ENCODED_BINARY, base64.StdEncoding.EncodeToString(binaryValue)),
54+
VType: model.ValueType_STRING,
55+
}
56+
}
57+
58+
func assertEncoded(t *testing.T, prefix string, encodedTags []model.KeyValue) {
59+
// Binary values are at position 0 and 3
60+
key1 := fmt.Sprintf("%s-binary1-data", prefix)
61+
assert.Equal(t, getExpectedStrV(key1, binaryValue1), encodedTags[0])
62+
63+
key2 := fmt.Sprintf("%s-binary2-data", prefix)
64+
assert.Equal(t, getExpectedStrV(key2, binaryValue2), encodedTags[3])
65+
}
66+
67+
func assertOnlyBinariesModified(t *testing.T, binaryVIdx map[int]struct{}, expected []model.KeyValue, actual []model.KeyValue) {
68+
assert.Equal(t, len(expected), len(actual))
69+
for i, tag := range actual {
70+
_, isBinary := binaryVIdx[i]
71+
if isBinary {
72+
assert.NotEqual(t, expected[i], tag)
73+
} else {
74+
assert.Equal(t, expected[i], tag)
75+
}
76+
}
77+
78+
}
79+
80+
func TestEncodeBinaryTag(t *testing.T) {
81+
logs := []model.Log{
82+
{
83+
Fields: keyValuesFixture("log1"),
84+
},
85+
{
86+
Fields: keyValuesFixture("log2"),
87+
},
88+
}
89+
process := model.Process{
90+
Tags: keyValuesFixture("process"),
91+
}
92+
span := model.Span{
93+
Tags: keyValuesFixture("span"),
94+
Process: &process,
95+
Logs: logs,
96+
}
97+
98+
encodeBinaryTags(&span)
99+
100+
// All the binary items are in the same index position because we are using
101+
// the same []KeyValue fixture for span, process and logs;
102+
binaryVIdxs := map[int]struct{}{0: {}, 3: {}}
103+
104+
assertEncoded(t, "span", span.Tags)
105+
assertOnlyBinariesModified(t, binaryVIdxs, keyValuesFixture("span"), span.Tags)
106+
assertEncoded(t, "process", span.Process.Tags)
107+
assertOnlyBinariesModified(t, binaryVIdxs, keyValuesFixture("process"), span.Process.Tags)
108+
assertEncoded(t, "log1", span.Logs[0].Fields)
109+
assertOnlyBinariesModified(t, binaryVIdxs, keyValuesFixture("log1"), span.Logs[0].Fields)
110+
assertEncoded(t, "log2", span.Logs[1].Fields)
111+
assertOnlyBinariesModified(t, binaryVIdxs, keyValuesFixture("log2"), span.Logs[1].Fields)
112+
}
113+
114+
func keyValuesEncodedFixture() []model.KeyValue {
115+
return []model.KeyValue{
116+
{
117+
Key: "binary1-data",
118+
VStr: fmt.Sprintf("%s%s", MEDIA_TYPE_ENCODED_BINARY, base64.StdEncoding.EncodeToString(binaryValue1)),
119+
VType: model.ValueType_STRING,
120+
},
121+
{
122+
Key: "string-data",
123+
VStr: "My string",
124+
VType: model.ValueType_STRING,
125+
},
126+
{
127+
Key: "int64-data",
128+
VInt64: 42,
129+
VType: model.ValueType_INT64,
130+
},
131+
{
132+
Key: "binary2-data",
133+
VStr: fmt.Sprintf("%s%s", MEDIA_TYPE_ENCODED_BINARY, base64.StdEncoding.EncodeToString(binaryValue2)),
134+
VType: model.ValueType_STRING,
135+
},
136+
{
137+
Key: "float64-data",
138+
VFloat64: 42.42,
139+
VType: model.ValueType_FLOAT64,
140+
},
141+
{
142+
Key: "bool-data",
143+
VBool: true,
144+
VType: model.ValueType_BOOL,
145+
},
146+
{
147+
Key: "no-prefix",
148+
VStr: base64.StdEncoding.EncodeToString(binaryValue1),
149+
VType: model.ValueType_STRING,
150+
},
151+
{
152+
Key: "no-binary-with-prefix",
153+
VStr: MEDIA_TYPE_ENCODED_BINARY + "a normal string tag",
154+
VType: model.ValueType_STRING,
155+
},
156+
}
157+
}
158+
159+
func TestDecodeBinaryTags(t *testing.T) {
160+
fixture := keyValuesEncodedFixture()
161+
decodeBinaryTags(fixture)
162+
163+
expectedBinaryV := model.KeyValue{
164+
Key: "binary1-data",
165+
VBinary: binaryValue1,
166+
VType: model.ValueType_BINARY,
167+
}
168+
expectedBinaryV2 := model.KeyValue{
169+
Key: "binary2-data",
170+
VBinary: binaryValue2,
171+
VType: model.ValueType_BINARY,
172+
}
173+
174+
assert.Equal(t, expectedBinaryV, fixture[0])
175+
assert.Equal(t, expectedBinaryV2, fixture[3])
176+
177+
// Only the string values that have the MEDIA_TYPE_ENCODED_BINARY prefix
178+
// and are followed by a valid base64 encoded string are modified.
179+
binaryVIdxs := map[int]struct{}{0: {}, 3: {}}
180+
assertOnlyBinariesModified(t, binaryVIdxs, keyValuesEncodedFixture(), fixture)
181+
}

‎pkg/jaeger/store/find_traces.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ func batchSliceToTraceSlice(bSlice []*model.Batch) []*model.Trace {
7474
}
7575
//copy over the process from the batch
7676
span.Process = batch.Process
77+
decodeSpanBinaryTags(span)
7778
trace.Spans = append(trace.Spans, span)
7879
}
7980
}

‎pkg/jaeger/store/store.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ func (p *Store) StreamingSpanWriter() spanstore.Writer {
5050
}
5151

5252
func (p *Store) WriteSpan(ctx context.Context, span *model.Span) error {
53+
encodeBinaryTags(span)
5354
batches := []*model.Batch{
5455
{
5556
Spans: []*model.Span{span},

0 commit comments

Comments
 (0)