Skip to content

Commit 8c966fc

Browse files
authored
[translator/azurelogs] Improve performance (#39340)
#### Description This PR is only to improve performance, it does not change any functionality or any output as you can see by the passing unit tests. These are the main changes: - Iterate over the azure logs only one time. Previously we had a slice for the keys, and a map that store all logs corresponding to the same resource id. - Remove the map `mappings`. It is expensive to look up the field this way. As an alternative, we now have a function that checks the field, and adds that to the attribute. This function is used from the beginning, as we know the category right away. - Use config fastest for `jsoniter` and borrow iterator. Results: ``` goos: linux goarch: amd64 pkg: github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azurelogs cpu: 11th Gen Intel(R) Core(TM) i7-11800H @ 2.30GHz │ before │ this PR │ │ sec/op │ sec/op vs base │ UnmarshalLogs/1000_record-16 2.226m ± 2% 1.590m ± 6% -28.56% (p=0.000 n=10) UnmarshalLogs/1_record-16 2.890µ ± 1% 2.155µ ± 1% -25.45% (p=0.000 n=10) UnmarshalLogs/100_record-16 217.9µ ± 2% 155.4µ ± 1% -28.69% (p=0.000 n=10) geomean 111.9µ 81.05µ -27.58% │ before │ this PR │ │ B/op │ B/op vs base │ UnmarshalLogs/1000_record-16 2.093Mi ± 0% 1.293Mi ± 0% -38.24% (p=0.000 n=10) UnmarshalLogs/1_record-16 2.484Ki ± 0% 1.506Ki ± 0% -39.39% (p=0.000 n=10) UnmarshalLogs/100_record-16 216.0Ki ± 0% 144.9Ki ± 0% -32.91% (p=0.000 n=10) geomean 104.8Ki 66.11Ki -36.91% │ before │ this PR │ │ allocs/op │ allocs/op vs base │ UnmarshalLogs/1000_record-16 38.05k ± 0% 20.03k ± 0% -47.36% (p=0.000 n=10) UnmarshalLogs/1_record-16 52.00 ± 0% 31.00 ± 0% -40.38% (p=0.000 n=10) UnmarshalLogs/100_record-16 3.835k ± 0% 2.025k ± 0% -47.20% (p=0.000 n=10) geomean 1.965k 1.079k -45.07% ``` Performance increased in all metrics. There are still improvements that can be done, but I will not add them to this PR so it won't get too big: - We should not carry an attributes map, but instead we should add them to the record as soon as possible. These issues might also get affected by #39186 if it goes forward. <!-- Issue number (e.g. #1234) or full URL to issue, if applicable. --> #### Link to tracking issue Relates #39119. <!--Describe what testing was performed and which tests were added.--> #### Testing Unit tests and benchmark.
1 parent e4b7066 commit 8c966fc

10 files changed

+460
-587
lines changed
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Use this changelog template to create an entry for release notes.
2+
3+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
4+
change_type: enhancement
5+
6+
# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
7+
component: pkg
8+
9+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
10+
note: Improve performance of azure logs translator.
11+
12+
# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
13+
issues: [39340]
14+
15+
# (Optional) One or more lines of additional information to render under the primary note.
16+
# These lines will be padded with 2 spaces and then inserted directly into the document.
17+
# Use pipe (|) for multiline entries.
18+
subtext:
19+
20+
# If your change doesn't affect end users or the exported elements of any package,
21+
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
22+
# Optional: The change log or logs in which this entry should be included.
23+
# e.g. '[user]' or '[user, api]'
24+
# Include 'user' if the change is relevant to end users.
25+
# Include 'api' if there is a change to a library API.
26+
# Default: '[user]'
27+
change_logs: []

pkg/translator/azurelogs/benchmark_test.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import (
1212
"go.uber.org/zap"
1313
)
1414

15-
func createBuf(b *testing.B, nRecords int) []byte {
15+
// newBuf creates an azure resource log with
16+
// as many records as defined in nRecords
17+
func newBuf(b *testing.B, nRecords int) []byte {
1618
rec := azureLogRecord{
1719
Time: "2024-04-24T12:06:12.0000000Z",
1820
ResourceID: "/test",
@@ -56,11 +58,12 @@ func BenchmarkUnmarshalLogs(b *testing.B) {
5658
Logger: zap.NewNop(),
5759
}
5860
for name, test := range tests {
59-
buf := createBuf(b, test.nRecords)
61+
buf := newBuf(b, test.nRecords)
6062
b.Run(name, func(b *testing.B) {
6163
b.ReportAllocs()
6264
for i := 0; i < b.N; i++ {
63-
_, _ = u.UnmarshalLogs(buf)
65+
_, err := u.UnmarshalLogs(buf)
66+
require.NoError(b, err)
6467
}
6568
})
6669
}

pkg/translator/azurelogs/complex_conversions.go

Lines changed: 0 additions & 107 deletions
This file was deleted.

pkg/translator/azurelogs/complex_conversions_test.go

Lines changed: 0 additions & 76 deletions
This file was deleted.

pkg/translator/azurelogs/go.mod

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ require (
1414
go.opentelemetry.io/collector/semconv v0.124.0
1515
go.uber.org/goleak v1.3.0
1616
go.uber.org/zap v1.27.0
17-
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
1817
)
1918

2019
require (

pkg/translator/azurelogs/go.sum

Lines changed: 0 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/translator/azurelogs/normalize.go

Lines changed: 24 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,35 @@
44
package azurelogs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azurelogs"
55

66
import (
7-
"fmt"
87
"strconv"
98
"strings"
109
)
1110

12-
func normalizeValue(key string, val any) any {
13-
switch key {
14-
case
15-
"http.request.body.size",
16-
"http.request.size",
17-
"http.response.body.size",
18-
"http.response.size",
19-
"http.response.status_code",
20-
"server.port":
21-
return toInt(val)
22-
case "http.server.request.duration":
23-
return toFloat(val)
24-
case "network.protocol.name":
25-
return toLower(val)
11+
func tryParseFloat64(value any) (float64, bool) {
12+
switch v := value.(type) {
13+
case float32:
14+
return float64(v), true
15+
case float64:
16+
return v, true
17+
case int:
18+
return float64(v), true
19+
case int32:
20+
return float64(v), true
21+
case int64:
22+
return float64(v), true
23+
case string:
24+
f, err := strconv.ParseFloat(v, 64)
25+
return f, err == nil
26+
default:
27+
return 0, false
2628
}
27-
return val
2829
}
2930

3031
func toLower(value any) any {
31-
switch v := value.(type) {
32-
case string:
33-
return strings.ToLower(v)
34-
default:
35-
return strings.ToLower(fmt.Sprint(value))
32+
if s, ok := value.(string); ok {
33+
return strings.ToLower(s)
3634
}
35+
return value
3736
}
3837

3938
func toFloat(value any) any {
@@ -49,8 +48,7 @@ func toFloat(value any) any {
4948
case int64:
5049
return float64(v)
5150
case string:
52-
f, err := strconv.ParseFloat(v, 64)
53-
if err == nil {
51+
if f, err := strconv.ParseFloat(v, 64); err == nil {
5452
return f
5553
}
5654
}
@@ -62,12 +60,11 @@ func toInt(value any) any {
6260
case int:
6361
return int64(v)
6462
case int32:
65-
return int64(int(v))
63+
return int64(v)
6664
case int64:
67-
return value.(int64)
65+
return v
6866
case string:
69-
i, err := strconv.ParseInt(v, 10, 64)
70-
if err == nil {
67+
if i, err := strconv.ParseInt(v, 10, 64); err == nil {
7168
return i
7269
}
7370
}

0 commit comments

Comments
 (0)