Skip to content

Commit 59b8596

Browse files
constanca-mdragonlord93
constanca-m
authored and
dragonlord93
committed
[translator/azurelogs] Azure logs fix resource attributes (open-telemetry#39571)
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue. Ex. Adding a feature - Explain what this achieves.--> #### Description The issue for this PR was first reported here: open-telemetry#39186. Currently, the resource logs have no attributes. This means that all records are supposedly the same, which is not true. This PR adds attributes to the resources. <!-- Issue number (e.g. open-telemetry#1234) or full URL to issue, if applicable. --> #### Link to tracking issue Relates open-telemetry#39186. <!--Describe what testing was performed and which tests were added.--> #### Testing Unit tests fixed.
1 parent 79bb7f4 commit 59b8596

19 files changed

+622
-621
lines changed

‎.chloggen/azure-logs-fix-resource-attributes.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Use this changelog template to create an entry for release notes.
2+
3+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
4+
change_type: bug_fix
5+
6+
# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
7+
component: pkg
8+
9+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
10+
note: Fix empty resource attributes in the azure logs translator
11+
12+
# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
13+
issues: [39571]
14+
15+
# (Optional) One or more lines of additional information to render under the primary note.
16+
# These lines will be padded with 2 spaces and then inserted directly into the document.
17+
# Use pipe (|) for multiline entries.
18+
subtext:
19+
20+
# If your change doesn't affect end users or the exported elements of any package,
21+
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
22+
# Optional: The change log or logs in which this entry should be included.
23+
# e.g. '[user]' or '[user, api]'
24+
# Include 'user' if the change is relevant to end users.
25+
# Include 'api' if there is a change to a library API.
26+
# Default: '[user]'
27+
change_logs: []

‎pkg/translator/azurelogs/property_names.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,12 @@ const (
1616
categoryFrontdoorWebApplicationFirewallLog = "FrontdoorWebApplicationFirewallLog"
1717
categoryAppServiceAppLogs = "AppServiceAppLogs"
1818
categoryAppServiceAuditLogs = "AppServiceAuditLogs"
19-
categoryAppServiceAuthenticationLogs = "AppServiceAuthenticationLogs"
20-
categoryAppServiceConsoleLogs = "AppServiceConsoleLogs"
21-
categoryAppServiceHTTPLogs = "AppServiceHTTPLogs"
22-
categoryAppServiceIPSecAuditLogs = "AppServiceIPSecAuditLogs"
23-
categoryAppServicePlatformLogs = "AppServicePlatformLogs"
19+
// TODO Add log and expected file to the unit tests for authentication logs
20+
categoryAppServiceAuthenticationLogs = "AppServiceAuthenticationLogs"
21+
categoryAppServiceConsoleLogs = "AppServiceConsoleLogs"
22+
categoryAppServiceHTTPLogs = "AppServiceHTTPLogs"
23+
categoryAppServiceIPSecAuditLogs = "AppServiceIPSecAuditLogs"
24+
categoryAppServicePlatformLogs = "AppServicePlatformLogs"
2425
)
2526

2627
func handleAzureCDNAccessLog(field string, value any, attrs map[string]any, attrsProps map[string]any) {

‎pkg/translator/azurelogs/resourcelogs_to_logs.go

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,6 @@ const (
2222
// Constants for OpenTelemetry Specs
2323
scopeName = "otelcol/azureresourcelogs"
2424

25-
// Constants for Azure Log Record Attributes
26-
// TODO: Remove once these are available in semconv
27-
eventNameValue = "az.resource.log"
28-
2925
// Constants for Azure Log Record body fields
3026
azureCategory = "category"
3127
azureCorrelationID = "correlation.id"
@@ -114,18 +110,17 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) {
114110
lr.SetSeverityText(log.Level.String())
115111
}
116112

117-
lr.Attributes().PutStr(conventions.AttributeCloudResourceID, log.ResourceID)
118-
lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure)
119-
lr.Attributes().PutStr(conventions.AttributeEventName, eventNameValue)
120-
121113
if err := lr.Body().FromRaw(extractRawAttributes(log)); err != nil {
122114
return plog.Logs{}, err
123115
}
124116
}
125117

126118
l := plog.NewLogs()
127-
for _, scopeLogs := range allResourceScopeLogs {
119+
for resourceID, scopeLogs := range allResourceScopeLogs {
128120
rl := l.ResourceLogs().AppendEmpty()
121+
rl.Resource().Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure)
122+
rl.Resource().Attributes().PutStr(conventions.AttributeCloudResourceID, resourceID)
123+
rl.Resource().Attributes().PutStr(conventions.AttributeEventName, "az.resource.log")
129124
scopeLogs.MoveTo(rl.ScopeLogs().AppendEmpty())
130125
}
131126

‎pkg/translator/azurelogs/resourcelogs_to_logs_test.go

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -316,12 +316,10 @@ func TestUnmarshalLogs_Files(t *testing.T) {
316316
logFilename: "log-bad-level.json",
317317
expectedFilename: "log-bad-level-expected.yaml",
318318
},
319-
// TODO Add unit test again once bug gets fixed.
320-
// Bug https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/39186#issuecomment-2798517892
321-
// "log_maximum": {
322-
// logFilename: "log-maximum.json",
323-
// expectedFilename: "log-maximum-expected.yaml",
324-
// },
319+
"log_maximum": {
320+
logFilename: "log-maximum.json",
321+
expectedFilename: "log-maximum-expected.yaml",
322+
},
325323
"log_minimum": {
326324
logFilename: "log-minimum.json",
327325
expectedFilename: "log-minimum-expected.yaml",
@@ -347,7 +345,7 @@ func TestUnmarshalLogs_Files(t *testing.T) {
347345

348346
expectedLogs, err := golden.ReadLogs(filepath.Join(expectedDir, test.expectedFilename))
349347
require.NoError(t, err)
350-
require.NoError(t, plogtest.CompareLogs(expectedLogs, logs))
348+
require.NoError(t, plogtest.CompareLogs(expectedLogs, logs, plogtest.IgnoreResourceLogsOrder()))
351349
})
352350
}
353351
}

‎pkg/translator/azurelogs/testdata/expected/access-log-expected.yaml

Lines changed: 50 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,99 +1,99 @@
11
resourceLogs:
2-
- resource: {}
2+
- resource:
3+
attributes:
4+
- key: cloud.provider
5+
value:
6+
stringValue: azure
7+
- key: cloud.resource_id
8+
value:
9+
stringValue: /RESOURCE_ID
10+
- key: event.name
11+
value:
12+
stringValue: az.resource.log
313
scopeLogs:
414
- logRecords:
5-
- attributes:
6-
- key: cloud.resource_id
7-
value:
8-
stringValue: /RESOURCE_ID
9-
- key: cloud.provider
10-
value:
11-
stringValue: azure
12-
- key: event.name
13-
value:
14-
stringValue: az.resource.log
15-
body:
15+
- body:
1616
kvlistValue:
1717
values:
18+
- key: http.request.size
19+
value:
20+
intValue: "1234"
1821
- key: tls.protocol.version
1922
value:
2023
stringValue: "1.3"
24+
- key: http.request.method
25+
value:
26+
stringValue: GET
27+
- key: az.service_request_id
28+
value:
29+
stringValue: TRACKING_REFERENCE
2130
- key: network.protocol.version
2231
value:
2332
stringValue: 1.1.0.0
24-
- key: client.port
25-
value:
26-
stringValue: "0"
27-
- key: tls.protocol.name
33+
- key: category
2834
value:
29-
stringValue: tls
35+
stringValue: AzureCdnAccessLog
3036
- key: client.address
3137
value:
3238
stringValue: 42.42.42.42
3339
- key: http.response.status_code
3440
value:
3541
intValue: "200"
36-
- key: http.request.method
42+
- key: user_agent.original
3743
value:
38-
stringValue: GET
44+
stringValue: Mozilla/5.0
45+
- key: client.port
46+
value:
47+
stringValue: "0"
3948
- key: properties
4049
value:
4150
kvlistValue:
4251
values:
43-
- key: POP
44-
value:
45-
stringValue: LON
4652
- key: isReceivedFromClient
4753
value:
4854
boolValue: false
49-
- key: Result
55+
- key: TimeTaken
5056
value:
51-
stringValue: N/A
52-
- key: RulesEngineMatchNames
57+
stringValue: "0.230"
58+
- key: BackendHostName
5359
value:
54-
arrayValue: {}
55-
- key: TimeToFirstByte
60+
stringValue: backendhost.net
61+
- key: RoutingRuleName
5662
value:
57-
stringValue: "0.420"
63+
stringValue: default-route
64+
- key: Result
65+
value:
66+
stringValue: N/A
5867
- key: SNI
5968
value:
6069
stringValue: originshield|parentcache|https|tier2
61-
- key: RoutingRuleName
70+
- key: RulesEngineMatchNames
6271
value:
63-
stringValue: default-route
72+
arrayValue: {}
6473
- key: HttpStatusDetails
6574
value:
6675
stringValue: "200"
67-
- key: TimeTaken
76+
- key: POP
6877
value:
69-
stringValue: "0.230"
70-
- key: BackendHostName
78+
stringValue: LON
79+
- key: TimeToFirstByte
7180
value:
72-
stringValue: backendhost.net
73-
- key: operation.name
74-
value:
75-
stringValue: Microsoft.AzureCdn/Profiles/AccessLog
76-
- key: http.response.size
77-
value:
78-
intValue: "12345"
81+
stringValue: "0.420"
7982
- key: url.full
8083
value:
8184
stringValue: https://test.net/
85+
- key: tls.protocol.name
86+
value:
87+
stringValue: tls
8288
- key: error.type
8389
value:
8490
stringValue: NoError
85-
- key: category
86-
value:
87-
stringValue: AzureCdnAccessLog
88-
- key: http.request.size
89-
value:
90-
intValue: "1234"
91-
- key: az.service_request_id
91+
- key: operation.name
9292
value:
93-
stringValue: TRACKING_REFERENCE
94-
- key: user_agent.original
93+
stringValue: Microsoft.AzureCdn/Profiles/AccessLog
94+
- key: http.response.size
9595
value:
96-
stringValue: Mozilla/5.0
96+
intValue: "12345"
9797
spanId: ""
9898
timeUnixNano: "1713960372000000000"
9999
traceId: ""

‎pkg/translator/azurelogs/testdata/expected/audit-logs-2-expected.yaml

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,44 @@
11
resourceLogs:
2-
- resource: {}
2+
- resource:
3+
attributes:
4+
- key: cloud.provider
5+
value:
6+
stringValue: azure
7+
- key: cloud.resource_id
8+
value:
9+
stringValue: /RESOURCE_ID
10+
- key: event.name
11+
value:
12+
stringValue: az.resource.log
313
scopeLogs:
414
- logRecords:
5-
- attributes:
6-
- key: cloud.resource_id
7-
value:
8-
stringValue: /RESOURCE_ID
9-
- key: cloud.provider
10-
value:
11-
stringValue: azure
12-
- key: event.name
13-
value:
14-
stringValue: az.resource.log
15-
body:
15+
- body:
1616
kvlistValue:
1717
values:
18+
- key: http.request.header.x-azure-fdid
19+
value:
20+
stringValue: FDID
21+
- key: http.request.header.x-fd-healthprobe
22+
value:
23+
stringValue: HEALTH_PROBE
24+
- key: http.request.header.x-forwarded-for
25+
value:
26+
stringValue: FORWARDED_FOR
1827
- key: http.request.header.x-forwarded-host
1928
value:
2029
stringValue: FORWARDED_HOST
2130
- key: client.address
2231
value:
2332
stringValue: 42.42.42.42
24-
- key: url.domain
25-
value:
26-
stringValue: HOST
2733
- key: category
2834
value:
2935
stringValue: AppServiceIPSecAuditLogs
3036
- key: operation.name
3137
value:
3238
stringValue: IPSecAuditLog
33-
- key: http.request.header.x-azure-fdid
34-
value:
35-
stringValue: FDID
36-
- key: http.request.header.x-fd-healthprobe
37-
value:
38-
stringValue: HEALTH_PROBE
39-
- key: http.request.header.x-forwarded-for
39+
- key: url.domain
4040
value:
41-
stringValue: FORWARDED_FOR
41+
stringValue: HOST
4242
spanId: ""
4343
timeUnixNano: "1713960372000000000"
4444
traceId: ""

‎pkg/translator/azurelogs/testdata/expected/audit-logs-expected.yaml

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,32 @@
11
resourceLogs:
2-
- resource: {}
2+
- resource:
3+
attributes:
4+
- key: cloud.provider
5+
value:
6+
stringValue: azure
7+
- key: cloud.resource_id
8+
value:
9+
stringValue: /SUBSCRIPTIONS/DA2DD5CC-E7BC-4DB6-94D9-0AFB3BD30577/RESOURCEGROUPS/FRETBADGER/PROVIDERS/MICROSOFT.WEB/SITES/FBEHTESTAPP
10+
- key: event.name
11+
value:
12+
stringValue: az.resource.log
313
scopeLogs:
414
- logRecords:
5-
- attributes:
6-
- key: cloud.resource_id
7-
value:
8-
stringValue: /SUBSCRIPTIONS/DA2DD5CC-E7BC-4DB6-94D9-0AFB3BD30577/RESOURCEGROUPS/FRETBADGER/PROVIDERS/MICROSOFT.WEB/SITES/FBEHTESTAPP
9-
- key: cloud.provider
10-
value:
11-
stringValue: azure
12-
- key: event.name
13-
value:
14-
stringValue: az.resource.log
15-
body:
15+
- body:
1616
kvlistValue:
1717
values:
1818
- key: operation.name
1919
value:
2020
stringValue: Authorization
21-
- key: enduser.id
22-
value:
23-
stringValue: USER_ID
2421
- key: client.address
2522
value:
2623
stringValue: 42.42.42.42
2724
- key: network.protocol.name
2825
value:
2926
stringValue: kudu
27+
- key: enduser.id
28+
value:
29+
stringValue: USER_ID
3030
- key: properties
3131
value:
3232
kvlistValue:

0 commit comments

Comments
 (0)