receiver/azuremonitor - Added maximumResourcesPerBatch config

Author: suvaidkhan

.chloggen/receiver_azuremonitor-configuration-MaximumResourcesPerBatch.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: azuremonitorreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Added new config that sets the number of unique resource IDs to fetch per Batch API call"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [40112]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

receiver/azuremonitorreceiver/README.md

@@ -38,6 +38,7 @@ The following settings are optional:
 - `dimensions.enabled` (default = `true`): allows to opt out from automatically split by all the dimensions of the resource type.
 - `dimensions.overrides` (default = `{}`): if dimensions are enabled, it allows you to specify a set of dimensions for a particular metric. This is a two levels map with first key being the resource type and second key being the metric name. Programmatic value should be used for metric name https://learn.microsoft.com/en-us/azure/azure-monitor/reference/metrics-index
 - `use_batch_api` (default = `false`): Use the batch API to fetch metrics. This is useful when the number of subscriptions is high and the API calls are rate limited.
+- `maximum_resources_per_batch` (default = 50): If batch is enabled, the maximum number of unique resource IDs to fetch per API call, current limit is 50 (as of 06/16/2025)
 
 Authenticating using service principal requires following additional settings:
 
@@ -92,7 +93,7 @@ Good news is that it's **very easy for you to try out!**
 ```yaml
 receivers:
   azuremonitor:
-    use_bath_api: true
+    use_batch_api: true
     ... # no change for other configs
 ```
 

receiver/azuremonitorreceiver/config.go

@@ -28,6 +28,7 @@ var (
 	errMissingClientSecret    = errors.New(`"ClientSecret" is not specified in config`)
 	errMissingFedTokenFile    = errors.New(`"FederatedTokenFile" is not specified in config`)
 	errInvalidCloud           = errors.New(`"Cloud" is invalid`)
+	errInvalidMaxResPerBatch  = errors.New(`"MaximumResourcesPerBatch" is invalid`)
 
 	monitorServices = []string{
 		"Microsoft.EventGrid/eventSubscriptions",
@@ -254,6 +255,7 @@ type Config struct {
 	AppendTagsAsAttributes            bool                          `mapstructure:"append_tags_as_attributes"`
 	UseBatchAPI                       bool                          `mapstructure:"use_batch_api"`
 	Dimensions                        DimensionsConfig              `mapstructure:"dimensions"`
+	MaximumResourcesPerBatch          int                           `mapstructure:"maximum_resources_per_batch"`
 
 	// Authentication accepts the component azureauthextension,
 	// and uses it to get an access token to make requests.
@@ -284,6 +286,8 @@ const (
 	managedIdentity    = "managed_identity"
 )
 
+const defaultMaximumResourcesPerBatch = 50
+
 // Validate validates the configuration by checking for missing or invalid fields
 func (c Config) Validate() (err error) {
 	if len(c.SubscriptionIDs) == 0 && !c.DiscoverSubscriptions {
@@ -329,5 +333,9 @@ func (c Config) Validate() (err error) {
 		err = multierr.Append(err, errInvalidCloud)
 	}
 
+	if c.UseBatchAPI && c.MaximumResourcesPerBatch != 0 && (c.MaximumResourcesPerBatch > defaultMaximumResourcesPerBatch || c.MaximumResourcesPerBatch < 0) {
+		err = multierr.Append(err, errInvalidMaxResPerBatch)
+	}
+
 	return
 }

receiver/azuremonitorreceiver/config_test.go

@@ -109,6 +109,14 @@ func TestLoadConfig(t *testing.T) {
 				return cfg
 			}(),
 		},
+		{
+			id:          component.NewIDWithName(metadata.Type, "max_resources_per_batch"),
+			expectedErr: errInvalidMaxResPerBatch.Error(),
+		},
+		{
+			id:          component.NewIDWithName(metadata.Type, "max_resources_per_batch_negative_value"),
+			expectedErr: errInvalidMaxResPerBatch.Error(),
+		},
 	}
 
 	for _, tt := range tests {

receiver/azuremonitorreceiver/scraper_batch.go

@@ -396,6 +396,10 @@ func (s *azureBatchScraper) storeMetricsDefinitionByType(subscriptionID string,
 
 func (s *azureBatchScraper) getBatchMetricsValues(ctx context.Context, subscriptionID, resourceType string) {
 	resType := *s.resourceTypes[subscriptionID][resourceType]
+	maxPerBatch := defaultMaximumResourcesPerBatch
+	if s.cfg.MaximumResourcesPerBatch > 0 {
+		maxPerBatch = s.cfg.MaximumResourcesPerBatch
+	}
 
 	for compositeKey, metricsByGrain := range resType.metricsByCompositeKey {
 		now := time.Now().UTC()
@@ -419,7 +423,7 @@ func (s *azureBatchScraper) getBatchMetricsValues(ctx context.Context, subscript
 
 				startResources := 0
 				for startResources < len(resType.resourceIDs) {
-					endResources := startResources + 50 // getBatch API is limited to 50 resources max
+					endResources := startResources + maxPerBatch
 					if endResources > len(resType.resourceIDs) {
 						endResources = len(resType.resourceIDs)
 					}

receiver/azuremonitorreceiver/testdata/config.yaml

@@ -40,3 +40,19 @@ azuremonitor/valid_authenticator_2:
   auth:
     authenticator: azureauth
   credentials: does-not-matter
+
+azuremonitor/max_resources_per_batch:
+  discover_subscriptions: true
+  auth:
+    authenticator: azureauth
+  credentials: does-not-matter
+  use_batch_api: true
+  maximum_resources_per_batch: 51
+
+azuremonitor/max_resources_per_batch_negative_value:
+  discover_subscriptions: true
+  auth:
+    authenticator: azureauth
+  credentials: does-not-matter
+  use_batch_api: true
+  maximum_resources_per_batch: -1