fix: work around AWS S3 limits

Fixes issue #2627

We get/put metadata in dynamodb and it appears there are limits enforced
by AWS S3 APIs.

https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html

If you request more than 100 items, BatchGetItem returns a
ValidationException with the message "Too many items requested for the
BatchGetItem call."

Signed-off-by: Ramkumar Chinchani <[email protected]>

Author: Ramkumar Chinchani

pkg/meta/dynamodb/dynamodb.go

@@ -1970,24 +1970,39 @@ func (dwr DynamoDB) DeleteUserData(ctx context.Context) error {
 	return err
 }
 
+const AwsS3BatchLimit = 100
+
 func (dwr *DynamoDB) fetchImageMetaAttributesByDigest(ctx context.Context, digests []string,
 ) ([]map[string]types.AttributeValue, error) {
-	resp, err := dwr.Client.BatchGetItem(ctx, &dynamodb.BatchGetItemInput{
-		RequestItems: map[string]types.KeysAndAttributes{
-			dwr.ImageMetaTablename: {
-				Keys: getBatchImageKeys(digests),
+	// AWS S3 as a limit (=100) on number of keys that can retrieved in one
+	// request, so break it up
+	batchedResp := []map[string]types.AttributeValue{}
+
+	var end int
+
+	for start := 0; start < len(digests); {
+		end = min(len(digests)-start, AwsS3BatchLimit)
+
+		resp, err := dwr.Client.BatchGetItem(ctx, &dynamodb.BatchGetItemInput{
+			RequestItems: map[string]types.KeysAndAttributes{
+				dwr.ImageMetaTablename: {
+					Keys: getBatchImageKeys(digests[start:end]),
+				},
 			},
-		},
-	})
-	if err != nil {
-		return nil, err
-	}
+		})
+		if err != nil {
+			return nil, err
+		}
 
-	if len(resp.Responses[dwr.ImageMetaTablename]) != len(digests) {
-		return nil, zerr.ErrImageMetaNotFound
+		if len(resp.Responses[dwr.ImageMetaTablename]) != (end - start) {
+			return nil, zerr.ErrImageMetaNotFound
+		}
+
+		batchedResp = append(batchedResp, resp.Responses[dwr.ImageMetaTablename]...)
+		start = end
 	}
 
-	return resp.Responses[dwr.ImageMetaTablename], nil
+	return batchedResp, nil
 }
 
 func getBatchImageKeys(digests []string) []map[string]types.AttributeValue {