Query: Adds LINQ support for Multi-key Group By translation (#4857)

# Pull Request Template

## Description

This PR features the new syntax for multi-key group by queries, allowing
for grouping of multiple properties.

Full syntax: 
```CSharp

// Function Signature
public static System.Collections.Generic.IEnumerable<System.Linq.IGrouping<TKey,TElement>> GroupBy<TSource,TKey,TElement> (this System.Collections.Generic.IEnumerable<TSource> source, Func<TSource,TKey> keySelector, Func<TSource,TElement> elementSelector);

// Usage
Enumerables.GroupBy( groupByKeys => keySelectiorLambda, (keys, values) => valueSelectorLambda);
```

Example
- Grouping by single keys and projecting single value
 ```Csharp
Enumerable.GroupBy( /*keySelector*/ k => k.Id, /*valueSelector*/ (key,
values) => values.Count());
```

- Grouping by single keys and projecting multiple values
 ```Csharp
Enumerable.GroupBy( 
               /*keySelector*/ k => k.Id, 
               /*valueSelector*/ (key, values) => new
                {
                    idField = key.key1,
                    count = values.Count(),
                });
```

- Grouping by multiple keys and projecting single value
 ```Csharp
Enumerable.GroupBy( 
               /*keySelector*/  k => new
                  {
                      key1 = k.Id,
                      key2 = k.Int
                  } , 
               /*valueSelector*/ (key, values) => values.Count());
```

- Grouping by multiple keys and projecting multiple values
```Csharp
Enumerable.GroupBy(
/*keySelector*/
              k => new
                  {
                      key1 = k.Id,
                      key2 = k.Int
                  } , 
/*valueSelector*/
                (key, values) => new
                {
                    idField = key.key1,
                    count = values.Count(),
                })
```
## Type of change

Please delete options that are not relevant.

- [] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [] Breaking change (fix or feature that would cause existing functionality to not work as expected)
- [] This change requires a documentation update

## Closing issues
closes ##1202

---------

Co-authored-by: leminh98 <[email protected]>

Author: leminh98

Microsoft.Azure.Cosmos/src/Linq/ExpressionToSQL.cs

@@ -15,7 +15,8 @@ namespace Microsoft.Azure.Cosmos.Linq
     using System.Linq.Expressions;
     using System.Reflection;
     using System.Text.RegularExpressions;
-    using Microsoft.Azure.Cosmos.CosmosElements;
+    using Microsoft.Azure.Cosmos.CosmosElements;
+    using Microsoft.Azure.Cosmos.Query.Core.ClientDistributionPlan.Cql;
     using Microsoft.Azure.Cosmos.Serialization.HybridRow;
     using Microsoft.Azure.Cosmos.Serializer;
     using Microsoft.Azure.Cosmos.Spatial;
@@ -1361,14 +1362,6 @@ private static SqlSelectClause VisitGroupByAggregateMethodCall(MethodCallExpress
                 throw new DocumentQueryException(ClientResources.ExpectedMethodCallsMethods);
             }
 
-            Expression inputCollection = inputExpression.Arguments[0]; // all these methods are static extension methods, so argument[0] is the collection
-
-            Collection collection = ExpressionToSql.Translate(inputCollection, context);
-            context.PushCollection(collection);
-
-            bool shouldBeOnNewQuery = context.CurrentQuery.ShouldBeOnNewQuery(inputExpression.Method.Name, inputExpression.Arguments.Count);
-            context.PushSubqueryBinding(shouldBeOnNewQuery);
-
             if (context.LastExpressionIsGroupBy)
             {
                 throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, "Group By cannot be followed by other methods"));
@@ -1406,8 +1399,6 @@ private static SqlSelectClause VisitGroupByAggregateMethodCall(MethodCallExpress
                     throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.MethodNotSupported, inputExpression.Method.Name));
             }
 
-            context.PopSubqueryBinding();
-            context.PopCollection();
             context.PopMethod();
             return select;
         }
@@ -1793,44 +1784,111 @@ private static Collection VisitGroupBy(Type returnElementType, ReadOnlyCollectio
             {
                 throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.InvalidArgumentsCount, LinqMethods.GroupBy, 3, arguments.Count));
             }
-
-            // bind the parameters in the value selector to the current input
-            foreach (ParameterExpression par in Utilities.GetLambda(arguments[2]).Parameters)
-            {
-                context.PushParameter(par, context.CurrentSubqueryBinding.ShouldBeOnNewQuery);
-            }
-
+
+            // Key Selector handling
             // First argument is input, second is key selector and third is value selector
-            LambdaExpression keySelectorLambda = Utilities.GetLambda(arguments[1]);
-
-            // Current GroupBy doesn't allow subquery, so we need to visit non subquery scalar lambda
-            SqlScalarExpression keySelectorFunc = ExpressionToSql.VisitNonSubqueryScalarLambda(keySelectorLambda, context);
-
-            SqlGroupByClause groupby = SqlGroupByClause.Create(keySelectorFunc);
-
-            context.CurrentQuery = context.CurrentQuery.AddGroupByClause(groupby, context);
-
-            // Create a GroupBy collection and bind the new GroupBy collection to the new parameters created from the key
-            Collection collection = ExpressionToSql.ConvertToCollection(keySelectorFunc);
-            collection.isOuter = true;
-            collection.Name = "GroupBy";
-
-            ParameterExpression parameterExpression = context.GenerateFreshParameter(returnElementType, keySelectorFunc.ToString(), includeSuffix: false);
-            Binding binding = new Binding(parameterExpression, collection.inner, isInCollection: false, isInputParameter: true);
-
-            context.CurrentQuery.GroupByParameter = new FromParameterBindings();
-            context.CurrentQuery.GroupByParameter.Add(binding);
-
-            // The alias for the key in the value selector lambda is the first arguemt lambda - we bound it to the parameter expression, which already has substitution
-            ParameterExpression valueSelectorKeyExpressionAlias = Utilities.GetLambda(arguments[2]).Parameters[0];
-            context.GroupByKeySubstitution.AddSubstitution(valueSelectorKeyExpressionAlias, parameterExpression/*Utilities.GetLambda(arguments[1]).Body*/);
-
-            // Translate the body of the value selector lambda
+            LambdaExpression keySelectorLambda = Utilities.GetLambda(arguments[1]);
+
+            Collection collection = new Collection("Group By");
+            context.CurrentQuery.GroupByParameter = new FromParameterBindings();
+
+            SqlGroupByClause groupby;
+            ParameterExpression parameterExpression;
+            switch (keySelectorLambda.Body.NodeType)
+            {
+                case ExpressionType.Parameter:
+                case ExpressionType.Call:
+                case ExpressionType.MemberAccess:
+                    {
+                        // bind the parameters in the value selector to the current input
+                        foreach (ParameterExpression par in Utilities.GetLambda(arguments[2]).Parameters)
+                        {
+                            context.PushParameter(par, context.CurrentSubqueryBinding.ShouldBeOnNewQuery);
+                        }
+
+                        //Current GroupBy doesn't allow subquery, so we need to visit non subquery scalar lambda
+                        SqlScalarExpression keySelectorFunc = ExpressionToSql.VisitNonSubqueryScalarLambda(keySelectorLambda, context);
+
+                        // The group by clause don't need to handle the value selector, so adding the clause to the uery now.
+                        groupby = SqlGroupByClause.Create(keySelectorFunc);
+                        parameterExpression = context.GenerateFreshParameter(returnElementType, keySelectorFunc.ToString(), includeSuffix: false);
+
+                        break;
+                    }
+                case ExpressionType.New:
+                    {
+                        // bind the parameters in the key selector to the current input - in this case, the value selector key is being substituted by the key selector
+                        foreach (ParameterExpression par in Utilities.GetLambda(arguments[1]).Parameters)
+                        {
+                            context.PushParameter(par, context.CurrentSubqueryBinding.ShouldBeOnNewQuery);
+                        }
+
+                        NewExpression newExpression = (NewExpression)keySelectorLambda.Body;
+
+                        if (newExpression.Members == null)
+                        {
+                            throw new DocumentQueryException(ClientResources.ConstructorInvocationNotSupported);
+                        }
+
+                        ReadOnlyCollection<Expression> newExpressionArguments = newExpression.Arguments;
+
+                        List<SqlScalarExpression> keySelectorFunctions = new List<SqlScalarExpression>();
+                        for (int i = 0; i < newExpressionArguments.Count; i++)
+                        {
+                            //Current GroupBy doesn't allow subquery, so we need to visit non subquery scalara
+                            SqlScalarExpression keySelectorFunc = ExpressionToSql.VisitNonSubqueryScalarExpression(newExpressionArguments[i], context);
+                            keySelectorFunctions.Add(keySelectorFunc);
+                        }
+
+                        groupby = SqlGroupByClause.Create(keySelectorFunctions.ToImmutableArray());
+                        parameterExpression = context.GenerateFreshParameter(returnElementType, keySelectorFunctions.ToString(), includeSuffix: false);
+
+                        break;
+                    }
+                default:
+                    throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.ExpressionTypeIsNotSupported, keySelectorLambda.Body.NodeType));
+            }
+
+            // The group by clause don't need to handle the value selector, so adding the clause to the qery now.
+            context.CurrentQuery = context.CurrentQuery.AddGroupByClause(groupby, context);
+
+            // Bind the alias
+            Binding binding = new Binding(parameterExpression, collection.inner, isInCollection: false, isInputParameter: true);
+            context.CurrentQuery.GroupByParameter.Add(binding);
+
+            // The alias for the key in the value selector lambda is the first arguemt lambda - we bound it to the parameter expression, which already has substitution
+            ParameterExpression valueSelectorKeyExpressionAlias = Utilities.GetLambda(arguments[2]).Parameters[0];
+            context.GroupByKeySubstitution.AddSubstitution(valueSelectorKeyExpressionAlias, parameterExpression);
+
+            // Value Selector Handingling
+            // Translate the body of the value selector lambda
             Expression valueSelectorExpression = Utilities.GetLambda(arguments[2]).Body;
 
             // The value selector function needs to be either a MethodCall or an AnonymousType
             switch (valueSelectorExpression.NodeType)
-            {
+            {
+                case ExpressionType.MemberAccess:
+                    {
+                        MemberExpression memberAccessExpression = (MemberExpression)valueSelectorExpression;
+
+                        if (memberAccessExpression.Expression.NodeType == ExpressionType.Parameter)
+                        {
+                            // Look up the object of the expression to see if it is the key
+                            ParameterExpression memberAccessObject = (ParameterExpression)memberAccessExpression.Expression;
+                            Expression subst = context.GroupByKeySubstitution.Lookup(memberAccessObject);
+                            if (subst != null)
+                            {
+                                // If  there is a match, we construct a new Member Access expression with the substituted expression and visit it to create a select clause
+                                MemberExpression newMemberAccessExpression = memberAccessExpression.Update(keySelectorLambda.Body); 
+                                SqlScalarExpression selectExpression = ExpressionToSql.VisitMemberAccess(newMemberAccessExpression, context);
+
+                                SqlSelectSpec sqlSpec = SqlSelectValueSpec.Create(selectExpression);
+                                SqlSelectClause select = SqlSelectClause.Create(sqlSpec, null);
+                                context.CurrentQuery = context.CurrentQuery.AddSelectClause(select, context);
+                            }
+                        }
+                        break;
+                    }
                 case ExpressionType.Constant:
                     {
                         ConstantExpression constantExpression = (ConstantExpression)valueSelectorExpression;
@@ -1908,6 +1966,27 @@ private static Collection VisitGroupBy(Type returnElementType, ReadOnlyCollectio
                                         selectItems[i] = prop;
                                         break;
                                     }
+                                case ExpressionType.MemberAccess:
+                                    {
+                                        MemberExpression memberAccessExpression = (MemberExpression)arg;
+
+                                        if (memberAccessExpression.Expression.NodeType == ExpressionType.Parameter)
+                                        {
+                                            // Look up the object of the expression to see if it is the key
+                                            ParameterExpression memberAccessObject = (ParameterExpression)memberAccessExpression.Expression;
+                                            Expression subst = context.GroupByKeySubstitution.Lookup(memberAccessObject);
+                                            if (subst != null)
+                                            {
+                                                // If  there is a match, we construct a new Member Access expression with the substituted expression and visit it to create a select clause
+                                                MemberExpression newMemberAccessExpression = memberAccessExpression.Update(keySelectorLambda.Body); /*System.Linq.Expressions.Expression.Field(subst, memberAccessExpression.Member.Name);*/
+                                                SqlScalarExpression selectExpression = ExpressionToSql.VisitMemberAccess(newMemberAccessExpression, context);
+
+                                                SqlSelectItem prop = SqlSelectItem.Create(selectExpression, alias);
+                                                selectItems[i] = prop;
+                                            }
+                                        }
+                                        break;
+                                    }
                                 default:
                                     throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.ExpressionTypeIsNotSupported, arg.NodeType));
                             }
@@ -1921,13 +2000,34 @@ private static Collection VisitGroupBy(Type returnElementType, ReadOnlyCollectio
                     }
                 default:
                     throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, ClientResources.ExpressionTypeIsNotSupported, valueSelectorExpression.NodeType));
-            }
-
-            foreach (ParameterExpression par in Utilities.GetLambda(arguments[2]).Parameters)
-            {
-                context.PopParameter();
-            }
-
+            }
+
+            // Pop the correct number of items off the parameter stack
+            switch (keySelectorLambda.Body.NodeType)
+            {
+                case ExpressionType.Parameter:
+                case ExpressionType.Call:
+                case ExpressionType.MemberAccess:
+                    {
+                        foreach (ParameterExpression param in Utilities.GetLambda(arguments[2]).Parameters)
+                        {
+                            context.PopParameter();
+                        }
+                        break;
+                    }
+                case ExpressionType.New:
+                    {
+                        //bind the parameters in the value selector to the current input
+                        foreach (ParameterExpression param in Utilities.GetLambda(arguments[1]).Parameters)
+                        {
+                            context.PopParameter();
+                        }
+                        break;
+                    }
+                default:
+                    break;
+            }
+
             return collection;
         }
 
@@ -2234,7 +2334,7 @@ private static SqlInputPathCollection ConvertMemberIndexerToPath(SqlMemberIndexe
                 if (parent == null)
                 {
                     break;
-                }
+                }
 
                 if (parent is SqlPropertyRefScalarExpression sqlPropertyRefScalarExpression)
                 {

Microsoft.Azure.Cosmos/src/Linq/QueryUnderConstruction.cs

@@ -250,8 +250,8 @@ public QueryUnderConstruction FlattenAsPossible()
                     break;
                 }
 
-                // In case of Select -> Group by cases, the Select query should not be flattened and kept as a subquery
-                if ((query.inputQuery?.selectClause != null) && (query.groupByClause != null))
+                // In case of Select/Order By -> Group by cases, the Select/Order By query should not be flattened and kept as a subquery
+                if (((query.inputQuery?.selectClause != null) || (query.inputQuery?.orderByClause != null)) && (query.groupByClause != null))
                 {
                     flattenQuery = this;
                     break;
@@ -564,11 +564,12 @@ public bool ShouldBeOnNewQuery(string methodName, int argumentCount)
                     break;
 
                 case LinqMethods.GroupBy:
-                    // New query is needed when there is already a Take or a Select or a Group by clause
+                    // New query is needed when there is already a Take or a Select or a Group by clause or an Order By clause
                     shouldPackage = (this.topSpec != null) ||
                         (this.offsetSpec != null) ||
                         (this.selectClause != null) || 
-                        (this.groupByClause != null);
+                        (this.groupByClause != null) || 
+                        (this.orderByClause != null);
                     break;
 
                 case LinqMethods.Skip:

Microsoft.Azure.Cosmos/src/Linq/TranslationContext.cs

@@ -232,12 +232,12 @@ public void PushCollection(Collection collection)
                 throw new ArgumentNullException("collection");
             }
 
-            if (this.CurrentQuery.GroupByParameter == null) this.collectionStack.Add(collection);
+            this.collectionStack.Add(collection);
         }
 
         public void PopCollection()
         {
-            if (this.CurrentQuery.GroupByParameter == null) this.collectionStack.RemoveAt(this.collectionStack.Count - 1);
+            this.collectionStack.RemoveAt(this.collectionStack.Count - 1);
         }
 
         /// <summary>