[SPARK-52165][SQL][SDP] Setup build scaffolding for the pipelines project

### What changes were proposed in this pull request?
- Defines a new project inside the Spark repository where code for Spark Declarative Pipelines will live ([SPIP](https://issues.apache.org/jira/browse/SPARK-51727)). This PR introduces the basic build scaffolding for Maven and SBT to define the pipelines project which will live under `sql/pipelines/core`.
- Ensure that tests for the pipelines project are triggered from Github Actions `Build` job

### Why are the changes needed?
Future code that implements functionality for Spark Declarative Pipelines will live in this new project.

### Does this PR introduce _any_ user-facing change?
No

### How was this patch tested?
This PR only includes new build definitions. This patch was tested just by ensuring that the project continues to build.

### Was this patch authored or co-authored using generative AI tooling?
No

Closes #50905 from jonmio/setup_pipelines_project.

Lead-authored-by: Jon Mio <[email protected]>
Co-authored-by: Gengliang Wang <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>

Author: jon-mio

dev/sparktestsupport/modules.py

@@ -408,6 +408,15 @@ def __hash__(self):
     ],
 )
 
+pipelines = Module(
+    name="pipelines",
+    dependencies=[],
+    source_file_regexes=["sql/pipelines"],
+    sbt_test_goals=[
+        "pipelines/test",
+    ],
+)
+
 connect = Module(
     name="connect",
     dependencies=[hive, avro, protobuf, mllib],

pom.xml

@@ -95,6 +95,7 @@
     <module>sql/catalyst</module>
     <module>sql/core</module>
     <module>sql/hive</module>
+    <module>sql/pipelines</module>
     <module>sql/connect/server</module>
     <module>sql/connect/common</module>
     <module>sql/connect/client/jvm</module>

project/SparkBuild.scala

@@ -58,10 +58,10 @@ object BuildCommons {
 
   val allProjects@Seq(
     core, graphx, mllib, mllibLocal, repl, networkCommon, networkShuffle, launcher, unsafe, tags, sketch, kvstore,
-    commonUtils, variant, _*
+    commonUtils, variant, pipelines, _*
   ) = Seq(
     "core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe",
-    "tags", "sketch", "kvstore", "common-utils", "variant"
+    "tags", "sketch", "kvstore", "common-utils", "variant", "pipelines"
   ).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ connectProjects
 
   val optionallyEnabledProjects@Seq(kubernetes, yarn,
@@ -372,8 +372,8 @@ object SparkBuild extends PomBuild {
   val mimaProjects = allProjects.filterNot { x =>
     Seq(
       spark, hive, hiveThriftServer, repl, networkCommon, networkShuffle, networkYarn,
-      unsafe, tags, tokenProviderKafka010, sqlKafka010, connectCommon, connect, connectClient,
-      variant, connectShims, profiler
+      unsafe, tags, tokenProviderKafka010, sqlKafka010, pipelines, connectCommon, connect,
+      connectClient, variant, connectShims, profiler
     ).contains(x)
   }
 

sql/pipelines/pom.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.spark</groupId>
+        <artifactId>spark-parent_2.13</artifactId>
+        <version>4.1.0-SNAPSHOT</version>
+        <relativePath>../../pom.xml</relativePath>
+    </parent>
+    <artifactId>spark-pipelines_2.13</artifactId>
+    <properties>
+        <sbt.project.name>pipelines</sbt.project.name>
+    </properties>
+    <packaging>jar</packaging>
+    <name>Spark Project Declarative Pipelines Library</name>
+    <url>https://spark.apache.org/</url>
+    <build>
+        <outputDirectory>core/target/scala-2.13/classes</outputDirectory>
+        <testOutputDirectory>core/target/scala-2.13/test-classes</testOutputDirectory>
+    </build>
+</project>