[HUDI-9246] Confine protobuf shading to the hudi-io package (#13065)

Author: the-other-tim-brown

hudi-client/hudi-client-common/pom.xml

@@ -47,6 +47,7 @@
       <groupId>org.apache.hudi</groupId>
       <artifactId>hudi-io</artifactId>
       <version>${project.version}</version>
+      <classifier>shaded</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>

hudi-client/hudi-spark-client/pom.xml

@@ -59,6 +59,7 @@
       <groupId>org.apache.hudi</groupId>
       <artifactId>hudi-io</artifactId>
       <version>${project.version}</version>
+      <classifier>shaded</classifier>
     </dependency>
 
     <dependency>

hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/HoodieSparkKryoRegistrar.scala

@@ -32,6 +32,7 @@ import com.esotericsoftware.kryo.serializers.JavaSerializer
 import com.google.protobuf.Message
 import com.twitter.chill.protobuf.ProtobufSerializer
 import org.apache.spark.serializer.KryoRegistrator
+import org.slf4j.LoggerFactory
 
 /**
  * NOTE: PLEASE READ CAREFULLY BEFORE CHANGING
@@ -50,6 +51,8 @@ import org.apache.spark.serializer.KryoRegistrator
  * </ol>
  */
 class HoodieSparkKryoRegistrar extends HoodieCommonKryoRegistrar with KryoRegistrator {
+  private val log = LoggerFactory.getLogger(classOf[HoodieSparkKryoRegistrar])
+
 
   override def registerClasses(kryo: Kryo): Unit = {
     ///////////////////////////////////////////////////////////////////////////
@@ -74,7 +77,14 @@ class HoodieSparkKryoRegistrar extends HoodieCommonKryoRegistrar with KryoRegist
     //       So we replace it with [[HadoopStorageConfiguration]] for Spark.
     kryo.register(classOf[HadoopStorageConfiguration], new JavaSerializer())
     // NOTE: Protobuf objects are not serializable by default using kryo, need to register them explicitly.
-    kryo.addDefaultSerializer(classOf[Message], new ProtobufSerializer())
+    //       Only initialize this serializer if Protobuf is on the classpath.
+    try {
+      if (Class.forName(classOf[Message].getName, false, getClass.getClassLoader) != null) {
+        kryo.addDefaultSerializer(classOf[Message], new ProtobufSerializer())
+      }
+    } catch {
+      case _: ClassNotFoundException | _: NoClassDefFoundError => log.warn("Protobuf classes not found on the classpath, skipping Protobuf serializer registration.")
+    }
   }
 
   /**

hudi-common/pom.xml

@@ -107,6 +107,7 @@
       <groupId>org.apache.hudi</groupId>
       <artifactId>hudi-io</artifactId>
       <version>${project.version}</version>
+      <classifier>shaded</classifier>
     </dependency>
 
     <dependency>

hudi-hadoop-common/pom.xml

@@ -77,6 +77,7 @@
       <groupId>org.apache.hudi</groupId>
       <artifactId>hudi-io</artifactId>
       <version>${project.version}</version>
+      <classifier>shaded</classifier>
     </dependency>
 
     <!-- Hadoop -->

hudi-io/pom.xml

@@ -96,6 +96,36 @@
         <groupId>org.jacoco</groupId>
         <artifactId>jacoco-maven-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>${maven-shade-plugin.version}</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <createDependencyReducedPom>false</createDependencyReducedPom>
+              <shadedArtifactAttached>true</shadedArtifactAttached>
+              <shadedClassifierName>shaded</shadedClassifierName>
+              <artifactSet>
+                <includes>
+                  <!-- native HFile reader uses protobuf -->
+                  <include>com.google.protobuf:protobuf-java</include>
+                </includes>
+              </artifactSet>
+              <relocations>
+                <relocation>
+                  <pattern>com.google.protobuf.</pattern>
+                  <shadedPattern>org.apache.hudi.com.google.protobuf.</shadedPattern>
+                </relocation>
+              </relocations>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

hudi-spark-datasource/hudi-spark/pom.xml

@@ -205,6 +205,7 @@
       <groupId>org.apache.hudi</groupId>
       <artifactId>hudi-io</artifactId>
       <version>${project.version}</version>
+      <classifier>shaded</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>

hudi-sync/hudi-hive-sync/pom.xml

@@ -53,6 +53,7 @@
       <groupId>org.apache.hudi</groupId>
       <artifactId>hudi-io</artifactId>
       <version>${project.version}</version>
+      <classifier>shaded</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>

hudi-utilities/pom.xml

@@ -172,6 +172,7 @@
       <groupId>org.apache.hudi</groupId>
       <artifactId>hudi-io</artifactId>
       <version>${project.version}</version>
+      <classifier>shaded</classifier>
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>

packaging/hudi-integ-test-bundle/pom.xml

@@ -122,6 +122,7 @@
                   <include>org.apache.parquet:parquet-avro</include>
                   <include>com.twitter:parquet-avro</include>
                   <include>com.twitter.common:objectsize</include>
+                  <include>com.google.protobuf:protobuf-java</include>
 
                   <include>io.confluent:kafka-avro-serializer</include>
                   <include>io.confluent:common-config</include>