Upgrade spark dependency to 3.3.0

.bsp/sbt.json

@@ -0,0 +1 @@
+{"name":"sbt","version":"1.4.9","bspVersion":"2.0.0-M5","languages":["scala"],"argv":["/Library/Java/JavaVirtualMachines/jdk-11.0.2.jdk/Contents/Home/bin/java","-Xms100m","-Xmx100m","-classpath","/Users/weichen.xu/Library/Application Support/JetBrains/IdeaIC2021.2/plugins/Scala/launcher/sbt-launch.jar","xsbt.boot.Boot","-bsp"]}

.travis.yml

@@ -41,19 +41,6 @@ jobs:
       script:
         - make test_root_sbt_project
 
-    - name: "Python 3.6 tests"
-      language: python
-      python: 3.6
-      install:
-        - pip install tox
-      before_script:
-        - >
-          curl
-          --create-dirs -L -o /home/travis/.sbt/launchers/1.4.9/sbt-launch.jar
-          https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.4.9/sbt-launch-1.4.9.jar
-      script:
-        - make py36_test
-
     - name: "Python 3.7 tests"
       language: python
       python: 3.7.9

Makefile

@@ -23,14 +23,10 @@ test_xgboost_runtime:
 test_xgboost_spark:
 	$(SBT) "+ mleap-xgboost-spark/test"
 
-.PHONY: py36_test
-py36_test:
-	source scripts/scala_classpath_for_python.sh && make -C python py36_test
-
 .PHONY: py37_test
 py37_test:
 	source scripts/scala_classpath_for_python.sh && make -C python py37_test
 
 .PHONY: test
-test: test_executor test_benchmark test_xgboost_runtime test_xgboost_spark test_root_sbt_project py36_test py37_test
+test: test_executor test_benchmark test_xgboost_runtime test_xgboost_spark test_root_sbt_project py37_test
 	@echo "All tests run successfully"

mleap-spark-testkit/src/main/scala/org/apache/spark/ml/parity/SparkParityBase.scala

@@ -59,10 +59,14 @@ object SparkParityBase extends FunSpec {
 
 
 object SparkEnv {
-  lazy val spark = SparkSession.builder().
-    appName("Spark/MLeap Parity Tests").
-    master("local[2]").
-    getOrCreate()
+  lazy val spark = {
+    val session = SparkSession.builder().
+      appName("Spark/MLeap Parity Tests").
+      master("local[2]").
+      getOrCreate()
+    session.sparkContext.setLogLevel("WARN")
+    session
+  }
 }
 
 

project/Dependencies.scala

@@ -6,19 +6,19 @@ import Keys._
 object Dependencies {
   import DependencyHelpers._
 
-  val sparkVersion = "3.2.0"
+  val sparkVersion = "3.3.0"
   val scalaTestVersion = "3.0.8"
   val junitVersion = "5.8.2"
   val akkaVersion = "2.6.14"
   val akkaHttpVersion = "10.2.4"
   val springBootVersion = "2.6.2"
   lazy val logbackVersion = "1.2.3"
   lazy val loggingVersion = "3.9.0"
-  lazy val slf4jVersion = "1.7.30"
+  lazy val slf4jVersion = "1.7.36"
   lazy val awsSdkVersion = "1.11.1033"
   val tensorflowJavaVersion = "0.4.0" // Match Tensorflow 2.7.0 https://github.com/tensorflow/java/#tensorflow-version-support
   val xgboostVersion = "1.5.2"
-  val breezeVersion = "1.0"
+  val breezeVersion = "1.2"
   val hadoopVersion = "2.7.4" // matches spark version
   val platforms = "windows-x86_64,linux-x86_64,macosx-x86_64"
   val tensorflowPlatforms : Array[String] =  sys.env.getOrElse("TENSORFLOW_PLATFORMS", platforms).split(",")

python/Makefile

@@ -5,7 +5,7 @@ $(error SCALA_CLASS_PATH for python tests is not set. Please check out \
 the top-level Makefile on how to source scala_classpath_for_python.sh)
 endif
 
-.PHONY: help env clean py36_test py37_test test build upload
+.PHONY: help env clean py37_test test build upload
 
 help:
 	@echo "  env         create a development environment using virtualenv"
@@ -25,13 +25,10 @@ clean:
 	find . -name '*~' -exec rm -f {} \;
 	find . -name '__pycache__' | xargs -r rm -rf
 
-py36_test:
-	tox -e py36 -v
-
 py37_test:
 	tox -e py37 -v
 
-test: py36_test py37_test
+test: py37_test
 	@echo "All python tests completed"
 
 build: clean

python/mleap/sklearn/preprocessing/data.py

@@ -1026,8 +1026,8 @@ def transform(self, y):
         :return:
         """
         if isinstance(y, pd.DataFrame):
-            x = y.ix[:,0]
-            y = y.ix[:,1]
+            x = y.iloc[:,0]
+            y = y.iloc[:,1]
         else:
             x = y[:,0]
             y = y[:,1]

python/requirements-dev.txt

@@ -3,4 +3,4 @@ coverage<5.0.0
 ipdb
 nose
 nose-exclude>=0.5.0
-pyspark==3.2.0
+pyspark==3.3.0

python/requirements.txt

@@ -1,7 +1,7 @@
 numpy>=1.8.2
 six>=1.10.0
 scipy>=0.13.0b1
-pandas>=0.18.1, <= 0.24.2
+pandas>=1.0.5
 scikit-learn>=0.22.0,<0.23.0
 gensim<4.1.0
 urllib3==1.26.5

python/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py36,py37
+envlist = py37
 skipdist = true
 
 [testenv]