Feature/worker runners fixes

.gitignore

@@ -3,6 +3,7 @@ target/
 .vagrant
 derby.log
 metastore_db/
+spark-warehouse/
 *.db
 file.dbb
 stdout.txt

.travis.yml

@@ -26,6 +26,6 @@ env:
   global:
     secure: f+Kg5LkG5fBsJ5/V7dOxWGMUeWPBU8vUNDk0HsytNtm9mTnospTiFykoxiXbKNhVlmfhmAsyseIcKIrFwSoz/6rlJXiHYdTeH/DSoCa/JLDLMn/7zJp8v6lwer2KEpc+cnUiG5TMR1QmGJ2yVrw1DrwUqs9hBrWNP1bmr1wXGgrde56fnFXKy6YlToUWqtqHn0lMYaV/gBH7jdCdC4Q3n1FlAe9a8x7Lecy3sm489Fpyf6MuekdrulmW15YXJ71XAiEwGA24zxqOd5Pp2Vcyg2UNVtTyjyqlybhX65Ov/hzup44xjdyEFW0u7jt7AdIKTXBrHVvfoLg5c52Jltr+yhJA37zgntWRIA1pRWxJXOpfL5zaICLSVovS0J4xfdPOgK6o4kCVGBNQdy/jeKyFtPAE2SFjDQ4pCeSBUJ3uk0w2MEnyTluZHg2vQtjKXDVsLNxD6sr6UPUURgMm8d7BR8tqVzCUU5kwdR1WQVzkNEmurXv0Zuohjc1NK8f9Qwqd7DlEAtdtpSVg1F9V5qzOeEGq0aYy6Uu2R8WEN8SiZITdVgL0LZQo12kOPqoSJLNERx0KFNIaqBtsuqnZirdCPgF7FeZou5iN5sxVSX7OL8/AnuhV/W8R/V0n2K90rXY6OXfOwl4ojVxc7uoavHmLzhmfDVUcQv22AXwicT3DQRU=
   matrix:
-    - SPARK_VERSION=2.0.0
     - SPARK_VERSION=2.1.0
     - SPARK_VERSION=2.2.0
+    - SPARK_VERSION=2.3.0

Jenkinsfile

@@ -1,8 +1,8 @@
 //Spark Version
 versions = [
-        "2.0.0",
         "2.1.0",
-        "2.2.0"
+        "2.2.0",
+        "2.3.0"
 ]
 
 def branches = [:]

configs/default.conf

@@ -1,8 +1,8 @@
 mist {
   context-defaults.spark-conf = {
     spark.master = "local[*]"
-    //spark.master = "spark://holy-excalibur:7077"
   }
 
   workers.runner = "local"
 }
+

configs/docker.conf

@@ -3,15 +3,15 @@ mist {
     spark.master = "local[*]"
   }
 
-  http {
-    on = true
-    host = "0.0.0.0"
-    port = 2004
-  }
+  cluster.host = "auto"
+  http.host = "auto"
+  log-service.host = "auto"
 
   workers {
     runner = "docker"
-    docker-host = "localhost"
-    docker-port = 2375
+    docker {
+      image = "hydrosphere/mist:${version}-${sparkVersion}"
+      network-type = "auto-master"
+    }
   }
 }

docker-entrypoint.sh

@@ -3,13 +3,6 @@
 set -e
 cd ${MIST_HOME}
 
-if [ "$1" = 'mist' ]; then
-  export IP=`ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p'`
-  echo "$IP    master" >> /etc/hosts
-
-  export MIST_OPTS="$MIST_OPTS -Dmist.cluster.host=$IP -Dmist.http.host=$IP -Dmist.log-service.host=$IP"
-  shift
-  exec ./bin/mist-master start --debug true $@
-else
-  exec "$@"
-fi
+MASTER_ID=$(cat /proc/1/cgroup | grep "/docker/" | head -n 1 | awk -F "/" '{print $NF}')
+export MIST_OPTS="$MIST_OPTS -Dmist.workers.docker.auto-master-network.container-id=$MASTER_ID"
+exec ./bin/mist-master start --debug true $@

mist-lib/src/test/scala/mist/api/BaseContextsSpec.scala

@@ -2,12 +2,13 @@ package mist.api
 
 import io.hydrosphere.mist.api.{RuntimeJobInfo, SetupConfiguration}
 import org.apache.spark.SparkContext
+import org.apache.spark.sql.{SaveMode, SparkSession}
 import org.apache.spark.sql.hive.HiveContext
 import org.scalatest.{FunSpec, Matchers}
 
 class BaseContextsSpec extends FunSpec with Matchers with TestSparkContext {
 
-  import BaseContexts._
+  import Contexts._
   import mist.api.args.ArgsInstances._
 
   it("for spark context") {
@@ -32,23 +33,15 @@ class BaseContextsSpec extends FunSpec with Matchers with TestSparkContext {
     this.getClass.getClassLoader.getResource(path).getPath
   }
 
-  it("for only hive context") {
-    val spJob = onHiveContext((hiveCtx: HiveContext) => {
-      val df = hiveCtx.read.json(pathToResource("hive_job_data.json"))
-      df.registerTempTable("temp")
-      hiveCtx.sql("SELECT MAX(age) AS avg_age FROM temp")
-        .take(1)(0).getLong(0)
-    })
-    spJob.invoke(testCtx())
-    val res = spJob.invoke(testCtx())
-    res shouldBe JobSuccess(30)
-  }
+  it("session with hive") {
+    val spJob = onSparkSessionWithHive((spark: SparkSession) => {
+      val df = spark.read.json(pathToResource("hive_job_data.json"))
+      df.createOrReplaceTempView("temp")
+      df.cache()
+      spark.sql("DROP TABLE IF EXISTS temp_hive")
+      spark.table("temp").write.mode(SaveMode.Overwrite).saveAsTable("temp_hive")
 
-  it("allow more than one hive context") {
-    val spJob = onHiveContext((hiveCtx: HiveContext) => {
-      val df = hiveCtx.read.json(pathToResource("hive_job_data.json"))
-      df.registerTempTable("temp")
-      hiveCtx.sql("SELECT MAX(age) AS avg_age FROM temp")
+      spark.sql("SELECT MAX(age) AS avg_age FROM temp_hive")
         .take(1)(0).getLong(0)
     })
     spJob.invoke(testCtx())