Merge pull request #1017 from parente/fix-pyspark

Fix pyspark path

Author: parente

pyspark-notebook/Dockerfile

@@ -38,11 +38,11 @@ RUN apt-get -y update && \
     rm -rf /var/lib/apt/lists/*
 
 # Spark and Mesos config
-ENV SPARK_HOME=/usr/local/spark \
-    PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip \
+ENV SPARK_HOME=/usr/local/spark
+ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip \
     MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so \
     SPARK_OPTS="--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info" \
-    PATH=$PATH:/usr/local/spark/bin
+    PATH=$PATH:$SPARK_HOME/bin
 
 USER $NB_UID
 

pyspark-notebook/test/test_spark.py

@@ -16,4 +16,15 @@ def test_spark_shell(container):
     c.wait(timeout=30)
     logs = c.logs(stdout=True).decode('utf-8')
     LOGGER.debug(logs)
-    assert 'res0: Int = 2' in logs
+    assert 'res0: Int = 2' in logs
+
+def test_pyspark(container):
+    """PySpark should be in the Python path"""
+    c = container.run(
+        tty=True,
+        command=['start.sh', 'python', '-c', '"import pyspark"']
+    )
+    rv = c.wait(timeout=30)
+    assert rv == 0 or rv["StatusCode"] == 0
+    logs = c.logs(stdout=True).decode('utf-8')
+    LOGGER.debug(logs)