Merge pull request #5 from open-datastudio/spark_301

Support spark 3.0.1

Author: Leemoonsoo

README.md

@@ -27,6 +27,8 @@ Let's get started!
 pip install ods
 ```
 
+Python `3.6`, `3.7`, `3.8` are supported.
+
 ### Initialize
 
 1. Login staroid.com and get an [access token](https://staroid.com/settings/accesstokens). And set the `STAROID_ACCESS_TOKEN` environment variable. See [here](https://github.com/staroids/staroid-python#configuration) for more detail.
@@ -81,6 +83,16 @@ spark = ods.spark(spark_conf = {
 }).session()
 ```
 
+configure spark version
+
+```python
+import ods
+spark = ods.spark(spark_version = "3.0.1").session()
+```
+
+Currently, spark `3.0.1`, `3.0.0` are supported.
+
+
 Check [tests/test_spark.py](https://github.com/open-datastudio/ods/blob/master/tests/test_spark.py) for complete working example.
 
 ## Dask
@@ -104,7 +116,6 @@ import ods
 ods.ray(cluster_name="")
 ```
 
-
 ## Get involved
 
 Open data studio is an open source project. Please give us feedback and feel free to get involved!

ods/ods.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 
 class Ods:
-    def __init__(self, staroid=None, ske=None, cache_dir=None, chisel_path=None):
+    def __init__(self, staroid=None, ske=None, cache_dir=None):
         self.__tunnel_processes = {}
         self.__ske = None
 
@@ -28,8 +28,6 @@ def __init__(self, staroid=None, ske=None, cache_dir=None, chisel_path=None):
         if ske != None:
             self.__ske = ske
 
-        self.__chisel_path = chisel_path
-
     def create_or_get_cache_dir(self, module = ""):
         "create (if not exists) or return cache dir path for module"
         cache_dir = "{}/{}".format(self.__cache_dir, module)
@@ -127,8 +125,27 @@ def init(ske=None, reinit=True):
         __singleton["instance"] = Ods(ske=ske)
     return __singleton["instance"]
 
-def spark(name, spark_conf=None, chisel_path=None, worker_num=1, worker_type="standard-4", worker_isolation="dedicated", delta=False, aws=True):
+def spark(
+        name,
+        spark_conf=None,
+        spark_version="3.0.1",
+        spark_home=None,
+        worker_num=1,
+        worker_type="standard-4",
+        worker_isolation="dedicated",
+        delta=False,
+        aws=True):
     init(reinit=False)
 
-    cluster = SparkCluster(__singleton["instance"], name, spark_conf=spark_conf, worker_num=worker_num, worker_type=worker_type, worker_isolation=worker_isolation, delta=delta, aws=aws)
+    cluster = SparkCluster(
+        __singleton["instance"],
+        name,
+        spark_conf=spark_conf,
+        spark_version=spark_version,
+        spark_home=spark_home,
+        worker_num=worker_num,
+        worker_type=worker_type,
+        worker_isolation=worker_isolation,
+        delta=delta,
+        aws=aws)
     return cluster

ods/spark_cluster/spark_cluster.py

@@ -8,6 +8,15 @@
 from kubernetes import client
 
 SPARK_ARTIFACTS={
+    "3.0.1": {
+        "image": "opendatastudio/spark-py:v3.0.1-staroid-20200908-01",
+        "dist": "https://archive.apache.org/dist/spark/spark-3.0.1/spark-3.0.1-bin-hadoop3.2.tgz",
+        "commit_url": "GITHUB/open-datastudio/spark-serverless:master",
+        "jars": {
+            "hadoop-aws": "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.0/hadoop-aws-3.2.0.jar",
+            "aws-java-sdk": "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.563/aws-java-sdk-bundle-1.11.563.jar"
+        }
+    },
     "3.0.0": {
         "image": "opendatastudio/spark-py:v3.0.0-staroid-20200830-01",
         "dist": "https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop3.2.tgz",
@@ -31,7 +40,7 @@ def __init__(
         opends,
         cluster_name,
         spark_conf=None,
-        spark_version="3.0.0",
+        spark_version="3.0.1",
         spark_home=None,
         worker_num=1,
         worker_type="standard-4",

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="ods",
-    version="0.0.4",
+    version="0.0.5",
     license='MIT',
     author="Open Data Studio",
     author_email="moon@staroid.com",