diff --git a/README.md b/README.md index 4e27c37..3f2914e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ echo "export MARVIN_DATA_PATH=$HOME/marvin/data" >> $HOME/.bash_profile echo "source virtualenvwrapper.sh" >> $HOME/.bash_profile source ~/.bash_profile -```` +``` 5. Clone and install python-toolbox @@ -71,7 +71,7 @@ mkvirtualenv python-toolbox-env setvirtualenvproject make marvin -```` +``` 6. Test the installation ``` @@ -179,5 +179,23 @@ marvin engine-generateenv ../engines/iris-species-engine/ workon iris-species-engine-env marvin engine-dryrun ``` +### Install Marvin using docker to developer + +``` +# expose ports 8000 and 8888 to random ports +$ docker run -it -P allanbatista/marvin-dev:0.0.2 /bin/bash -c "su marvin" + +# expose ports 8000 and 8888 +$ docker run -it -p 8000:8000 -p 8888:8888 allanbatista/marvin-dev:0.0.2 /bin/bash -c "su marvin" + +# use toolbox env +$ workon marvin-toolbox-env + +# create new engine +$ marvin engine-generate +$ workon NAME-engine-env +$ marvin notebook +``` + -> Marvin is a project started at B2W Digital offices and released open source on September 2017. +> Marvin is a project started at B2W Digital offices and released open source on September 2017. \ No newline at end of file diff --git a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base new file mode 100644 index 0000000..e73ae87 --- /dev/null +++ b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base @@ -0,0 +1,64 @@ +FROM ubuntu:16.04 + +MAINTAINER Allan Batista + +SHELL ["/bin/bash", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive +ENV SLEEP_MILLIS 0 +ENV SHELL=/bin/bash + +USER root + +############################################################## +# Install the system dependencies for default installation +############################################################## + +RUN apt-get update -y && \ + apt-get install -y build-essential && \ + apt-get install -y maven git python cmake software-properties-common curl libstdc++6 wget python2.7-dev python-pip ipython libffi-dev libssl-dev libxml2-dev libxslt1-dev libpng12-dev libfreetype6-dev python-tk libsasl2-dev python-pip graphviz unzip + +# Install Oracle JDK +RUN add-apt-repository ppa:webupd8team/java -y && \ + apt-get -qq update && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get install -y oracle-java8-installer + +RUN pip install --upgrade pip && \ + pip install virtualenvwrapper && \ + echo "source /usr/local/bin/virtualenvwrapper.sh" >> ~/.bashrc + +RUN apt-get clean + +############################################################## +# Define all environment variables to be used +############################################################## + +ENV MARVIN_HOME=/opt/marvin +ENV MARVIN_DATA_PATH=$MARVIN_HOME/data +ENV MARVIN_TOOLBOX_HOME=$MARVIN_HOME/toolbox +ENV MARVIN_TOOLBOX_ENV=marvin-toolbox-env +ENV WORKON_HOME=$MARVIN_TOOLBOX_HOME/.virtualenvs + +############################################################## +# Create all folders needed +############################################################## + +RUN mkdir -p $MARVIN_HOME && \ + mkdir -p $MARVIN_DATA_PATH && \ + mkdir -p /var/log/marvin/engines && \ + mkdir -p /var/run/marvin/engines + +WORKDIR $MARVIN_HOME + +############################################################## +# Install toolbox +############################################################## + +RUN git clone https://github.com/marvin-ai/marvin-python-toolbox.git toolbox && \ + cd $MARVIN_TOOLBOX_HOME && \ + source /usr/local/bin/virtualenvwrapper.sh && \ + mkvirtualenv $MARVIN_TOOLBOX_ENV && \ + setvirtualenvproject && \ + make marvin \ No newline at end of file diff --git a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev new file mode 100644 index 0000000..80457a7 --- /dev/null +++ b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev @@ -0,0 +1,50 @@ +FROM allanbatista/marvin-base:0.0.2 + +MAINTAINER Allan Batista + +ENV DEBIAN_FRONTEND=noninteractive +ENV SLEEP_MILLIS 0 + +USER root + +EXPOSE 8888 +EXPOSE 8000 + +############################################################## +# Install Spark and Hadoop +############################################################## + +ENV SPARK_HOME=/opt/spark +ENV SPARK_CONF_DIR=$SPARK_HOME/conf +ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR +ENV YARN_CONF_DIR=$SPARK_CONF_DIR + +RUN cd /tmp && \ + wget https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz && \ + tar -xf spark-2.1.1-bin-hadoop2.6.tgz && \ + mv /tmp/spark-2.1.1-bin-hadoop2.6 $SPARK_HOME && \ + rm /tmp/spark-2.1.1-bin-hadoop2.6.tgz + +############################################################## +# Create user +############################################################## + +RUN useradd --create-home --shell /bin/bash --user-group marvin && \ + usermod -aG root marvin && \ + echo "source /usr/local/bin/virtualenvwrapper.sh" >> /home/marvin/.bashrc + +############################################################## +# Test Marvin +############################################################## + +RUN source /usr/local/bin/virtualenvwrapper.sh && \ + workon $MARVIN_TOOLBOX_ENV && \ + marvin test + +############################################################## +# Create user permissions +############################################################## + +RUN chmod 775 -R $MARVIN_HOME && \ + rm -Rf /tmp/* && \ + chmod 775 /tmp \ No newline at end of file