From dc24e5a6f16944b7badd5f670a931273274d7249 Mon Sep 17 00:00:00 2001 From: Allan Batista Date: Tue, 27 Feb 2018 13:35:22 -0300 Subject: [PATCH 1/3] Init create Dockerfile-base --- .../python-engine/docker/Dockerfile-base | 73 +++++++++++++++++++ .../python-engine/docker/Dockerfile-dev | 15 ++++ 2 files changed, 88 insertions(+) create mode 100644 marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base create mode 100644 marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev diff --git a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base new file mode 100644 index 0000000..5e3fb60 --- /dev/null +++ b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base @@ -0,0 +1,73 @@ +FROM ubuntu:16.04 + +MAINTAINER Allan Batista + +SHELL ["/bin/bash", "-c"] + +############################################################## +# Define all environment variables to be used +############################################################## + +ENV DEBIAN_FRONTEND=noninteractive +ENV SLEEP_MILLIS 0 +ENV SHELL=/bin/bash +ENV MARVIN_HOME=/opt/marvin +ENV MARVIN_DATA_PATH=$MARVIN_HOME/marvin-data +ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine +ENV MARVIN_ENGINE_ENV=marvin-engine-env +ENV WORKON_HOME=$MARVIN_HOME/.virtualenvs +RUN chsh -s /usr/local/bin/bash root +WORKDIR $MARVIN_HOME + +USER root + +############################################################## +# Install the system dependencies for default installation +############################################################## + +RUN apt-get update -y && \ + apt-get install -y build-essential && \ + apt-get install -y maven git python cmake software-properties-common curl libstdc++6 wget python2.7-dev python-pip ipython libffi-dev libssl-dev libxml2-dev libxslt1-dev libpng12-dev libfreetype6-dev python-tk libsasl2-dev python-pip graphviz unzip + +# Install Oracle JDK +RUN add-apt-repository ppa:webupd8team/java -y && \ + apt-get -qq update && \ + echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ + echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ + apt-get install -y oracle-java8-installer + +RUN pip install --upgrade pip && \ + pip install virtualenvwrapper + +RUN apt-get clean + +############################################################## +# Create all folders needed +############################################################## + +RUN mkdir -p $MARVIN_HOME && \ + mkdir -p $MARVIN_DATA_PATH && \ + mkdir -p $MARVIN_ENGINE_HOME && \ + mkdir -p /var/log/marvin/engines && \ + mkdir -p /var/run/marvin/engines + +############################################################## +# Install toolbox +############################################################## + +RUN cd /tmp && \ + wget https://github.com/marvin-ai/marvin-python-toolbox/archive/v0.0.2.zip && \ + unzip v0.0.2.zip && \ + cd marvin-python-toolbox-0.0.2 && \ + mv * $MARVIN_ENGINE_HOME + +RUN echo "source /usr/local/bin/virtualenvwrapper.sh" >> ~/.bashrc + +############################################################## +# Create the virtualenv configuration +############################################################## + +RUN source /usr/local/bin/virtualenvwrapper.sh && \ + mkvirtualenv $MARVIN_ENGINE_ENV && \ + workon $MARVIN_ENGINE_ENV && \ + pip install . --process-dependency-links \ No newline at end of file diff --git a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev new file mode 100644 index 0000000..d72a69d --- /dev/null +++ b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev @@ -0,0 +1,15 @@ +FROM marvin-base:0.0.2 + +MAINTAINER Allan Batista + +ENV DEBIAN_FRONTEND=noninteractive +ENV SLEEP_MILLIS 0 + +USER root + +############################################################## +# Create user +############################################################## + +RUN adduser marvin --quiet && \ + usermod -aG sudo root marvin \ No newline at end of file From 7a5a63130ce8bfe572882723baba7e634f2ca32f Mon Sep 17 00:00:00 2001 From: Allan Batista Date: Wed, 28 Feb 2018 17:33:43 -0300 Subject: [PATCH 2/3] create a docker image do developer --- README.md | 24 +++++++-- .../python-engine/docker/Dockerfile-base | 49 ++++++++----------- .../python-engine/docker/Dockerfile-dev | 37 ++++++++++++-- 3 files changed, 75 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 4e27c37..3f2914e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ echo "export MARVIN_DATA_PATH=$HOME/marvin/data" >> $HOME/.bash_profile echo "source virtualenvwrapper.sh" >> $HOME/.bash_profile source ~/.bash_profile -```` +``` 5. Clone and install python-toolbox @@ -71,7 +71,7 @@ mkvirtualenv python-toolbox-env setvirtualenvproject make marvin -```` +``` 6. Test the installation ``` @@ -179,5 +179,23 @@ marvin engine-generateenv ../engines/iris-species-engine/ workon iris-species-engine-env marvin engine-dryrun ``` +### Install Marvin using docker to developer + +``` +# expose ports 8000 and 8888 to random ports +$ docker run -it -P allanbatista/marvin-dev:0.0.2 /bin/bash -c "su marvin" + +# expose ports 8000 and 8888 +$ docker run -it -p 8000:8000 -p 8888:8888 allanbatista/marvin-dev:0.0.2 /bin/bash -c "su marvin" + +# use toolbox env +$ workon marvin-toolbox-env + +# create new engine +$ marvin engine-generate +$ workon NAME-engine-env +$ marvin notebook +``` + -> Marvin is a project started at B2W Digital offices and released open source on September 2017. +> Marvin is a project started at B2W Digital offices and released open source on September 2017. \ No newline at end of file diff --git a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base index 5e3fb60..e73ae87 100644 --- a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base +++ b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-base @@ -4,20 +4,9 @@ MAINTAINER Allan Batista SHELL ["/bin/bash", "-c"] -############################################################## -# Define all environment variables to be used -############################################################## - ENV DEBIAN_FRONTEND=noninteractive ENV SLEEP_MILLIS 0 ENV SHELL=/bin/bash -ENV MARVIN_HOME=/opt/marvin -ENV MARVIN_DATA_PATH=$MARVIN_HOME/marvin-data -ENV MARVIN_ENGINE_HOME=$MARVIN_HOME/engine -ENV MARVIN_ENGINE_ENV=marvin-engine-env -ENV WORKON_HOME=$MARVIN_HOME/.virtualenvs -RUN chsh -s /usr/local/bin/bash root -WORKDIR $MARVIN_HOME USER root @@ -37,37 +26,39 @@ RUN add-apt-repository ppa:webupd8team/java -y && \ apt-get install -y oracle-java8-installer RUN pip install --upgrade pip && \ - pip install virtualenvwrapper + pip install virtualenvwrapper && \ + echo "source /usr/local/bin/virtualenvwrapper.sh" >> ~/.bashrc RUN apt-get clean +############################################################## +# Define all environment variables to be used +############################################################## + +ENV MARVIN_HOME=/opt/marvin +ENV MARVIN_DATA_PATH=$MARVIN_HOME/data +ENV MARVIN_TOOLBOX_HOME=$MARVIN_HOME/toolbox +ENV MARVIN_TOOLBOX_ENV=marvin-toolbox-env +ENV WORKON_HOME=$MARVIN_TOOLBOX_HOME/.virtualenvs + ############################################################## # Create all folders needed ############################################################## RUN mkdir -p $MARVIN_HOME && \ mkdir -p $MARVIN_DATA_PATH && \ - mkdir -p $MARVIN_ENGINE_HOME && \ mkdir -p /var/log/marvin/engines && \ mkdir -p /var/run/marvin/engines -############################################################## -# Install toolbox -############################################################## - -RUN cd /tmp && \ - wget https://github.com/marvin-ai/marvin-python-toolbox/archive/v0.0.2.zip && \ - unzip v0.0.2.zip && \ - cd marvin-python-toolbox-0.0.2 && \ - mv * $MARVIN_ENGINE_HOME - -RUN echo "source /usr/local/bin/virtualenvwrapper.sh" >> ~/.bashrc +WORKDIR $MARVIN_HOME ############################################################## -# Create the virtualenv configuration +# Install toolbox ############################################################## -RUN source /usr/local/bin/virtualenvwrapper.sh && \ - mkvirtualenv $MARVIN_ENGINE_ENV && \ - workon $MARVIN_ENGINE_ENV && \ - pip install . --process-dependency-links \ No newline at end of file +RUN git clone https://github.com/marvin-ai/marvin-python-toolbox.git toolbox && \ + cd $MARVIN_TOOLBOX_HOME && \ + source /usr/local/bin/virtualenvwrapper.sh && \ + mkvirtualenv $MARVIN_TOOLBOX_ENV && \ + setvirtualenvproject && \ + make marvin \ No newline at end of file diff --git a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev index d72a69d..aaf3c67 100644 --- a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev +++ b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev @@ -1,4 +1,4 @@ -FROM marvin-base:0.0.2 +FROM allanbatista/marvin-base:0.0.2 MAINTAINER Allan Batista @@ -7,9 +7,40 @@ ENV SLEEP_MILLIS 0 USER root +EXPOSE 8888 +EXPOSE 8000 + +############################################################## +# Install Spark and Hadoop +############################################################## + +ENV SPARK_HOME=/opt/spark +ENV SPARK_CONF_DIR=$SPARK_HOME/conf +ENV HADOOP_CONF_DIR=$SPARK_CONF_DIR +ENV YARN_CONF_DIR=$SPARK_CONF_DIR + +RUN cd /tmp && \ + wget https://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.6.tgz && \ + tar -xf spark-2.1.1-bin-hadoop2.6.tgz && \ + mv /tmp/spark-2.1.1-bin-hadoop2.6 $SPARK_HOME && \ + rm /tmp/spark-2.1.1-bin-hadoop2.6.tgz + ############################################################## # Create user ############################################################## -RUN adduser marvin --quiet && \ - usermod -aG sudo root marvin \ No newline at end of file +RUN useradd --create-home --shell /bin/bash --user-group marvin && \ + usermod -aG root marvin && \ + echo "source /usr/local/bin/virtualenvwrapper.sh" >> /home/marvin/.bashrc && \ + chmod 775 $MARVIN_HOME && \ + chmod 775 $MARVIN_DATA_PATH && \ + chmod 775 $WORKON_HOME && \ + chmod 775 $MARVIN_TOOLBOX_HOME -R + +############################################################## +# Test Marvin +############################################################## + +RUN source /usr/local/bin/virtualenvwrapper.sh && \ + workon $MARVIN_TOOLBOX_ENV && \ + marvin test \ No newline at end of file From c6dd193ccb3bc5d7ae849b293d5042a352fe19ae Mon Sep 17 00:00:00 2001 From: Allan Batista Date: Fri, 16 Mar 2018 17:17:49 -0300 Subject: [PATCH 3/3] fix docker user permission --- .../python-engine/docker/Dockerfile-dev | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev index aaf3c67..80457a7 100644 --- a/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev +++ b/marvin_python_toolbox/management/templates/python-engine/docker/Dockerfile-dev @@ -31,11 +31,7 @@ RUN cd /tmp && \ RUN useradd --create-home --shell /bin/bash --user-group marvin && \ usermod -aG root marvin && \ - echo "source /usr/local/bin/virtualenvwrapper.sh" >> /home/marvin/.bashrc && \ - chmod 775 $MARVIN_HOME && \ - chmod 775 $MARVIN_DATA_PATH && \ - chmod 775 $WORKON_HOME && \ - chmod 775 $MARVIN_TOOLBOX_HOME -R + echo "source /usr/local/bin/virtualenvwrapper.sh" >> /home/marvin/.bashrc ############################################################## # Test Marvin @@ -43,4 +39,12 @@ RUN useradd --create-home --shell /bin/bash --user-group marvin && \ RUN source /usr/local/bin/virtualenvwrapper.sh && \ workon $MARVIN_TOOLBOX_ENV && \ - marvin test \ No newline at end of file + marvin test + +############################################################## +# Create user permissions +############################################################## + +RUN chmod 775 -R $MARVIN_HOME && \ + rm -Rf /tmp/* && \ + chmod 775 /tmp \ No newline at end of file