add layers

Author: vincentsarago

.github/workflows/ci.yml

@@ -16,12 +16,30 @@ jobs:
         gdal-version: [2.4, 3.1, 3.2]
     steps:
       - uses: actions/checkout@v2
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install docker boto3 click
+
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
       - name: set amazonlinux version
         id: amazonlinux
         run: echo "::set-output name=version::$(if [ '${{ matrix.image-name }}' == 'base-2' ]; then echo '-al2'; else echo ''; fi)"
@@ -32,7 +50,7 @@ jobs:
         id: common-cache
         with:
           path: /tmp/docker
-          key: ${{ matrix.image-name }}-${{ hashFiles(format('common/{0}/Dockerfile', matrix.image-name)) }}
+          key: ${{ matrix.image-name }}-${{ hashFiles(format('dockerfiles/common/{0}/Dockerfile', matrix.image-name)) }}
 
       - name: Restore Common image
         if: steps.common-cache.outputs.cache-hit == 'true'
@@ -41,7 +59,7 @@ jobs:
       - name: Build and Cache Common image
         if: steps.common-cache.outputs.cache-hit != 'true'
         run: |
-          docker build -f common/${{ matrix.image-name }}/Dockerfile -t ${{ matrix.image-name }}:build .
+          docker build -f dockerfiles/common/${{ matrix.image-name }}/Dockerfile -t ${{ matrix.image-name }}:build .
           mkdir -p /tmp/docker
           docker image save -o /tmp/docker/${{ matrix.image-name }}.tar ${{ matrix.image-name }}:build
 
@@ -50,7 +68,7 @@ jobs:
         id: gdal-cache
         with:
           path: /tmp/docker
-          key: gdal${{ matrix.gdal-version }}-${{ hashFiles(format('gdal{0}/Dockerfile', matrix.gdal-version)) }}-${{ hashFiles(format('common/{0}/Dockerfile', matrix.image-name)) }}
+          key: gdal${{ matrix.gdal-version }}-${{ hashFiles(format('dockerfiles/gdal{0}/Dockerfile', matrix.gdal-version)) }}-${{ hashFiles(format('dockerfiles/common/{0}/Dockerfile', matrix.image-name)) }}
 
       - name: Restore gdal image
         if: steps.gdal-cache.outputs.cache-hit == 'true'
@@ -61,7 +79,7 @@ jobs:
         run: |
           docker build \
             --build-arg IMAGE_VERSION=${{ matrix.image-name }} \
-            -f gdal${{ matrix.gdal-version }}/Dockerfile \
+            -f dockerfiles/gdal${{ matrix.gdal-version }}/Dockerfile \
             -t lambgeo/lambda-gdal:${{ matrix.gdal-version }}${{ steps.amazonlinux.outputs.version }} .
           mkdir -p /tmp/docker
           docker image save \
@@ -75,6 +93,10 @@ jobs:
             lambgeo/lambda-gdal:${{ matrix.gdal-version }}${{ steps.amazonlinux.outputs.version }} \
             sh -c "cd /local/tests && sh tests.sh"
 
+      - name: Build and Deploy layers
+        if: github.ref == 'refs/heads/master'
+        run: python scripts/deploy.py ${{ matrix.gdal-version }} ${{ matrix.image-name }} --deploy
+
       - name: Push to DockerHub
         if: github.ref == 'refs/heads/master'
         run: docker push lambgeo/lambda-gdal:${{ matrix.gdal-version }}${{ steps.amazonlinux.outputs.version }}
@@ -86,7 +108,7 @@ jobs:
           for runtime in ${runtimes}; do
             docker build \
               --build-arg GDAL_VERSION=${{ matrix.gdal-version }} \
-              -f runtimes/${runtime} \
+              -f dockerfiles/runtimes/${runtime} \
               -t lambgeo/lambda-gdal:${{ matrix.gdal-version }}-${runtime} .
             docker push lambgeo/lambda-gdal:${{ matrix.gdal-version }}-${runtime}
           done

README.md

@@ -51,35 +51,63 @@ Based on lambci/lambda-base-2:build (amazonlinux2) for newer runtimes (e.g pytho
 1. Dockerfile
 
 ```Dockerfile
-FROM lambgeo/lambda-gdal:3.2-python3.8
+FROM lambgeo/lambda-gdal:3.2-al2 as gdal
+
+# We use lambci docker image for the runtime
+FROM lambci/lambda:build-python3.8
 
 ENV PACKAGE_PREFIX=/var/task
 
+# Bring C libs from lambgeo/lambda-gdal image
+COPY --from=gdal /opt/lib/ ${PACKAGE_PREFIX}/lib/
+COPY --from=gdal /opt/include/ ${PACKAGE_PREFIX}/include/
+COPY --from=gdal /opt/share/ ${PACKAGE_PREFIX}/share/
+COPY --from=gdal /opt/bin/ ${PACKAGE_PREFIX}/bin/
+ENV \
+  GDAL_DATA=${PACKAGE_PREFIX}/share/gdal \
+  PROJ_LIB=${PACKAGE_PREFIX}/share/proj \
+  GDAL_CONFIG=${PACKAGE_PREFIX}/bin/gdal-config \
+  GEOS_CONFIG=${PACKAGE_PREFIX}/bin/geos-config \
+  PATH=${PACKAGE_PREFIX}/bin:$PATH
+
+# Set some useful env
+ENV \
+  LANG=en_US.UTF-8 \
+  LC_ALL=en_US.UTF-8 \
+  CFLAGS="--std=c99"
+
 # Copy any local files to the package
 COPY handler.py ${PACKAGE_PREFIX}/handler.py
 
-# Install some requirements
+# This is needed for rasterio
+RUN pip3 install cython numpy --no-binary numpy
+
+# Install some requirements to `/var/task` (using `-t` otpion)
 RUN pip install numpy rasterio mercantile --no-binary :all: -t ${PACKAGE_PREFIX}/
 
-# Cleanup the package of useless files
-RUN rm -rdf $PACKAGE_PREFIX/boto3/ \
-  && rm -rdf $PACKAGE_PREFIX/botocore/ \
-  && rm -rdf $PACKAGE_PREFIX/docutils/ \
-  && rm -rdf $PACKAGE_PREFIX/dateutil/ \
-  && rm -rdf $PACKAGE_PREFIX/jmespath/ \
-  && rm -rdf $PACKAGE_PREFIX/s3transfer/ \
-  && rm -rdf $PACKAGE_PREFIX/numpy/doc/ \
-  && rm -rdf $PREFIX/share/doc \
-  && rm -rdf $PREFIX/share/man \
-  && rm -rdf $PREFIX/share/hdf*
+# Reduce size of the C libs
+RUN cd $PACKAGE_PREFIX && find lib -name \*.so\* -exec strip {} \;
+
+# Create package.zip
+RUN cd $PACKAGE_PREFIX && zip -r9q /tmp/package.zip *
+```
+
+Or if you are working with python, you can use lambgeo pre-build docker images:
+
+```Dockerfile
+FROM lambgeo/lambda-gdal:3.2-python3.8
+
+# Copy any local files to the package
+COPY handler.py ${PACKAGE_PREFIX}/handler.py
+
+# Install some requirements to `/var/task` (using `-t` otpion)
+RUN pip install numpy rasterio mercantile --no-binary :all: -t ${PACKAGE_PREFIX}/
 
 # Reduce size of the C libs
 RUN cd $PREFIX && find lib -name \*.so\* -exec strip {} \;
 
-# Copy python files
+# Create package.zip
 RUN cd $PACKAGE_PREFIX && zip -r9q /tmp/package.zip *
-
-# Copy shared libs
 RUN cd $PREFIX && zip -r9q --symlinks /tmp/package.zip lib/*.so* share
 RUN cd $PREFIX && zip -r9q --symlinks /tmp/package.zip bin/gdal* bin/ogr* bin/geos* bin/nearblack
 ```
@@ -97,6 +125,7 @@ Package content should be like:
 ```
 package.zip
   |
+  |___ bin/      # GDAL binaries
   |___ lib/      # Shared libraries (GDAL, PROJ, GEOS...)
   |___ share/    # GDAL/PROJ data directories
   |___ rasterio/
@@ -115,6 +144,128 @@ For Rasterio or other libraries to be aware of GDAL/PROJ C libraries, you need t
 
 Starting with gdal3.1 (PROJ 7.1), you can set `PROJ_NETWORK=ON` to use remote grids: https://proj.org/usage/network.html
 
+---
+
+## AWS Lambda Layers
+
+gdal | amazonlinux version| version | size (Mb)| unzipped size (Mb)| arn
+  ---|                 ---|      ---|       ---|                ---| ---
+3.2  |                   1|        1|      43.8|              138.8| arn:aws:lambda:{REGION}:524387336408:layer:gdal32:1
+3.1  |                   1|        1|      43.7|              128.4| arn:aws:lambda:{REGION}:524387336408:layer:gdal31:1
+2.4  |                   1|        1|      36.3|              121.3| arn:aws:lambda:{REGION}:524387336408:layer:gdal24:1
+  ---|                    |      ---|       ---|                ---| ---
+3.2  |                   2|        1|      44.4|                140| arn:aws:lambda:{REGION}:524387336408:layer:gdal32-al2:1
+3.1  |                   2|        1|      44.3|              139.7| arn:aws:lambda:{REGION}:524387336408:layer:gdal31-al2:1
+2.4  |                   2|        1|      36.7|                130| arn:aws:lambda:{REGION}:524387336408:layer:gdal24-al2:1
+
+**Layer content:**
+
+```
+layer.zip
+  |
+  |___ bin/      # Binaries
+  |___ lib/      # Shared libraries (GDAL, PROJ, GEOS...)
+  |___ share/    # GDAL/PROJ data directories
+```
+
+The layer content will be unzip in `/opt` directory in AWS Lambda. For the python libs to be able to use the C libraries you have to make sure to set 2 important environment variables:
+
+- **GDAL_DATA:** /opt/share/gdal
+- **PROJ_LIB:** /opt/share/proj
+
+### How To
+
+There are 2 ways to use the layers:
+
+#### 1. Simple (No dependencies)
+
+If you don't need to add more runtime dependencies, you can just create a lambda package (zip file) with you lambda handler.
+
+```bash
+zip -r9q package.zip handler.py
+```
+
+**Content:**
+
+```
+package.zip
+  |___ handler.py   # aws lambda python handler
+```
+
+**AWS Lambda Config:**
+- arn: `arn:aws:lambda:us-east-1:524387336408:layer:gdal32:1` (example)
+- env:
+  - **GDAL_DATA:** /opt/share/gdal
+  - **PROJ_LIB:** /opt/share/proj
+- lambda handler: `handler.handler`
+
+
+#### 2. Advanced (need other python dependencies)
+
+If your lambda handler needs more dependencies you'll have to use the exact same environment. To ease this you can find the docker images for each lambda on docker hub.
+
+- Create a docker file
+
+```dockerfile
+FROM lambgeo/lambda-gdal:3.2-al2
+
+# We use lambci docker image for the runtime
+FROM lambci/lambda:build-python3.8
+
+# Bring C libs from lambgeo/lambda-gdal image
+COPY --from=gdal /opt/lib/ /opt/lib/
+COPY --from=gdal /opt/include/ /opt/include/
+COPY --from=gdal /opt/share/ /opt/share/
+COPY --from=gdal /opt/bin/ /opt/bin/
+ENV \
+  GDAL_DATA=/opt/share/gdal \
+  PROJ_LIB=/opt/share/proj \
+  GDAL_CONFIG=/opt/bin/gdal-config \
+  GEOS_CONFIG=/opt/bin/geos-config \
+  PATH=/opt/bin:$PATH
+
+# Set some useful env
+ENV \
+  LANG=en_US.UTF-8 \
+  LC_ALL=en_US.UTF-8 \
+  CFLAGS="--std=c99"
+
+ENV PYTHONUSERBASE=/var/task
+
+# Install dependencies
+COPY handler.py $PYTHONUSERBASE/handler.py
+
+# Here we use the `--user` option to make sure to not replicate modules.
+RUN pip install rio-tiler --user
+
+# Move some files around
+RUN mv ${PYTHONUSERBASE}/lib/python3.8/site-packages/* ${PYTHONUSERBASE}/
+RUN rm -rf ${PYTHONUSERBASE}/lib
+
+echo "Create archive"
+RUN cd $PYTHONUSERBASE && zip -r9q /tmp/package.zip *
+```
+
+- create package
+```bash
+docker build --tag package:latest .
+docker run --name lambda -w /var/task -itd package:latest bash
+docker cp lambda:/tmp/package.zip package.zip
+docker stop lambda
+docker rm lambda
+```
+
+**Content:**
+
+```
+package.zip
+  |___ handler.py   # aws lambda python handler
+  |___ module1/     # dependencies
+  |___ module2/
+  |___ module3/
+  |___ ...
+```
+
 
 ### Refactor
 

common/base-2/Dockerfile dockerfiles/common/base-2/Dockerfilecommon/base-2/Dockerfile renamed to dockerfiles/common/base-2/Dockerfile

@@ -0,0 +1,19 @@
+#!/bin/bash
+echo "-----------------------"
+echo "Creating lambda layer"
+echo "-----------------------"
+
+echo "Remove useless files"
+rm -rdf $PREFIX/share/doc \
+&& rm -rdf $PREFIX/share/man \
+&& rm -rdf $PREFIX/share/hdf*
+
+echo "Strip shared libraries"
+cd $PREFIX && find lib -name \*.so\* -exec strip {} \;
+
+echo "Create archives"
+cd $PREFIX && zip -r9q --symlinks /tmp/package.zip lib/*.so*
+cd $PREFIX && zip -r9q --symlinks /tmp/package.zip share
+cd $PREFIX && zip -r9q --symlinks /tmp/package.zip bin/gdal* bin/ogr* bin/geos* bin/nearblack
+
+cp /tmp/package.zip /local/package.zip