Skip to content

Commit eb0d497

Browse files
vincentsaragovincentsarago
committed
just do one thing but do it well
1 parent 0608ba2 commit eb0d497

File tree

19 files changed

+225
-795
lines changed

19 files changed

+225
-795
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 40 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,10 @@ jobs:
88
if: "!contains(github.event.head_commit.message, '[skip ci]')"
99
strategy:
1010
matrix:
11-
image-name: [base, base-2]
11+
image-name: ['lambda', 'lambda2']
1212
gdal-version: [2.4, 3.1]
1313
steps:
1414
- uses: actions/checkout@v2
15-
- name: Set up Python
16-
uses: actions/setup-python@v1
17-
with:
18-
python-version: "3.x"
19-
- name: Install dependencies
20-
run: |
21-
python -m pip install --upgrade pip
22-
python -m pip install boto3 click
2315
- name: Login to DockerHub
2416
uses: docker/login-action@v1
2517
with:
@@ -46,52 +38,42 @@ jobs:
4638
mkdir -p /tmp/docker
4739
docker image save -o /tmp/docker/${{ matrix.image-name }}.tar ${{ matrix.image-name }}:build
4840
49-
- name: GDAL build cache
50-
uses: actions/cache@v2
51-
id: gdal-cache
52-
with:
53-
path: /tmp/docker
54-
key: gdal${{ matrix.gdal-version }}-${{ hashFiles(format('gdal{0}/Dockerfile', matrix.gdal-version)) }}-${{ hashFiles(format('common/{0}/Dockerfile', matrix.image-name)) }}
55-
56-
- name: Restore gdal image
57-
if: steps.gdal-cache.outputs.cache-hit == 'true'
58-
run: docker image load -i /tmp/docker/lambda-${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar
59-
60-
- name: Build gdal docker image
61-
run: |
62-
docker build \
63-
--build-arg IMAGE_VERSION=${{ matrix.image-name }} \
64-
-f gdal${{ matrix.gdal-version }}/Dockerfile \
65-
-t lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} .
66-
67-
- name: Save gdal image in Cache
68-
if: steps.gdal-cache.outputs.cache-hit != 'true'
69-
run: |
70-
mkdir -p /tmp/docker
71-
docker image save \
72-
-o /tmp/docker/lambda-${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar \
73-
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }}
74-
75-
- name: Run Tests
76-
run: |
77-
docker run \
78-
-v ${{ github.workspace }}:/local --rm \
79-
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} \
80-
sh -c "cd /local/tests && sh tests.sh"
81-
82-
- name: Push to DockerHub
83-
if: contains(github.ref, 'tags')
84-
run: docker push lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }}
85-
86-
- name: Create and Publish Layer
87-
if: contains(github.ref, 'tags')
88-
run: |
89-
docker run \
90-
-v ${{ github.workspace }}:/local --rm \
91-
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} \
92-
sh -c "cd /local/scripts && sh create-lambda-layer.sh"
93-
94-
python scripts/deploy-layer.py package.zip ${{ matrix.gdal-version }} ${{ matrix.image-version }}
41+
# - name: GDAL build cache
42+
# uses: actions/cache@v2
43+
# id: gdal-cache
44+
# with:
45+
# path: /tmp/docker
46+
# key: gdal${{ matrix.gdal-version }}-${{ hashFiles(format('gdal{0}/Dockerfile', matrix.gdal-version)) }}-${{ hashFiles(format('common/{0}/Dockerfile', matrix.image-name)) }}
47+
48+
# - name: Restore gdal image
49+
# if: steps.gdal-cache.outputs.cache-hit == 'true'
50+
# run: docker image load -i /tmp/docker/lambda-${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar
51+
52+
# - name: Build gdal docker image
53+
# run: |
54+
# docker build \
55+
# --build-arg IMAGE_VERSION=${{ matrix.image-name }} \
56+
# -f gdal${{ matrix.gdal-version }}/Dockerfile \
57+
# -t lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} .
58+
59+
# - name: Save gdal image in Cache
60+
# if: steps.gdal-cache.outputs.cache-hit != 'true'
61+
# run: |
62+
# mkdir -p /tmp/docker
63+
# docker image save \
64+
# -o /tmp/docker/lambda-${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar \
65+
# lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }}
66+
67+
# - name: Run Tests
68+
# run: |
69+
# docker run \
70+
# -v ${{ github.workspace }}:/local --rm \
71+
# lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} \
72+
# sh -c "cd /local/tests && sh tests.sh"
73+
74+
# - name: Push to DockerHub
75+
# if: contains(github.ref, 'tags')
76+
# run: docker push lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }}
9577

9678
runtimes:
9779
needs: [build]
@@ -110,12 +92,12 @@ jobs:
11092

11193
- name: Push to DockerHub
11294
env:
113-
GDAL_VERSIONS: "2.4 3.1"
95+
GDAL_VERSIONS: "2.4 3.1 3.2"
11496
run: |
11597
for GDAL_VERSION in ${GDAL_VERSIONS}; do
11698
docker build \
11799
--build-arg GDAL_VERSION=${GDAL_VERSION} \
118100
-f runtimes/${{ matrix.runtimes }} \
119-
-t lambgeo/lambda:gdal${GDAL_VERSION}-${{ matrix.runtimes }} .
120-
docker push lambgeo/lambda:gdal${GDAL_VERSION}-${{ matrix.runtimes }}
101+
-t lambgeo/lambda-gdal:${GDAL_VERSION}-${{ matrix.runtimes }} .
102+
docker push lambgeo/lambda-gdal:${GDAL_VERSION}-${{ matrix.runtimes }}
121103
done

‎README.md‎

Lines changed: 67 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@
33
This project is currently in a Work In Progress status. We are doing a global refactor.
44
For the stable version of the docs please refer to https://github.com/lambgeo/docker-lambda/tree/ef66339724b1b7e1a375df912dfd58a9c59ac109
55

6-
# GDAL based docker-lambda
6+
# GDAL based docker image made for AWS Lambda
77

88
<p align="center">
99
<img src="https://user-images.githubusercontent.com/10407788/95621320-7b226080-0a3f-11eb-8194-4b55a5555836.png" style="max-width: 800px;" alt="docker-lambda"></a>
1010
</p>
1111
<p align="center">
12-
<em>AWS lambda (Amazonlinux) like docker images and lambda layer with GDAL.</em>
12+
<em>AWS lambda (Amazonlinux) like docker images with GDAL.</em>
1313
</p>
1414
<p align="center">
1515
<a href="https://github.com/cogeotiff/rio-tiler/actions?query=workflow%3ACI" target="_blank">
@@ -21,85 +21,95 @@ For the stable version of the docs please refer to https://github.com/lambgeo/do
2121
# Docker Images
2222

2323
Based on lambci/lambda-base:build (amazonlinux)
24-
- GDAL 3.1.3 (Oct. 2020)
25-
- **lambgeo/lambda-base:gdal3.1**
26-
- **lambgeo/lambda:gdal3.1-py3.7**
24+
- GDAL 3.2.0 (Oct. 2020)
25+
- **lambgeo/lambda-gdal:3.2**
26+
- **lambgeo/lambda-gdal:3.2-py3.7**
27+
28+
- GDAL 3.1.4 (Oct. 2020)
29+
- **lambgeo/lambda-gdal:3.1**
30+
- **lambgeo/lambda-gdal:3.1-py3.7**
2731

2832
- GDAL 2.4.4 (June 2020)
29-
- **lambgeo/lambda-base:gdal2.4**
30-
- **lambgeo/lambda:gdal2.4-py3.7**
33+
- **lambgeo/lambda-gdal:2.4**
34+
- **lambgeo/lambda-gdal:2.4-py3.7**
35+
36+
Based on lambci/lambda-base-2:build (amazonlinux2) for newer runtimes (e.g python 3.8)
37+
- GDAL 3.2.0 (Oct. 2020)
38+
- **lambgeo/lambda2-gdal:3.2**
39+
- **lambgeo/lambda2-gdal:3.2-py3.8**
3140

32-
Based on lambci/lambda-base-2:build (amazonlinux2)
33-
- GDAL 3.1.3 (Oct. 2020)
34-
- **lambgeo/lambda-base-2:gdal3.1**
35-
- **lambgeo/lambda:gdal3.1-py3.8**
41+
- GDAL 3.1.4 (Oct. 2020)
42+
- **lambgeo/lambda2-gdal:3.1**
43+
- **lambgeo/lambda2-gdal:3.1-py3.8**
3644

3745
- GDAL 2.4.4 (June 2020)
38-
- **lambgeo/lambda-base-2:gdal2.4**
39-
- **lambgeo/lambda:gdal2.4-py3.8**
46+
- **lambgeo/lambda2-gdal:2.4**
47+
- **lambgeo/lambda2-gdal:2.4-py3.8**
4048

49+
## Creating Lambda packages
4150

42-
# Lambda Layers
51+
1. Dockerfile
4352

44-
### **amazonlinux**
53+
```Dockerfile
54+
FROM lambgeo/lambda2-gdal:3.2-py3.8
4555

46-
name | gdal | runtime | version | size (Mb)| unzipped size (Mb)| arn
47-
---| ---| ---| ---| ---| ---| ---
48-
gdal24 | 2.4.4| All | 2| 15.4| 50.1| arn:aws:lambda:us-east-1:524387336408:layer:gdal24:2
49-
gdal31 | 3.1.3| All | 2| 25| 64.5| arn:aws:lambda:us-east-1:524387336408:layer:gdal31:2
56+
ENV PACKAGE_PREFIX=/var/task
5057

58+
# Copy any local files to the package
59+
COPY handler.py ${PACKAGE_PREFIX}/handler.py
5160

52-
### **amazonlinux:2 (al2)**
61+
# Install some requirements
62+
RUN pip install numpy rasterio mercantile --no-binary :all: -t ${PACKAGE_PREFIX}/
5363

54-
name | gdal | runtime | version | size (Mb)| unzipped size (Mb)| arn
55-
---| ---| ---| ---| ---| ---| ---
56-
gdal24-al2 | 2.4.4| All | 1| 14| 41.7| arn:aws:lambda:us-east-1:524387336408:layer:gdal24-al2:1
57-
gdal31-al2 | 3.1.3| All | 1| 22.9| 53.6| arn:aws:lambda:us-east-1:524387336408:layer:gdal31-al2:1
64+
# Cleanup the package of useless files
65+
RUN rm -rdf $PACKAGE_PREFIX/boto3/ \
66+
&& rm -rdf $PACKAGE_PREFIX/botocore/ \
67+
&& rm -rdf $PACKAGE_PREFIX/docutils/ \
68+
&& rm -rdf $PACKAGE_PREFIX/dateutil/ \
69+
&& rm -rdf $PACKAGE_PREFIX/jmespath/ \
70+
&& rm -rdf $PACKAGE_PREFIX/s3transfer/ \
71+
&& rm -rdf $PACKAGE_PREFIX/numpy/doc/ \
72+
&& rm -rdf $PREFIX/share/doc \
73+
&& rm -rdf $PREFIX/share/man \
74+
&& rm -rdf $PREFIX/share/hdf*
5875

76+
# Reduce size of the C libs
77+
RUN cd $PREFIX && find lib -name \*.so\* -exec strip {} \;
5978

60-
[Full list of version and ARN](/arns.json)
79+
# Copy python files
80+
RUN cd $PACKAGE_PREFIX && zip -r9q /tmp/package.zip *
6181

62-
### Regions
63-
- ap-northeast-1
64-
- ap-northeast-2
65-
- ap-south-1
66-
- ap-southeast-1
67-
- ap-southeast-2
68-
- ca-central-1
69-
- eu-central-1
70-
- eu-north-1
71-
- eu-west-1
72-
- eu-west-2
73-
- eu-west-3
74-
- sa-east-1
75-
- us-east-1
76-
- us-east-2
77-
- us-west-1
78-
- us-west-2
82+
# Copy shared libs
83+
RUN cd $PREFIX && zip -r9q --symlinks /tmp/package.zip lib/*.so* share
84+
RUN cd $PREFIX && zip -r9q --symlinks /tmp/package.zip bin/gdal* bin/ogr* bin/geos* bin/nearblack
85+
```
7986

80-
### content
87+
2. Build and create package.zip
8188

89+
```bash
90+
docker build --tag package:latest .
91+
docker run --name lambda -w /var/task --volume $(shell pwd)/:/local -itd package:latest bash
92+
docker cp lambda:/tmp/package.zip package.zip
93+
docker stop lambda
94+
docker rm lambda
95+
```
96+
Package content should be like:
8297
```
83-
layer.zip
98+
package.zip
8499
|
85-
|___ bin/ # Binaries
86100
|___ lib/ # Shared libraries (GDAL, PROJ, GEOS...)
87101
|___ share/ # GDAL/PROJ data directories
102+
|___ rasterio/
103+
....
104+
|___ handler.py
105+
|___ other python module
88106
```
89107

90-
## AWS Lambda config
91-
92-
When using lambgeo layer you **HAVE TO** set GDAL_DATA and PROJ_LIB environment variable.
93-
94-
- When using lambgeo gdal layer
95-
96-
- **GDAL_DATA:** /opt/share/gdal
97-
- **PROJ_LIB:** /opt/share/proj
98-
99-
- If you create a package using the gdalX.X docker image.
108+
3. Deploy and Set Environment variables
100109

101-
- **GDAL_DATA:** /var/task/share/gdal
102-
- **PROJ_LIB:** /var/task/share/proj
110+
For Rasterio or other libraries to be aware of GDAL/PROJ C libraries, you need to set up those 2 envs:
111+
- **GDAL_DATA:** /var/task/share/gdal
112+
- **PROJ_LIB:** /var/task/share/proj
103113

104114
### Other variable
105115

0 commit comments

Comments
 (0)