Skip to content

Commit a273107

Browse files
vincentsaragovincentsarago
authored
Merge pull request #25 from lambgeo/refact2
just do one thing but do it well
2 parents 0608ba2 + a664613 commit a273107

File tree

21 files changed

+268
-820
lines changed

21 files changed

+268
-820
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 31 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -8,45 +8,40 @@ jobs:
88
if: "!contains(github.event.head_commit.message, '[skip ci]')"
99
strategy:
1010
matrix:
11-
image-name: [base, base-2]
12-
gdal-version: [2.4, 3.1]
11+
image-name: ['base', 'base-2']
12+
gdal-version: [2.4, 3.1, 3.2]
1313
steps:
1414
- uses: actions/checkout@v2
15-
- name: Set up Python
16-
uses: actions/setup-python@v1
17-
with:
18-
python-version: "3.x"
19-
- name: Install dependencies
20-
run: |
21-
python -m pip install --upgrade pip
22-
python -m pip install boto3 click
2315
- name: Login to DockerHub
2416
uses: docker/login-action@v1
2517
with:
2618
username: ${{ secrets.DOCKERHUB_USERNAME }}
2719
password: ${{ secrets.DOCKERHUB_TOKEN }}
2820

29-
- name: Common build cache
21+
- name: set amazonlinux version
22+
id: amazonlinux
23+
run: echo "::set-output name=version::$(echo ${{ matrix.image-name }} | sed -E 's/base(-)?//g' )"
24+
shell: bash
25+
26+
- name: setup Common cache
3027
uses: actions/cache@v2
3128
id: common-cache
3229
with:
3330
path: /tmp/docker
3431
key: ${{ matrix.image-name }}-${{ hashFiles(format('common/{0}/Dockerfile', matrix.image-name)) }}
3532

36-
- name: Restore base image
33+
- name: Restore Common image
3734
if: steps.common-cache.outputs.cache-hit == 'true'
3835
run: docker image load -i /tmp/docker/${{ matrix.image-name }}.tar
3936

40-
- name: Build base image
41-
run: docker build -f common/${{ matrix.image-name }}/Dockerfile -t ${{ matrix.image-name }}:build .
42-
43-
- name: Save base image in Cache
37+
- name: Build and Cache Common image
4438
if: steps.common-cache.outputs.cache-hit != 'true'
4539
run: |
40+
docker build -f common/${{ matrix.image-name }}/Dockerfile -t ${{ matrix.image-name }}:build .
4641
mkdir -p /tmp/docker
4742
docker image save -o /tmp/docker/${{ matrix.image-name }}.tar ${{ matrix.image-name }}:build
4843
49-
- name: GDAL build cache
44+
- name: setup GDAL cache
5045
uses: actions/cache@v2
5146
id: gdal-cache
5247
with:
@@ -55,67 +50,39 @@ jobs:
5550

5651
- name: Restore gdal image
5752
if: steps.gdal-cache.outputs.cache-hit == 'true'
58-
run: docker image load -i /tmp/docker/lambda-${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar
53+
run: docker image load -i /tmp/docker/${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar
5954

60-
- name: Build gdal docker image
55+
- name: Build and Cache GDAL image
56+
if: steps.gdal-cache.outputs.cache-hit != 'true'
6157
run: |
6258
docker build \
6359
--build-arg IMAGE_VERSION=${{ matrix.image-name }} \
6460
-f gdal${{ matrix.gdal-version }}/Dockerfile \
65-
-t lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} .
66-
67-
- name: Save gdal image in Cache
68-
if: steps.gdal-cache.outputs.cache-hit != 'true'
69-
run: |
61+
-t lambgeo/lambda${{ steps.amazonlinux.outputs.version }}-gdal:${{ matrix.gdal-version }} .
7062
mkdir -p /tmp/docker
7163
docker image save \
72-
-o /tmp/docker/lambda-${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar \
73-
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }}
64+
-o /tmp/docker/${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar \
65+
lambgeo/lambda${{ steps.amazonlinux.outputs.version }}-gdal:${{ matrix.gdal-version }}
7466
7567
- name: Run Tests
7668
run: |
7769
docker run \
7870
-v ${{ github.workspace }}:/local --rm \
79-
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} \
71+
lambgeo/lambda${{ steps.amazonlinux.outputs.version }}-gdal:${{ matrix.gdal-version }} \
8072
sh -c "cd /local/tests && sh tests.sh"
8173
8274
- name: Push to DockerHub
83-
if: contains(github.ref, 'tags')
84-
run: docker push lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }}
75+
if: github.ref == 'refs/heads/master'
76+
run: docker push lambgeo/lambda${{ steps.amazonlinux.outputs.version }}-gdal:${{ matrix.gdal-version }}
8577

86-
- name: Create and Publish Layer
87-
if: contains(github.ref, 'tags')
78+
- name: Build Runtime and Push to DockerHub
79+
if: github.ref == 'refs/heads/master'
8880
run: |
89-
docker run \
90-
-v ${{ github.workspace }}:/local --rm \
91-
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} \
92-
sh -c "cd /local/scripts && sh create-lambda-layer.sh"
93-
94-
python scripts/deploy-layer.py package.zip ${{ matrix.gdal-version }} ${{ matrix.image-version }}
95-
96-
runtimes:
97-
needs: [build]
98-
runs-on: ubuntu-latest
99-
if: contains(github.ref, 'tags')
100-
strategy:
101-
matrix:
102-
runtimes: [py3.7, py3.8]
103-
steps:
104-
- uses: actions/checkout@v2
105-
- name: Login to DockerHub
106-
uses: docker/login-action@v1
107-
with:
108-
username: ${{ secrets.DOCKERHUB_USERNAME }}
109-
password: ${{ secrets.DOCKERHUB_TOKEN }}
110-
111-
- name: Push to DockerHub
112-
env:
113-
GDAL_VERSIONS: "2.4 3.1"
114-
run: |
115-
for GDAL_VERSION in ${GDAL_VERSIONS}; do
116-
docker build \
117-
--build-arg GDAL_VERSION=${GDAL_VERSION} \
118-
-f runtimes/${{ matrix.runtimes }} \
119-
-t lambgeo/lambda:gdal${GDAL_VERSION}-${{ matrix.runtimes }} .
120-
docker push lambgeo/lambda:gdal${GDAL_VERSION}-${{ matrix.runtimes }}
121-
done
81+
runtimes=$(if [ '${{ steps.amazonlinux.outputs.version }}' == '2' ]; then echo 'python3.8'; else echo 'python3.7'; fi)
82+
for runtime in ${runtimes}; do
83+
docker build \
84+
--build-arg GDAL_VERSION=${{ matrix.gdal-version }} \
85+
-f runtimes/${runtime} \
86+
-t lambgeo/lambda-gdal:${{ matrix.gdal-version }}-${runtime} .
87+
docker push lambgeo/lambda-gdal:${{ matrix.gdal-version }}-${runtime}
88+
done

‎README.md‎

Lines changed: 75 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,11 @@
1-
# STATUS: WIP
21

3-
This project is currently in a Work In Progress status. We are doing a global refactor.
4-
For the stable version of the docs please refer to https://github.com/lambgeo/docker-lambda/tree/ef66339724b1b7e1a375df912dfd58a9c59ac109
5-
6-
# GDAL based docker-lambda
2+
# GDAL based docker image made for AWS Lambda
73

84
<p align="center">
95
<img src="https://user-images.githubusercontent.com/10407788/95621320-7b226080-0a3f-11eb-8194-4b55a5555836.png" style="max-width: 800px;" alt="docker-lambda"></a>
106
</p>
117
<p align="center">
12-
<em>AWS lambda (Amazonlinux) like docker images and lambda layer with GDAL.</em>
8+
<em>AWS lambda (Amazonlinux) like docker images with GDAL.</em>
139
</p>
1410
<p align="center">
1511
<a href="https://github.com/cogeotiff/rio-tiler/actions?query=workflow%3ACI" target="_blank">
@@ -21,86 +17,105 @@ For the stable version of the docs please refer to https://github.com/lambgeo/do
2117
# Docker Images
2218

2319
Based on lambci/lambda-base:build (amazonlinux)
24-
- GDAL 3.1.3 (Oct. 2020)
25-
- **lambgeo/lambda-base:gdal3.1**
26-
- **lambgeo/lambda:gdal3.1-py3.7**
20+
- GDAL 3.2.0 (Oct. 2020)
21+
- **lambgeo/lambda-gdal:3.2**
22+
23+
- GDAL 3.1.4 (Oct. 2020)
24+
- **lambgeo/lambda-gdal:3.1**
2725

2826
- GDAL 2.4.4 (June 2020)
29-
- **lambgeo/lambda-base:gdal2.4**
30-
- **lambgeo/lambda:gdal2.4-py3.7**
27+
- **lambgeo/lambda-gdal:2.4**
28+
29+
- For python 3.7
30+
- **lambgeo/lambda-gdal:3.2-python3.7**
31+
- **lambgeo/lambda-gdal:3.1-python3.7**
32+
- **lambgeo/lambda-gdal:2.4-python3.7**
3133

32-
Based on lambci/lambda-base-2:build (amazonlinux2)
33-
- GDAL 3.1.3 (Oct. 2020)
34-
- **lambgeo/lambda-base-2:gdal3.1**
35-
- **lambgeo/lambda:gdal3.1-py3.8**
34+
Based on lambci/lambda-base-2:build (amazonlinux2) for newer runtimes (e.g python 3.8)
35+
- GDAL 3.2.0 (Oct. 2020)
36+
- **lambgeo/lambda2-gdal:3.2**
37+
38+
- GDAL 3.1.4 (Oct. 2020)
39+
- **lambgeo/lambda2-gdal:3.1**
3640

3741
- GDAL 2.4.4 (June 2020)
38-
- **lambgeo/lambda-base-2:gdal2.4**
39-
- **lambgeo/lambda:gdal2.4-py3.8**
42+
- **lambgeo/lambda2-gdal:2.4**
43+
44+
- For python 3.8
45+
- **lambgeo/lambda-gdal:3.2-python3.8**
46+
- **lambgeo/lambda-gdal:3.1-python3.8**
47+
- **lambgeo/lambda-gdal:2.4-python3.8**
4048

49+
## Creating Lambda packages
4150

42-
# Lambda Layers
51+
1. Dockerfile
4352

44-
### **amazonlinux**
53+
```Dockerfile
54+
FROM lambgeo/lambda-gdal:3.2-python3.8
4555

46-
name | gdal | runtime | version | size (Mb)| unzipped size (Mb)| arn
47-
---| ---| ---| ---| ---| ---| ---
48-
gdal24 | 2.4.4| All | 2| 15.4| 50.1| arn:aws:lambda:us-east-1:524387336408:layer:gdal24:2
49-
gdal31 | 3.1.3| All | 2| 25| 64.5| arn:aws:lambda:us-east-1:524387336408:layer:gdal31:2
56+
ENV PACKAGE_PREFIX=/var/task
5057

58+
# Copy any local files to the package
59+
COPY handler.py ${PACKAGE_PREFIX}/handler.py
5160

52-
### **amazonlinux:2 (al2)**
61+
# Install some requirements
62+
RUN pip install numpy rasterio mercantile --no-binary :all: -t ${PACKAGE_PREFIX}/
5363

54-
name | gdal | runtime | version | size (Mb)| unzipped size (Mb)| arn
55-
---| ---| ---| ---| ---| ---| ---
56-
gdal24-al2 | 2.4.4| All | 1| 14| 41.7| arn:aws:lambda:us-east-1:524387336408:layer:gdal24-al2:1
57-
gdal31-al2 | 3.1.3| All | 1| 22.9| 53.6| arn:aws:lambda:us-east-1:524387336408:layer:gdal31-al2:1
64+
# Cleanup the package of useless files
65+
RUN rm -rdf $PACKAGE_PREFIX/boto3/ \
66+
&& rm -rdf $PACKAGE_PREFIX/botocore/ \
67+
&& rm -rdf $PACKAGE_PREFIX/docutils/ \
68+
&& rm -rdf $PACKAGE_PREFIX/dateutil/ \
69+
&& rm -rdf $PACKAGE_PREFIX/jmespath/ \
70+
&& rm -rdf $PACKAGE_PREFIX/s3transfer/ \
71+
&& rm -rdf $PACKAGE_PREFIX/numpy/doc/ \
72+
&& rm -rdf $PREFIX/share/doc \
73+
&& rm -rdf $PREFIX/share/man \
74+
&& rm -rdf $PREFIX/share/hdf*
5875

76+
# Reduce size of the C libs
77+
RUN cd $PREFIX && find lib -name \*.so\* -exec strip {} \;
5978

60-
[Full list of version and ARN](/arns.json)
79+
# Copy python files
80+
RUN cd $PACKAGE_PREFIX && zip -r9q /tmp/package.zip *
6181

62-
### Regions
63-
- ap-northeast-1
64-
- ap-northeast-2
65-
- ap-south-1
66-
- ap-southeast-1
67-
- ap-southeast-2
68-
- ca-central-1
69-
- eu-central-1
70-
- eu-north-1
71-
- eu-west-1
72-
- eu-west-2
73-
- eu-west-3
74-
- sa-east-1
75-
- us-east-1
76-
- us-east-2
77-
- us-west-1
78-
- us-west-2
82+
# Copy shared libs
83+
RUN cd $PREFIX && zip -r9q --symlinks /tmp/package.zip lib/*.so* share
84+
RUN cd $PREFIX && zip -r9q --symlinks /tmp/package.zip bin/gdal* bin/ogr* bin/geos* bin/nearblack
85+
```
7986

80-
### content
87+
2. Build and create package.zip
8188

89+
```bash
90+
docker build --tag package:latest .
91+
docker run --name lambda -w /var/task --volume $(shell pwd)/:/local -itd package:latest bash
92+
docker cp lambda:/tmp/package.zip package.zip
93+
docker stop lambda
94+
docker rm lambda
8295
```
83-
layer.zip
96+
Package content should be like:
97+
```
98+
package.zip
8499
|
85-
|___ bin/ # Binaries
86100
|___ lib/ # Shared libraries (GDAL, PROJ, GEOS...)
87101
|___ share/ # GDAL/PROJ data directories
102+
|___ rasterio/
103+
....
104+
|___ handler.py
105+
|___ other python module
88106
```
89107

90-
## AWS Lambda config
91-
92-
When using lambgeo layer you **HAVE TO** set GDAL_DATA and PROJ_LIB environment variable.
108+
3. Deploy and Set Environment variables
93109

94-
- When using lambgeo gdal layer
110+
For Rasterio or other libraries to be aware of GDAL/PROJ C libraries, you need to set up those 2 envs:
111+
- **GDAL_DATA:** /var/task/share/gdal
112+
- **PROJ_LIB:** /var/task/share/proj
95113

96-
- **GDAL_DATA:** /opt/share/gdal
97-
- **PROJ_LIB:** /opt/share/proj
114+
### Other variable
98115

99-
- If you create a package using the gdalX.X docker image.
116+
Starting with gdal3.1 (PROJ 7.1), you can set `PROJ_NETWORK=ON` to use remote grids: https://proj.org/usage/network.html
100117

101-
- **GDAL_DATA:** /var/task/share/gdal
102-
- **PROJ_LIB:** /var/task/share/proj
103118

104-
### Other variable
119+
### Refactor
105120

106-
Starting with gdal3.1 (PROJ 7.1), you can set `PROJ_NETWORK=ON` to use remote grids: https://proj.org/usage/network.html
121+
We recently refactored the repo, to see old documentation please refer to https://github.com/lambgeo/docker-lambda/tree/ef66339724b1b7e1a375df912dfd58a9c59ac109

0 commit comments

Comments
 (0)