Skip to content

Commit 4752614

Browse files
vincentsaragovincentsarago
committed
refact
1 parent ffb2c2d commit 4752614

File tree

21 files changed

+448
-341
lines changed

21 files changed

+448
-341
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 88 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -3,98 +3,147 @@ name: CI
33
on: [push]
44

55
jobs:
6-
base:
6+
build:
77
runs-on: ubuntu-latest
88
if: "!contains(github.event.head_commit.message, '[skip ci]')"
99
strategy:
1010
matrix:
11+
image-name: [base, base-2]
1112
gdal-version: [2.4, 3.1]
12-
1313
steps:
1414
- uses: actions/checkout@v2
15-
1615
- name: Set up Python
1716
uses: actions/setup-python@v1
1817
with:
1918
python-version: "3.x"
20-
2119
- name: Install dependencies
2220
run: |
2321
python -m pip install --upgrade pip
24-
python -m pip install boto3
25-
22+
python -m pip install boto3 click
2623
- name: Login to DockerHub
2724
uses: docker/login-action@v1
2825
with:
2926
username: ${{ secrets.DOCKERHUB_USERNAME }}
3027
password: ${{ secrets.DOCKERHUB_TOKEN }}
3128

32-
- name: Cache deps
29+
- name: Common build cache
3330
uses: actions/cache@v2
34-
id: cache-deps
31+
id: common-cache
3532
with:
3633
path: /tmp/docker
37-
key: ${{ runner.os }}-cache-gdal${{ matrix.gdal-version }}-${{ hashFiles(format('base/gdal{0}/Dockerfile', matrix.gdal-version)) }}
34+
key: ${{ matrix.image-name }}-${{ hashFiles(format('common/{0}/Dockerfile', matrix.image-name)) }}
35+
36+
- name: create base image
37+
if: steps.common-cache.outputs.cache-hit != 'true'
38+
run: |
39+
docker build -f common/${{ matrix.image-name }}/Dockerfile -t ${{ matrix.image-name }}:build .
40+
mkdir /tmp/docker
41+
docker image save -o /tmp/docker/${{ matrix.image-name }}.tar ${{ matrix.image-name }}:build
42+
43+
- name: Restore base image
44+
if: steps.common-cache.outputs.cache-hit != 'true'
45+
run: |
46+
docker image load -i /tmp/docker/${{ matrix.image-name }}.tar
47+
docker images
48+
49+
- name: GDAL build Cache
50+
uses: actions/cache@v2
51+
id: gdal-cache
52+
with:
53+
path: /tmp/docker
54+
key: gdal${{ matrix.gdal-version }}-${{ hashFiles(format('gdal{0}/Dockerfile', matrix.gdal-version)) }}-${{ hashFiles(format('common/{0}/Dockerfile', matrix.image-name)) }}
3855

3956
- name: create gdal docker image
40-
if: steps.cache-deps.outputs.cache-hit != 'true'
57+
if: steps.gdal-cache.outputs.cache-hit != 'true'
4158
run: |
4259
docker build \
43-
-f base/gdal${{ matrix.gdal-version }}/Dockerfile \
44-
-t lambgeo/lambda:gdal${{ matrix.gdal-version }} .
60+
--build-arg IMAGE_VERSION=${{ matrix.image-name }} \
61+
-f gdal${{ matrix.gdal-version }}/Dockerfile \
62+
-t lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} .
4563
mkdir /tmp/docker
4664
docker image save \
47-
-o /tmp/docker/base-gdal${{ matrix.gdal-version }}.tar \
48-
lambgeo/lambda:gdal${{ matrix.gdal-version }}
65+
-o /tmp/docker/lambda-${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar \
66+
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }}
4967
5068
- name: Restore dependencies
51-
if: steps.cache-deps.outputs.cache-hit == 'true'
69+
if: steps.gdal-cache.outputs.cache-hit == 'true'
5270
run: |
53-
docker image load -i /tmp/docker/base-gdal${{ matrix.gdal-version }}.tar
71+
docker image load -i /tmp/docker/lambda-${{ matrix.image-name }}-gdal${{ matrix.gdal-version }}.tar
5472
docker images
5573
5674
- name: Run Tests
5775
run: |
5876
docker run \
5977
-v ${{ github.workspace }}:/local --rm \
60-
lambgeo/lambda:gdal${{ matrix.gdal-version }} \
78+
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} \
6179
sh -c "cd /local/tests && sh tests.sh"
6280
63-
- name: Push to DockerHub and Publish Layer
81+
- name: Push to DockerHub
82+
if: contains(github.ref, 'tags')
83+
run: docker push lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }}
84+
85+
- name: Create and Publish Layer
6486
if: contains(github.ref, 'tags')
6587
run: |
6688
docker run \
6789
-v ${{ github.workspace }}:/local --rm \
68-
lambgeo/lambda:gdal${{ matrix.gdal-version }} \
90+
lambgeo/lambda-${{ matrix.image-name }}:gdal${{ matrix.gdal-version }} \
6991
sh -c "cd /local/scripts && sh create-lambda-layer.sh"
70-
docker push lambgeo/lambda:gdal${{ matrix.gdal-version }}
7192
72-
- name: Build RunTimes images
93+
python scripts/deploy-layer.py package.zip ${{ matrix.gdal-version }} ${{ matrix.image-version }}
94+
95+
base-runtime:
96+
needs: [build]
97+
runs-on: ubuntu-latest
98+
if: contains(github.ref, 'tags')
99+
strategy:
100+
matrix:
101+
gdal-version: [2.4, 3.1]
102+
steps:
103+
- uses: actions/checkout@v2
104+
- name: Login to DockerHub
105+
uses: docker/login-action@v1
106+
with:
107+
username: ${{ secrets.DOCKERHUB_USERNAME }}
108+
password: ${{ secrets.DOCKERHUB_TOKEN }}
109+
110+
- name: Push to DockerHub
73111
env:
74112
RUNTIMES: "py3.7"
75113
run: |
76114
for RUNTIME in ${RUNTIMES}; do
77115
docker build \
78116
--build-arg GDAL_VERSION=${{ matrix.gdal-version }} \
79-
-f base/runtimes/${RUNTIME} \
80-
-t lambgeo/lambda:gdal${{ matrix.gdal-version }}-${RUNTIME} .
81-
done
82-
# docker push lambgeo/lambda:gdal${GDAL_VERSION}-py${PYTHON_VERSION}
83-
117+
-f runtimes/${RUNTIME} \
118+
-t lambgeo/lambda-base:gdal${{ matrix.gdal-version }}-${RUNTIME} .
84119
85-
# PLACEHOLDER
86-
# base-2:
87-
# runs-on: ubuntu-latest
88-
# strategy:
89-
# matrix:
90-
# gdal-version: [2.4, 3.1]
120+
docker push lambgeo/lambda-base:gdal${{ matrix.gdal-version }}-${RUNTIME}
121+
done
91122
92-
# steps:
93-
# - uses: actions/checkout@v2
94-
# - name: Login to DockerHub
95-
# uses: docker/login-action@v1
96-
# with:
97-
# username: ${{ secrets.DOCKERHUB_USERNAME }}
98-
# password: ${{ secrets.DOCKERHUB_TOKEN }}
123+
base-2-runtime:
124+
needs: [build]
125+
runs-on: ubuntu-latest
126+
if: contains(github.ref, 'tags')
127+
strategy:
128+
matrix:
129+
gdal-version: [2.4, 3.1]
130+
steps:
131+
- uses: actions/checkout@v2
132+
- name: Login to DockerHub
133+
uses: docker/login-action@v1
134+
with:
135+
username: ${{ secrets.DOCKERHUB_USERNAME }}
136+
password: ${{ secrets.DOCKERHUB_TOKEN }}
99137

138+
- name: Push to DockerHub
139+
env:
140+
RUNTIMES: "py3.8"
141+
run: |
142+
for RUNTIME in ${RUNTIMES}; do
143+
docker build \
144+
--build-arg GDAL_VERSION=${{ matrix.gdal-version }} \
145+
-f runtimes/${RUNTIME} \
146+
-t lambgeo/lambda-base-2:gdal${{ matrix.gdal-version }}-${RUNTIME} .
100147
148+
docker push lambgeo/lambda-base-2:gdal${{ matrix.gdal-version }}-${RUNTIME}
149+
done

‎README.md‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<img src="" style="max-width: 800px;" alt="docker-lambda"></a>
55
</p>
66
<p align="center">
7-
<em>Create an AWS lambda like docker images and lambda layer with GDAL.</em>
7+
<em>AWS lambda (Amazonlinux) like docker images and lambda layer with GDAL.</em>
88
</p>
99
<p align="center">
1010
<a href="https://github.com/cogeotiff/rio-tiler/actions?query=workflow%3ACI" target="_blank">
@@ -14,37 +14,47 @@
1414

1515

1616
# Docker Images
17+
1718
Based on lambci/lambda-base:build (amazonlinux)
1819
- GDAL 3.1.3 (Oct. 2020)
19-
- **lambgeo/lambda:gdal3.1**
20+
- **lambgeo/lambda-base:gdal3.1**
21+
- **lambgeo/lambda-base:gdal3.1-py3.7**
2022

2123
- GDAL 2.4.4 (June 2020)
22-
- **lambgeo/lambda:gdal2.4**
24+
- **lambgeo/lambda-base:gdal2.4**
25+
- **lambgeo/lambda-base:gdal2.4-py3.7**
2326

24-
<!-- Based on lambci/lambda-base-2:build (amazonlinux2)
27+
Based on lambci/lambda-base-2:build (amazonlinux2)
2528
- GDAL 3.1.3 (Oct. 2020)
26-
- **lambgeo/lambda:base2-gdal3.1**
29+
- **lambgeo/lambda-base-2:gdal3.1**
30+
- **lambgeo/lambda-base-2:gdal3.1-py3.8**
2731

2832
- GDAL 2.4.4 (June 2020)
29-
- **lambgeo/lambda:base2-gdal2.4** -->
33+
- **lambgeo/lambda-base-2:gdal2.4**
34+
- **lambgeo/lambda-base-2:gdal2.4-py3.8**
3035

3136

3237
# Lambda Layers
33-
We are publishing each gdal version as lambda layer on the AWS infrastructure.
3438

35-
**amazonlinux** (version 1)
39+
### **amazonlinux**
40+
41+
name | gdal | runtime | version | size (Mb)| unzipped size (Mb)| arn
42+
---| ---| ---| ---| ---| ---| ---
43+
gdal24 | 2.4.4| All | 2| 15.4| 50.1| arn:aws:lambda:us-east-1:524387336408:layer:gdal24:2
44+
gdal31 | 3.1.3| All | 2| 25| 64.5| arn:aws:lambda:us-east-1:524387336408:layer:gdal31:2
45+
3646

37-
name | gdal | runtime | version | size (Mb)| unzipped size (Mb)| arn
38-
---| ---| ---| ---| ---| ---| ---
39-
gdal31 | 3.1| All | 2| 24| 61.7| arn:aws:lambda:us-east-1:524387336408:layer:gdal31:2
40-
gdal24 | 2.4| All | 2| 14.8| 48.6| arn:aws:lambda:us-east-1:524387336408:layer:gdal24:2
47+
### **amazonlinux:2 (al2)**
4148

49+
name | gdal | runtime | version | size (Mb)| unzipped size (Mb)| arn
50+
---| ---| ---| ---| ---| ---| ---
51+
gdal24-al2 | 2.4.4| All | 1| 14| 41.7| arn:aws:lambda:us-east-1:524387336408:layer:gdal24-al2:1
52+
gdal31-al2 | 3.1.3| All | 1| 22.9| 53.6| arn:aws:lambda:us-east-1:524387336408:layer:gdal31-al2:1
4253

43-
<!-- **amazonlinux:2** (for latest runtime like python 3.8)-->
4454

4555
[Full list of version and ARN](/arns.json)
4656

47-
#### Regions
57+
### Regions
4858
- ap-northeast-1
4959
- ap-northeast-2
5060
- ap-south-1
@@ -62,7 +72,7 @@ gdal24 | 2.4| All | 2| 14.8| 48.6| arn:aws:lambd
6272
- us-west-1
6373
- us-west-2
6474

65-
#### content
75+
### content
6676

6777
```
6878
layer.zip

0 commit comments

Comments
 (0)