Skip to content

Commit 1a5dff4

Browse files
ericdaatericdaat
committed
Switch to Celery executor
1 parent 017ec9e commit 1a5dff4

File tree

5 files changed

+103
-16
lines changed

5 files changed

+103
-16
lines changed

‎README.md‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,18 @@
11
# Data Stack
22

3+
## Presentation
4+
35
A sample data stack running on Docker, that contains the following services:
46

5-
- [Airflow](https://airflow.apache.org/) (with its Postgres database)
6-
- [Metabase](https://metabase.com/) (with its Postgres database)
7-
- [MariaDB](https://mariadb.org/) (as main database)
7+
- [Airflow](https://airflow.apache.org/)
8+
- [Metabase](https://metabase.com/)
9+
- [MariaDB](https://mariadb.org/)
810

911
There is also a python package containing an example module, used by the
1012
example Airflow DAG.
1113

14+
## Usage
15+
1216
Run it with:
1317

1418
``` text
@@ -20,8 +24,8 @@ Then visit:
2024
- [localhost:3000](http://localhost:3000): for Metabase
2125
- [localhost:8080](http://localhost:8080): for Airflow
2226

23-
Add your Airflow DAGS in the [airflow-dags](./airflow-dags) folder.
27+
Add your Airflow DAGS in the [dags](./dags) folder.
2428

25-
Credits to:
29+
## References
2630

2731
- [puckel/docker-airflow](https://github.com/puckel/docker-airflow)

airflow-dags/example.py renamed to dags/example.py

File renamed without changes.

‎docker-compose.yml‎

Lines changed: 87 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ services:
1212
volumes:
1313
- ./docker-volumes/mariadb:/var/lib/mysql
1414

15+
############
16+
# Metabase #
17+
############
18+
1519
metabase:
1620
image: metabase/metabase
1721
ports:
@@ -35,22 +39,44 @@ services:
3539
volumes:
3640
- ./docker-volumes/metabase-postgres:/var/lib/postgresql/data/
3741

42+
###########
43+
# Airflow #
44+
###########
45+
46+
airflow-redis:
47+
image: 'redis:3.2.7'
48+
# command: redis-server --requirepass redispass
49+
50+
airflow-postgres:
51+
image: postgres:9.6
52+
environment:
53+
- POSTGRES_USER=airflow
54+
- POSTGRES_PASSWORD=airflow
55+
- POSTGRES_DB=airflow
56+
volumes:
57+
- ./docker-volumes/airflow-postgres:/var/lib/postgresql/data/
58+
3859
airflow-webserver:
3960
image: puckel/docker-airflow:1.10.2
4061
restart: always
4162
depends_on:
4263
- airflow-postgres
64+
- airflow-redis
4365
environment:
4466
- LOAD_EX=n
45-
- EXECUTOR=Local
67+
- FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
68+
- EXECUTOR=Celery
4669
- POSTGRES_HOST=airflow-postgres
70+
- REDIS_HOST=airflow-redis
4771
- PYTHONPATH=${PYTHONPATH}:/opt
72+
# - POSTGRES_USER=airflow
73+
# - POSTGRES_PASSWORD=airflow
74+
# - POSTGRES_DB=airflow
75+
# - REDIS_PASSWORD=redispass
4876
volumes:
49-
# Airflow DAGS
50-
- ./airflow-dags:/usr/local/airflow/dags
77+
- ./dags:/usr/local/airflow/dags
5178
# Uncomment to include custom plugins
52-
# - ./airflow-plugins:/usr/local/airflow/plugins
53-
# Python packages
79+
# - ./plugins:/usr/local/airflow/plugins
5480
- ./python_package:/opt/python_package/
5581
ports:
5682
- "8080:8080"
@@ -61,11 +87,61 @@ services:
6187
timeout: 30s
6288
retries: 3
6389

64-
airflow-postgres:
65-
image: postgres:9.6
90+
flower:
91+
image: puckel/docker-airflow:1.10.2
92+
restart: always
93+
depends_on:
94+
- airflow-redis
6695
environment:
67-
- POSTGRES_USER=airflow
68-
- POSTGRES_PASSWORD=airflow
69-
- POSTGRES_DB=airflow
96+
- EXECUTOR=Celery
97+
- REDIS_HOST=airflow-redis
98+
- POSTGRES_HOST=airflow-postgres
99+
# - REDIS_PASSWORD=redispass
100+
ports:
101+
- "5555:5555"
102+
command: flower
103+
104+
airflow-scheduler:
105+
image: puckel/docker-airflow:1.10.2
106+
restart: always
107+
depends_on:
108+
- airflow-webserver
109+
volumes:
110+
- ./dags:/usr/local/airflow/dags
111+
# Uncomment to include custom plugins
112+
# - ./plugins:/usr/local/airflow/plugins
113+
- ./python_package:/opt/python_package/
114+
environment:
115+
- LOAD_EX=n
116+
- FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
117+
- EXECUTOR=Celery
118+
- POSTGRES_HOST=airflow-postgres
119+
- REDIS_HOST=airflow-redis
120+
- PYTHONPATH=${PYTHONPATH}:/opt
121+
# - POSTGRES_USER=airflow
122+
# - POSTGRES_PASSWORD=airflow
123+
# - POSTGRES_DB=airflow
124+
# - REDIS_PASSWORD=redispass
125+
command: scheduler
126+
127+
worker:
128+
image: puckel/docker-airflow:1.10.2
129+
restart: always
130+
depends_on:
131+
- airflow-scheduler
70132
volumes:
71-
- ./docker-volumes/airflow-postgres:/var/lib/postgresql/data/
133+
- ./dags:/usr/local/airflow/dags
134+
# Uncomment to include custom plugins
135+
# - ./plugins:/usr/local/airflow/plugins
136+
- ./python_package:/opt/python_package/
137+
environment:
138+
- FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho=
139+
- EXECUTOR=Celery
140+
- POSTGRES_HOST=airflow-postgres
141+
- REDIS_HOST=airflow-redis
142+
- PYTHONPATH=${PYTHONPATH}:/opt
143+
# - POSTGRES_USER=airflow
144+
# - POSTGRES_PASSWORD=airflow
145+
# - POSTGRES_DB=airflow
146+
# - REDIS_PASSWORD=redispass
147+
command: worker

‎python_package/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
""" An example python package
2+
"""

‎python_package/example_module.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,7 @@
1+
""" A python example module.
2+
"""
3+
14
def some_function():
5+
""" Basic function that returns a string
6+
"""
27
return "Hello, world!"

0 commit comments

Comments
 (0)