first commit

Author: ericdaat

.gitignore

@@ -0,0 +1,2 @@
+docker-volumes
+*.pyc

README.md

@@ -0,0 +1,24 @@
+# Data Stack
+
+A sample data stack running on Docker, that contains the following services:
+
+- [Airflow](https://airflow.apache.org/) (with its Postgres database)
+- [Metabase](https://metabase.com/) (with its Postgres database)
+- [MariaDB](https://mariadb.org/) (as main database)
+
+Run it with:
+
+``` text
+docker-compose up -d
+```
+
+Then visit:
+
+- [localhost:3000](http://localhost:3000): for Metabase
+- [localhost:8080](http://localhost:8080): for Airflow
+
+Add your Airflow DAGS in the [airflow-dags](./airflow-dags) folder.
+
+Credits to:
+
+- [puckel/docker-airflow](https://github.com/puckel/docker-airflow)

airflow-dags/example.py

@@ -0,0 +1,48 @@
+"""
+Code that goes along with the Airflow located at:
+http://airflow.readthedocs.org/en/latest/tutorial.html
+"""
+from airflow import DAG
+from airflow.operators.bash_operator import BashOperator
+from datetime import datetime, timedelta
+
+
+default_args = {
+    "owner": "airflow",
+    "depends_on_past": False,
+    "start_date": datetime(2015, 6, 1),
+    "email": ["airflow@airflow.com"],
+    "email_on_failure": False,
+    "email_on_retry": False,
+    "retries": 1,
+    "retry_delay": timedelta(minutes=5),
+    # 'queue': 'bash_queue',
+    # 'pool': 'backfill',
+    # 'priority_weight': 10,
+    # 'end_date': datetime(2016, 1, 1),
+}
+
+dag = DAG("tutorial", default_args=default_args, schedule_interval=timedelta(1))
+
+# t1, t2 and t3 are examples of tasks created by instantiating operators
+t1 = BashOperator(task_id="print_date", bash_command="date", dag=dag)
+
+t2 = BashOperator(task_id="sleep", bash_command="sleep 5", retries=3, dag=dag)
+
+templated_command = """
+    {% for i in range(5) %}
+        echo "{{ ds }}"
+        echo "{{ macros.ds_add(ds, 7)}}"
+        echo "{{ params.my_param }}"
+    {% endfor %}
+"""
+
+t3 = BashOperator(
+    task_id="templated",
+    bash_command=templated_command,
+    params={"my_param": "Parameter I passed in"},
+    dag=dag,
+)
+
+t2.set_upstream(t1)
+t3.set_upstream(t1)

docker-compose.yml

@@ -0,0 +1,67 @@
+version: "3"
+
+services:
+  mariadb:
+    image: mariadb
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_USER: user
+      MYSQL_PASSWORD: password
+    ports:
+      - 3306:3306
+    volumes:
+      - ./docker-volumes/mariadb:/var/lib/mysql
+
+  metabase:
+      image: metabase/metabase
+      ports:
+        - 3000:3000
+      depends_on:
+        - metabase-postgres
+      environment:
+        MB_DB_TYPE: postgres
+        MB_DB_DBNAME: metabase
+        MB_DB_PORT: 5432
+        MB_DB_USER: user
+        MB_DB_PASS: password
+        MB_DB_HOST: metabase-postgres
+
+  metabase-postgres:
+    image: postgres
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_USER: user
+      POSTGRES_DB: metabase
+    volumes:
+      - ./docker-volumes/metabase-postgres:/var/lib/postgresql/data/
+
+  airflow-webserver:
+      image: puckel/docker-airflow:1.10.2
+      restart: always
+      depends_on:
+          - airflow-postgres
+      environment:
+          - LOAD_EX=n
+          - EXECUTOR=Local
+          - POSTGRES_HOST=airflow-postgres
+      volumes:
+          - ./airflow-dags:/usr/local/airflow/dags
+          # Uncomment to include custom plugins
+          # - ./airflow-plugins:/usr/local/airflow/plugins
+      ports:
+          - "8080:8080"
+      command: webserver
+      healthcheck:
+          test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
+          interval: 30s
+          timeout: 30s
+          retries: 3
+
+  airflow-postgres:
+      image: postgres:9.6
+      environment:
+          - POSTGRES_USER=airflow
+          - POSTGRES_PASSWORD=airflow
+          - POSTGRES_DB=airflow
+      volumes:
+      - ./docker-volumes/airflow-postgres:/var/lib/postgresql/data/