From 2aa77ea9b820455402693f89d3dbd41ec51d98a8 Mon Sep 17 00:00:00 2001 From: db260179 Date: Sat, 10 Nov 2018 17:33:22 +0000 Subject: [PATCH 1/8] Fix init.sh - indents are incorrect Fix EOSQL indents as breaking the sql commands Add check for already set MYSQL_GRANT_SLAVE_USER and password from environment variable --- mariadb/init.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/mariadb/init.sh b/mariadb/init.sh index 28ae16b..2f917a3 100755 --- a/mariadb/init.sh +++ b/mariadb/init.sh @@ -62,7 +62,7 @@ if [ ! -z "$MYSQL_MASTER_HOST" ]; then sleep 1 done - if [ ! -z "$MYSQL_GRANT_SLAVE_USER" -a ! -z "$MYSQL_GRANT_SLAVE_PASSWORD" ]; then + if [ -z "$MYSQL_GRANT_SLAVE_USER" ] || [ -z "$MYSQL_GRANT_SLAVE_PASSWORD" ]; then export MYSQL_GRANT_SLAVE_USER="$(pwgen -1 16)" echo "GENERATED SLAVE USER: $MYSQL_GRANT_SLAVE_USER" @@ -73,20 +73,29 @@ if [ ! -z "$MYSQL_MASTER_HOST" ]; then "${master_mysql[@]}" <<-EOSQL GRANT REPLICATION SLAVE ON *.* TO '$MYSQL_GRANT_SLAVE_USER'@'%' IDENTIFIED BY '$MYSQL_GRANT_SLAVE_PASSWORD'; FLUSH PRIVILEGES; - EOSQL +EOSQL fi + + # If the user sets these then just run the SQL grant for replicaton user + if [ ! -z "$MYSQL_GRANT_SLAVE_USER" ] || [ ! -z "$MYSQL_GRANT_SLAVE_PASSWORD" ]; then + + "${master_mysql[@]}" <<-EOSQL + GRANT REPLICATION SLAVE ON *.* TO '$MYSQL_GRANT_SLAVE_USER'@'%' IDENTIFIED BY '$MYSQL_GRANT_SLAVE_PASSWORD'; + FLUSH PRIVILEGES; +EOSQL + fi "${master_mysql[@]}" <<-EOSQL > $MYSQL_STATUS 2>&1 FLUSH TABLES WITH READ LOCK; SHOW MASTER STATUS; - EOSQL +EOSQL "${master_mysqldump[@]}" > $MYSQL_EXPORT 2>&1 "${master_mysql[@]}" <<-EOSQL UNLOCK TABLES; - EOSQL +EOSQL ## SLAVE @@ -120,7 +129,7 @@ if [ ! -z "$MYSQL_MASTER_HOST" ]; then MASTER_CONNECT_RETRY=$MYSQL_MASTER_CONNECT_RETRY; START SLAVE; SHOW SLAVE STATUS\G - EOSQL +EOSQL if [ -e "$MYSQL_STATUS" ]; then rm -f $MYSQL_STATUS @@ -151,7 +160,7 @@ else "${master_mysql[@]}" <<-EOSQL SHOW MASTER STATUS\G - EOSQL +EOSQL echo echo 'MySQL master process done. Ready for replication.' From 85687e8b8410e1c4c07b225144f7bd25cf1ba03f Mon Sep 17 00:00:00 2001 From: db260179 Date: Sat, 10 Nov 2018 17:46:29 +0000 Subject: [PATCH 2/8] Only run the init.sh on first install Copy the init.sh on first install then create a status file to ignore on further restarts --- mariadb/entrypoint.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mariadb/entrypoint.sh b/mariadb/entrypoint.sh index f759242..76f1d82 100755 --- a/mariadb/entrypoint.sh +++ b/mariadb/entrypoint.sh @@ -128,5 +128,10 @@ if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then fi fi fi - -docker-entrypoint.sh $@ \ No newline at end of file +# On first install cp the init.sh +if [ ! -e "/docker-entrypoint-initdb.d/.init" ]; then + chown mysql:mysql /usr/local/bin/init.sh + cp -f /usr/local/bin/init.sh /docker-entrypoint-initdb.d + touch /docker-entrypoint-initdb.d/.init +fi +docker-entrypoint.sh $@ From e115bd60ab4fa5b41aaf59804f39312821f00080 Mon Sep 17 00:00:00 2001 From: db260179 Date: Sat, 10 Nov 2018 17:48:19 +0000 Subject: [PATCH 3/8] Copy init.sh to /usr/local/bin then manage later Copying init.sh to /usr/local/bin allows us to use this only once on a fresh install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 13ed09b..179b381 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM mariadb:10.3 -COPY mariadb/init.sh /docker-entrypoint-initdb.d/ +COPY mariadb/init.sh /usr/local/bin/ COPY mariadb/entrypoint.sh /usr/local/bin/master-slave-entrypoint.sh ENTRYPOINT ["master-slave-entrypoint.sh"] From 51eeffa52859e78b0b36f991fa180b60949686aa Mon Sep 17 00:00:00 2001 From: db260179 Date: Sun, 11 Nov 2018 19:47:45 +0000 Subject: [PATCH 4/8] Remove docker-compose.dev.yml add readme changes --- README.md | 52 ++++++------------------------------------ docker-compose.dev.yml | 10 -------- 2 files changed, 7 insertions(+), 55 deletions(-) delete mode 100644 docker-compose.dev.yml diff --git a/README.md b/README.md index 6b2802e..f2ca47d 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,10 @@ > MariaDB 10.3 dockerized with replication. Master/Slave setup in 30 seconds. +> Added features for Docker swarm setups ## How to use -### Docker run - -@TODO - ### Docker compose Example docker-compose.yml for mariadb replication: @@ -18,7 +15,7 @@ version: '3' services: db-master: - image: caffeina/mariadb-replication + image: *your docker registry* restart: always volumes: - db-master-data:/var/lib/mysql:rw @@ -27,7 +24,7 @@ services: MYSQL_ROOT_PASSWORD: "mastersecret" db-slave: - image: caffeina/mariadb-replication + image: *your docker registry* restart: always depends_on: - db-master @@ -44,16 +41,14 @@ volumes: db-slave-data: ``` -@TODO - ```bash $ docker-compose up ``` -@TODO +# Build and run ```bash -$ docker-compose -f docker-compose.yml -f docker-compose.dev.yml up --build +$ docker-compose -f docker-compose.yml up --build ``` @@ -63,7 +58,7 @@ The `mariadb-replication` image is an extension of `mariadb:10.3`, you can use a ### Enviroment variables -@TODO Sets the connection parameters to the master. The `MYSQL_MASTER_HOST` variable is required. +Sets the connection parameters to the master (for slave container). The `MYSQL_MASTER_HOST` variable is required. ```yaml MYSQL_MASTER_HOST: "db-master" @@ -72,27 +67,21 @@ MYSQL_MASTER_USER: "root" # (Optional) Default: 'root' MYSQL_MASTER_PASSWORD: "secret" # (Optional) Default: '' ``` -@TODO The generated user and password will be printed to stdout. +The generated user and password will be printed to stdout. ```yaml MYSQL_GRANT_SLAVE_USER: "user" # (Optional) Default: *RANDOM STRING* MYSQL_GRANT_SLAVE_PASSWORD: "secret" # (Optional) Default: *RANDOM STRING* ``` -@TODO - ```yaml SERVER_ID: 2 # (Optional) Default: *RANDOM INT* ``` -@TODO - ```yaml EXPIRE_LOGS_DAYS: 5 # (Optional) Default: '10' ``` -@TODO - ```yaml MAX_BINLOG_SIZE: "50M" # (Optional) Default: '100M' ``` @@ -109,30 +98,3 @@ MYSQL_SLAVE_SKIP_ERRORS: "all" # (Optional) Default: 'OFF' - Add `innodb-read-only` parameter (Service restart on first run is needed); - Move `/etc/mysql/conf.d/master-slave.cnf` in other path (So the user can bind a volume to `/etc/mysql/conf.d/` for custom configuration); - Permit replication on existing database. - - -## Contributing - -How to get involved: - -1. [Star](https://github.com/gadiener/docker-mariadb-replication/stargazers) the project! -2. Answer questions that come through [GitHub issues](https://github.com/gadiener/docker-mariadb-replication/issues?state=open) -3. [Report a bug](https://github.com/gadiener/docker-mariadb-replication/issues/new) that you find - -This project follows the [GitFlow branching model](http://nvie.com/posts/a-successful-git-branching-model). The ```master``` branch always reflects a production-ready state while the latest development is taking place in the ```develop``` branch. - -Each time you want to work on a fix or a new feature, create a new branch based on the ```develop``` branch: ```git checkout -b BRANCH_NAME develop```. Only pull requests to the ```develop``` branch will be merged. - -Pull requests are **highly appreciated**. - -Solve a problem. Features are great, but even better is cleaning-up and fixing issues in the code that you discover. - - -## Versioning - -This project is maintained by using the [Semantic Versioning Specification (SemVer)](http://semver.org). - - -## Copyright and license - -Copyright 2017 [Caffeina](http://caffeina.com) srl under the [MIT license](LICENSE.md). \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml deleted file mode 100644 index 2f65fcb..0000000 --- a/docker-compose.dev.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '3' - -services: - db-master: - build: . - restart: on-failure - - db-slave: - build: . - restart: on-failure \ No newline at end of file From 2b5db6a727e99a0b4dd0dc5df081f86a07ab61ba Mon Sep 17 00:00:00 2001 From: db260179 Date: Mon, 12 Nov 2018 20:47:53 +0000 Subject: [PATCH 5/8] Remove .init, just check for init.sh No need to for the .init file as the mariadb container only runs on first run from the /docker-entrypoint-initdb.d But we need to check if the init.sh exists as we are not assuming that a user might mount the /docker-entrypoint-initdb.d as a volume The status folder will be used to when master is done and the slave is ready to proceed. --- mariadb/entrypoint.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mariadb/entrypoint.sh b/mariadb/entrypoint.sh index 76f1d82..e4f1cae 100755 --- a/mariadb/entrypoint.sh +++ b/mariadb/entrypoint.sh @@ -128,10 +128,12 @@ if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then fi fi fi -# On first install cp the init.sh -if [ ! -e "/docker-entrypoint-initdb.d/.init" ]; then - chown mysql:mysql /usr/local/bin/init.sh + +# Check if init.sh has been added and add status folder +if [ ! -e "/docker-entrypoint-initdb.d/init.sh" ]; then cp -f /usr/local/bin/init.sh /docker-entrypoint-initdb.d - touch /docker-entrypoint-initdb.d/.init + chown mysql:mysql /status + chown mysql:mysql /docker-entrypoint-initdb.d/* fi + docker-entrypoint.sh $@ From 5a3c5e80777c51e4e9f3671bd8dcebe4b7f6c512 Mon Sep 17 00:00:00 2001 From: db260179 Date: Mon, 12 Nov 2018 20:55:51 +0000 Subject: [PATCH 6/8] Add slave status checks as race conditions can occur By adding a status function in the init.sh it allows the master to finish its tasks first before the slave can start --- mariadb/init.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mariadb/init.sh b/mariadb/init.sh index 2f917a3..e54df97 100755 --- a/mariadb/init.sh +++ b/mariadb/init.sh @@ -53,6 +53,11 @@ if [ ! -z "$MYSQL_MASTER_HOST" ]; then master_mysqldump+=( "$MYSQL_DATABASE" ) fi + until [ -e /status/.done ]; do + echo 'Waiting for master to finish tasks..' + sleep 5 + done + for i in {30..0}; do if echo 'SELECT 1' | "${master_mysql[@]}" &> /dev/null; then echo 'MySQL master init process is complete...' @@ -165,4 +170,6 @@ EOSQL echo echo 'MySQL master process done. Ready for replication.' echo + echo 'Setting /status/.done for slave to start..' + touch /status/.done 2>&1 fi From 7e4fd18995f1cf1e8c23ed7c62caa440461288cc Mon Sep 17 00:00:00 2001 From: db260179 Date: Mon, 12 Nov 2018 21:01:01 +0000 Subject: [PATCH 7/8] Revert back the Readme Unintentional commit change revert back --- README.md | 52 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f2ca47d..0138211 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,13 @@ > MariaDB 10.3 dockerized with replication. Master/Slave setup in 30 seconds. -> Added features for Docker swarm setups ## How to use +### Docker run + +@TODO + ### Docker compose Example docker-compose.yml for mariadb replication: @@ -15,7 +18,7 @@ version: '3' services: db-master: - image: *your docker registry* + image: caffeina/mariadb-replication restart: always volumes: - db-master-data:/var/lib/mysql:rw @@ -24,7 +27,7 @@ services: MYSQL_ROOT_PASSWORD: "mastersecret" db-slave: - image: *your docker registry* + image: caffeina/mariadb-replication restart: always depends_on: - db-master @@ -41,14 +44,16 @@ volumes: db-slave-data: ``` +@TODO + ```bash $ docker-compose up ``` -# Build and run +@TODO ```bash -$ docker-compose -f docker-compose.yml up --build +$ docker-compose -f docker-compose.yml -f docker-compose.dev.yml up --build ``` @@ -58,7 +63,7 @@ The `mariadb-replication` image is an extension of `mariadb:10.3`, you can use a ### Enviroment variables -Sets the connection parameters to the master (for slave container). The `MYSQL_MASTER_HOST` variable is required. +@TODO Sets the connection parameters to the master. The `MYSQL_MASTER_HOST` variable is required. ```yaml MYSQL_MASTER_HOST: "db-master" @@ -67,21 +72,27 @@ MYSQL_MASTER_USER: "root" # (Optional) Default: 'root' MYSQL_MASTER_PASSWORD: "secret" # (Optional) Default: '' ``` -The generated user and password will be printed to stdout. +@TODO The generated user and password will be printed to stdout. ```yaml MYSQL_GRANT_SLAVE_USER: "user" # (Optional) Default: *RANDOM STRING* MYSQL_GRANT_SLAVE_PASSWORD: "secret" # (Optional) Default: *RANDOM STRING* ``` +@TODO + ```yaml SERVER_ID: 2 # (Optional) Default: *RANDOM INT* ``` +@TODO + ```yaml EXPIRE_LOGS_DAYS: 5 # (Optional) Default: '10' ``` +@TODO + ```yaml MAX_BINLOG_SIZE: "50M" # (Optional) Default: '100M' ``` @@ -98,3 +109,30 @@ MYSQL_SLAVE_SKIP_ERRORS: "all" # (Optional) Default: 'OFF' - Add `innodb-read-only` parameter (Service restart on first run is needed); - Move `/etc/mysql/conf.d/master-slave.cnf` in other path (So the user can bind a volume to `/etc/mysql/conf.d/` for custom configuration); - Permit replication on existing database. + + +## Contributing + +How to get involved: + +1. [Star](https://github.com/gadiener/docker-mariadb-replication/stargazers) the project! +2. Answer questions that come through [GitHub issues](https://github.com/gadiener/docker-mariadb-replication/issues?state=open) +3. [Report a bug](https://github.com/gadiener/docker-mariadb-replication/issues/new) that you find + +This project follows the [GitFlow branching model](http://nvie.com/posts/a-successful-git-branching-model). The ```master``` branch always reflects a production-ready state while the latest development is taking place in the ```develop``` branch. + +Each time you want to work on a fix or a new feature, create a new branch based on the ```develop``` branch: ```git checkout -b BRANCH_NAME develop```. Only pull requests to the ```develop``` branch will be merged. + +Pull requests are **highly appreciated**. + +Solve a problem. Features are great, but even better is cleaning-up and fixing issues in the code that you discover. + + +## Versioning + +This project is maintained by using the [Semantic Versioning Specification (SemVer)](http://semver.org). + + +## Copyright and license + +Copyright 2017 [Caffeina](http://caffeina.com) srl under the [MIT license](LICENSE.md). From 0b43da4953f47052664f0e6cd98fb597c8a06ffd Mon Sep 17 00:00:00 2001 From: db260179 Date: Mon, 12 Nov 2018 21:07:53 +0000 Subject: [PATCH 8/8] Make the /status folder for slave status Create a /status folder so that the master can create a status file once finished This allows the slave to only start once the master is ready. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 179b381..e532190 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ FROM mariadb:10.3 COPY mariadb/init.sh /usr/local/bin/ COPY mariadb/entrypoint.sh /usr/local/bin/master-slave-entrypoint.sh +RUN mkdir -p /status + ENTRYPOINT ["master-slave-entrypoint.sh"] CMD ["mysqld"]