From 4bece8fa297d2c9e75be89cc74d246c18fcc412d Mon Sep 17 00:00:00 2001 From: Filipp Ozinov Date: Sat, 8 Nov 2025 00:38:39 +0400 Subject: [PATCH 1/2] Config settings using env variables --- README.md | 21 +++++++- example_config.yaml | 4 ++ mysql_ch_replicator/config.py | 25 ++++++++++ tests/test_config_env_vars.py | 86 ++++++++++++++++++++++++++++++++ tests/tests_config_env_vars.yaml | 20 ++++++++ 5 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 tests/test_config_env_vars.py create mode 100644 tests/tests_config_env_vars.yaml diff --git a/README.md b/README.md index a077285..9ff9376 100644 --- a/README.md +++ b/README.md @@ -81,10 +81,25 @@ docker run -d \ --config /app/config.yaml run_all ``` +Or with environment variables for credentials: + +```bash +docker run -d \ + -v /path/to/your/config.yaml:/app/config.yaml \ + -v /path/to/your/data:/app/data \ + -e MYSQL_USER=root \ + -e MYSQL_PASSWORD=secret \ + -e CLICKHOUSE_USER=default \ + -e CLICKHOUSE_PASSWORD=secret \ + fippo/mysql-ch-replicator:latest \ + --config /app/config.yaml run_all +``` + Make sure to: 1. Mount your configuration file using the `-v` flag 2. Mount a persistent volume for the data directory 3. Adjust the paths according to your setup +4. Optionally use `-e` flags to override credentials via environment variables ## Usage @@ -187,7 +202,7 @@ __Hint__: _set `initial_replication_threads` to a number of cpu cores to acceler ### Configuration -`mysql_ch_replicator` can be configured through a configuration file. Here is the config example: +`mysql_ch_replicator` can be configured through a configuration file and also by using enviromnent variables to override some of config settings. Here is the config example: ```yaml mysql: @@ -299,6 +314,10 @@ databases: ['my_database_1', 'my_database_2'] tables: ['table_1', 'table_2*'] ``` +**Environment Variables**: MySQL and ClickHouse credentials can be overridden using environment variables for better security in containerized environments: +- MySQL: `MYSQL_HOST`, `MYSQL_PORT`, `MYSQL_USER`, `MYSQL_PASSWORD`, `MYSQL_CHARSET` +- ClickHouse: `CLICKHOUSE_HOST`, `CLICKHOUSE_PORT`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD` + ### Advanced Features #### Migrations & Schema Changes diff --git a/example_config.yaml b/example_config.yaml index a9f3209..1c8a2bc 100644 --- a/example_config.yaml +++ b/example_config.yaml @@ -1,4 +1,8 @@ +# MySQL and ClickHouse credentials can be overridden using environment variables: +# MySQL: MYSQL_HOST, MYSQL_PORT, MYSQL_USER, MYSQL_PASSWORD, MYSQL_CHARSET +# ClickHouse: CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD + mysql: host: 'localhost' port: 8306 diff --git a/mysql_ch_replicator/config.py b/mysql_ch_replicator/config.py index a7ddada..921d833 100644 --- a/mysql_ch_replicator/config.py +++ b/mysql_ch_replicator/config.py @@ -1,3 +1,4 @@ +import os import yaml import fnmatch import zoneinfo @@ -159,6 +160,9 @@ def load(self, settings_file): self.settings_file = settings_file self.mysql = MysqlSettings(**data.pop('mysql')) self.clickhouse = ClickhouseSettings(**data.pop('clickhouse')) + + self._apply_env_overrides() + self.databases = data.pop('databases') self.tables = data.pop('tables', '*') self.exclude_databases = data.pop('exclude_databases', '') @@ -206,6 +210,27 @@ def load(self, settings_file): raise Exception(f'Unsupported config options: {list(data.keys())}') self.validate() + def _apply_env_overrides(self): + if os.getenv('MYSQL_HOST'): + self.mysql.host = os.getenv('MYSQL_HOST') + if os.getenv('MYSQL_PORT'): + self.mysql.port = int(os.getenv('MYSQL_PORT')) + if os.getenv('MYSQL_USER'): + self.mysql.user = os.getenv('MYSQL_USER') + if os.getenv('MYSQL_PASSWORD'): + self.mysql.password = os.getenv('MYSQL_PASSWORD') + if os.getenv('MYSQL_CHARSET'): + self.mysql.charset = os.getenv('MYSQL_CHARSET') + + if os.getenv('CLICKHOUSE_HOST'): + self.clickhouse.host = os.getenv('CLICKHOUSE_HOST') + if os.getenv('CLICKHOUSE_PORT'): + self.clickhouse.port = int(os.getenv('CLICKHOUSE_PORT')) + if os.getenv('CLICKHOUSE_USER'): + self.clickhouse.user = os.getenv('CLICKHOUSE_USER') + if os.getenv('CLICKHOUSE_PASSWORD'): + self.clickhouse.password = os.getenv('CLICKHOUSE_PASSWORD') + @classmethod def is_pattern_matches(cls, substr, pattern): if not pattern or pattern == '*': diff --git a/tests/test_config_env_vars.py b/tests/test_config_env_vars.py new file mode 100644 index 0000000..513a545 --- /dev/null +++ b/tests/test_config_env_vars.py @@ -0,0 +1,86 @@ +import os +from pathlib import Path + +from mysql_ch_replicator.config import Settings + + +def test_env_vars_override_config(): + config_file = Path(__file__).parent / 'tests_config_env_vars.yaml' + + os.environ['MYSQL_HOST'] = 'mysql.env.host' + os.environ['MYSQL_PORT'] = '8306' + os.environ['MYSQL_USER'] = 'env_mysql_user' + os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass' + os.environ['MYSQL_CHARSET'] = 'utf8' + + os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host' + os.environ['CLICKHOUSE_PORT'] = '8323' + os.environ['CLICKHOUSE_USER'] = 'env_ch_user' + os.environ['CLICKHOUSE_PASSWORD'] = 'env_ch_pass' + + settings = Settings() + settings.load(str(config_file)) + + assert settings.mysql.host == 'mysql.env.host' + assert settings.mysql.port == 8306 + assert settings.mysql.user == 'env_mysql_user' + assert settings.mysql.password == 'env_mysql_pass' + assert settings.mysql.charset == 'utf8' + + assert settings.clickhouse.host == 'clickhouse.env.host' + assert settings.clickhouse.port == 8323 + assert settings.clickhouse.user == 'env_ch_user' + assert settings.clickhouse.password == 'env_ch_pass' + + del os.environ['MYSQL_HOST'] + del os.environ['MYSQL_PORT'] + del os.environ['MYSQL_USER'] + del os.environ['MYSQL_PASSWORD'] + del os.environ['MYSQL_CHARSET'] + del os.environ['CLICKHOUSE_HOST'] + del os.environ['CLICKHOUSE_PORT'] + del os.environ['CLICKHOUSE_USER'] + del os.environ['CLICKHOUSE_PASSWORD'] + + +def test_config_without_env_vars(): + config_file = Path(__file__).parent / 'tests_config_env_vars.yaml' + + settings = Settings() + settings.load(str(config_file)) + + assert settings.mysql.host == 'mysql.local' + assert settings.mysql.port == 3306 + assert settings.mysql.user == 'mysql_user' + assert settings.mysql.password == 'mysql_pass' + assert settings.mysql.charset == 'utf8mb4' + + assert settings.clickhouse.host == 'clickhouse.local' + assert settings.clickhouse.port == 9000 + assert settings.clickhouse.user == 'ch_user' + assert settings.clickhouse.password == 'ch_pass' + + +def test_partial_env_vars_override(): + config_file = Path(__file__).parent / 'tests_config_env_vars.yaml' + + os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass' + os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host' + + settings = Settings() + settings.load(str(config_file)) + + assert settings.mysql.host == 'mysql.local' + assert settings.mysql.port == 3306 + assert settings.mysql.user == 'mysql_user' + assert settings.mysql.password == 'env_mysql_pass' + assert settings.mysql.charset == 'utf8mb4' + + assert settings.clickhouse.host == 'clickhouse.env.host' + assert settings.clickhouse.port == 9000 + assert settings.clickhouse.user == 'ch_user' + assert settings.clickhouse.password == 'ch_pass' + + del os.environ['MYSQL_PASSWORD'] + del os.environ['CLICKHOUSE_HOST'] + diff --git a/tests/tests_config_env_vars.yaml b/tests/tests_config_env_vars.yaml new file mode 100644 index 0000000..6d6ed43 --- /dev/null +++ b/tests/tests_config_env_vars.yaml @@ -0,0 +1,20 @@ +mysql: + host: 'mysql.local' + port: 3306 + user: 'mysql_user' + password: 'mysql_pass' + charset: 'utf8mb4' + +clickhouse: + host: 'clickhouse.local' + port: 9000 + user: 'ch_user' + password: 'ch_pass' + +binlog_replicator: + data_dir: '/tmp/binlog' + records_per_file: 100000 + +databases: 'test_db' +log_level: 'info' + From f1b38329943460579ccff51c07b36d4b96091c4c Mon Sep 17 00:00:00 2001 From: Filipp Ozinov Date: Sat, 8 Nov 2025 00:43:34 +0400 Subject: [PATCH 2/2] Allow to skip mysql / clickhouse config sections at all --- mysql_ch_replicator/config.py | 4 +-- tests/test_config_env_vars.py | 36 +++++++++++++++++++++++ tests/tests_config_env_vars_no_creds.yaml | 7 +++++ 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 tests/tests_config_env_vars_no_creds.yaml diff --git a/mysql_ch_replicator/config.py b/mysql_ch_replicator/config.py index 921d833..31746b0 100644 --- a/mysql_ch_replicator/config.py +++ b/mysql_ch_replicator/config.py @@ -158,8 +158,8 @@ def load(self, settings_file): data = yaml.safe_load(data) self.settings_file = settings_file - self.mysql = MysqlSettings(**data.pop('mysql')) - self.clickhouse = ClickhouseSettings(**data.pop('clickhouse')) + self.mysql = MysqlSettings(**data.pop('mysql', {})) + self.clickhouse = ClickhouseSettings(**data.pop('clickhouse', {})) self._apply_env_overrides() diff --git a/tests/test_config_env_vars.py b/tests/test_config_env_vars.py index 513a545..9ef850f 100644 --- a/tests/test_config_env_vars.py +++ b/tests/test_config_env_vars.py @@ -84,3 +84,39 @@ def test_partial_env_vars_override(): del os.environ['MYSQL_PASSWORD'] del os.environ['CLICKHOUSE_HOST'] + +def test_config_without_mysql_clickhouse_sections(): + config_file = Path(__file__).parent / 'tests_config_env_vars_no_creds.yaml' + + os.environ['MYSQL_HOST'] = 'mysql.env.host' + os.environ['MYSQL_PORT'] = '8306' + os.environ['MYSQL_USER'] = 'env_mysql_user' + os.environ['MYSQL_PASSWORD'] = 'env_mysql_pass' + + os.environ['CLICKHOUSE_HOST'] = 'clickhouse.env.host' + os.environ['CLICKHOUSE_PORT'] = '8323' + os.environ['CLICKHOUSE_USER'] = 'env_ch_user' + os.environ['CLICKHOUSE_PASSWORD'] = 'env_ch_pass' + + settings = Settings() + settings.load(str(config_file)) + + assert settings.mysql.host == 'mysql.env.host' + assert settings.mysql.port == 8306 + assert settings.mysql.user == 'env_mysql_user' + assert settings.mysql.password == 'env_mysql_pass' + + assert settings.clickhouse.host == 'clickhouse.env.host' + assert settings.clickhouse.port == 8323 + assert settings.clickhouse.user == 'env_ch_user' + assert settings.clickhouse.password == 'env_ch_pass' + + del os.environ['MYSQL_HOST'] + del os.environ['MYSQL_PORT'] + del os.environ['MYSQL_USER'] + del os.environ['MYSQL_PASSWORD'] + del os.environ['CLICKHOUSE_HOST'] + del os.environ['CLICKHOUSE_PORT'] + del os.environ['CLICKHOUSE_USER'] + del os.environ['CLICKHOUSE_PASSWORD'] + diff --git a/tests/tests_config_env_vars_no_creds.yaml b/tests/tests_config_env_vars_no_creds.yaml new file mode 100644 index 0000000..2b9d469 --- /dev/null +++ b/tests/tests_config_env_vars_no_creds.yaml @@ -0,0 +1,7 @@ +binlog_replicator: + data_dir: '/tmp/binlog' + records_per_file: 100000 + +databases: 'test_db' +log_level: 'info' +