diff --git a/src/models/rewards_eligibility_oracle.py b/src/models/rewards_eligibility_oracle.py index e72ade6..df1132f 100644 --- a/src/models/rewards_eligibility_oracle.py +++ b/src/models/rewards_eligibility_oracle.py @@ -33,10 +33,9 @@ def main(run_date_override: date = None): """ Main entry point for the Rewards Eligibility Oracle. This function: - 1. Sets up Google credentials (if not already set up by scheduler) - 2. Fetches and processes indexer eligibility data - 3. Submits eligible indexers to the blockchain - 4. Sends Slack notifications about the run status + 1. Fetches and processes indexer eligibility data + 2. Submits eligible indexers to the blockchain + 3. Sends Slack notifications about the run status Args: run_date_override: If provided, use this date for the run instead of today. diff --git a/src/models/scheduler.py b/src/models/scheduler.py index d66d387..d67fa23 100644 --- a/src/models/scheduler.py +++ b/src/models/scheduler.py @@ -159,6 +159,9 @@ def initialize(self): try: validate_all_required_env_vars() + # Prepare credentials using inline JSON or file paths + credential_manager.prepare_credentials_for_adc() + # Validate credentials early (Fail Fast) try: credential_manager.get_google_credentials() diff --git a/src/utils/configuration.py b/src/utils/configuration.py index 06cd307..09649f9 100644 --- a/src/utils/configuration.py +++ b/src/utils/configuration.py @@ -367,8 +367,10 @@ def get_google_credentials(self) -> google.auth.credentials.Credentials: return credentials except Exception as e: - error_msg = f"Failed to load Google Cloud credentials: {e}" - logger.error(error_msg) + error_msg = ( + "Failed to load Google Cloud credentials. Check GOOGLE_APPLICATION_CREDENTIALS configuration." + ) + logger.error(f"{error_msg} Error details: {type(e).__name__}") raise ValueError(error_msg) @@ -407,8 +409,14 @@ def _parse_and_validate_credentials_json(self, creds_env: str) -> dict: return creds_data - except Exception as e: - raise ValueError(f"Invalid credentials JSON: {e}") from e + except json.JSONDecodeError: + raise ValueError("Invalid credentials JSON format. Expected valid JSON string.") + except ValueError: + # Re-raise our own validation errors with their specific messages + raise + except Exception: + # Catch-all for unexpected errors - don't leak credential data + raise ValueError("Invalid or incomplete credentials. Check credentials structure.") def _setup_user_credentials_from_dict(self, creds_data: dict) -> None: @@ -444,8 +452,70 @@ def _setup_service_account_credentials_from_dict(self, creds_data: dict) -> None logger.info("Successfully loaded service account credentials from environment variable") # If the credentials creation fails, raise an error - except Exception as e: - raise ValueError(f"Invalid service account credentials: {e}") from e + except Exception: + raise ValueError( + "Invalid service account credentials. Check private_key, client_email, and project_id fields." + ) + + + def prepare_credentials_for_adc(self) -> None: + """ + Prepare Google credentials for Application Default Credentials (ADC). + + Supports both inline JSON and file paths: + - Inline JSON: Writes temp file and updates env var + - File path: Validates existence, logs warning if not found + + This enables google.auth.default() to work with both credential sources while + maintaining official API usage. + + Raises: + ValueError: If inline JSON is invalid or incomplete + """ + creds_env = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS") + + # If not set, log warning and return + if not creds_env: + logger.warning("GOOGLE_APPLICATION_CREDENTIALS not set. Will fall back to ADC.") + return + + # Inline JSON pattern + if creds_env.strip().startswith("{"): + creds_data = None + try: + # Validate JSON structure + creds_data = self._parse_and_validate_credentials_json(creds_env) + + # Write to temp file + temp_path = Path("/tmp/gcp-credentials.json") + with open(temp_path, "w", encoding="utf-8") as f: + json.dump(creds_data, f) + + # Set restrictive permissions + temp_path.chmod(0o600) + + # Update env var + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(temp_path) + + logger.info("Prepared inline JSON credentials for ADC") + + except ValueError: + # Re-raise our own validation errors + raise + + except Exception: + # Catch unexpected errors without leaking credential data + raise ValueError("Failed to prepare inline credentials. Check JSON format and structure.") + + finally: + # Clear data from memory + if creds_data: + creds_data.clear() + + # File path pattern + elif not Path(creds_env).exists(): + logger.warning(f"Credentials file not found: {creds_env}") + logger.warning("Will attempt to use gcloud CLI or other ADC sources") def setup_google_credentials(self) -> None: @@ -477,8 +547,12 @@ def setup_google_credentials(self) -> None: self._setup_service_account_credentials_from_dict(creds_data.copy()) # If the credentials parsing fails, raise an error - except Exception as e: - raise ValueError(f"Error processing inline credentials: {e}") from e + except ValueError: + # Re-raise our own validation errors + raise + except Exception: + # Catch unexpected errors without leaking credential data + raise ValueError("Error processing inline credentials. Check format and required fields.") # Clear the credentials from memory finally: diff --git a/tests/test_configuration.py b/tests/test_configuration.py index f87b65a..c82f863 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -3,6 +3,7 @@ """ import json +import os from pathlib import Path from unittest.mock import MagicMock, patch @@ -592,19 +593,33 @@ def test_setup_service_account_fails_on_sdk_error(self, mock_env, mock_google_au """ GIVEN the Google SDK fails to create credentials from service account info WHEN _setup_service_account_credentials_from_dict is called - THEN it should raise a ValueError. + THEN it should raise a ValueError without leaking credential values. """ # Arrange + # Simulate an SDK error that might contain credentials + error_with_creds = ( + 'SDK Error: {"private_key": "-----BEGIN PRIVATE KEY-----SECRET123", "project_id": "test"}' + ) mock_google_auth["service_account"].Credentials.from_service_account_info.side_effect = Exception( - "SDK Error" + error_with_creds ) manager = CredentialManager() creds_data = json.loads(mock_service_account_json) # Act & Assert - with pytest.raises(ValueError, match="Invalid service account credentials: SDK Error"): + with pytest.raises(ValueError) as exc_info: manager._setup_service_account_credentials_from_dict(creds_data) + error_message = str(exc_info.value) + + # Verify error does not leak actual credential values + assert "SECRET123" not in error_message + assert "BEGIN PRIVATE KEY" not in error_message + assert '{"private_key"' not in error_message + + # Verify it has the generic error message + assert "Invalid service account credentials" in error_message + def test_setup_google_credentials_succeeds_with_authorized_user_json( self, mock_env, mock_google_auth, mock_auth_user_json @@ -814,6 +829,38 @@ def test_get_google_credentials_fails_fast_when_auth_fails(self, mock_env): manager.get_google_credentials() + def test_get_google_credentials_does_not_leak_credentials_in_error(self, mock_env): + """ + GIVEN google.auth.default() raises exception containing credentials + WHEN get_google_credentials() called + THEN Error message does not contain sensitive credential data + """ + # Arrange + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/nonexistent/file.json") + manager = CredentialManager() + + # Simulate an exception that contains credentials in the message + sensitive_error = Exception( + 'Failed to load: {"type": "service_account", "private_key": "SECRET_KEY_123", "project_id": "test"}' + ) + + with patch("google.auth.default", side_effect=sensitive_error): + # Act & Assert + with pytest.raises(ValueError) as exc_info: + manager.get_google_credentials() + + error_message = str(exc_info.value) + + # Verify error message does not contain sensitive data from the original exception + assert "SECRET_KEY_123" not in error_message + assert "private_key" not in error_message + assert "service_account" not in error_message + + # Verify it contains the generic error message + assert "Failed to load Google Cloud credentials" in error_message + assert "Check GOOGLE_APPLICATION_CREDENTIALS configuration" in error_message + + def test_get_google_credentials_works_without_env_var(self, mock_env, mock_google_auth_default): """ GIVEN No GOOGLE_APPLICATION_CREDENTIALS set (gcloud CLI) @@ -832,6 +879,236 @@ def test_get_google_credentials_works_without_env_var(self, mock_env, mock_googl assert credentials is not None +class TestPrepareCredentialsForADC: + """Test prepare_credentials_for_adc() for Kubernetes and Docker compatibility""" + + + def test_prepare_credentials_for_adc_with_inline_json_service_account( + self, mock_env, tmp_path, mock_service_account_json + ): + """ + GIVEN inline service account JSON in GOOGLE_APPLICATION_CREDENTIALS + WHEN prepare_credentials_for_adc() called + THEN writes to temp file, updates env var, sets correct permissions + """ + # Arrange + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", mock_service_account_json) + manager = CredentialManager() + + with patch("src.utils.configuration.Path") as mock_path_cls: + mock_temp_file = MagicMock() + mock_path_cls.return_value = mock_temp_file + + # Mock the open context manager + with patch("builtins.open", create=True) as mock_open: + mock_file = MagicMock() + mock_open.return_value.__enter__.return_value = mock_file + + # Act + manager.prepare_credentials_for_adc() + + # Assert + # Verify temp file was opened for writing + mock_open.assert_called_once() + + # Verify file permissions were set to 0o600 + mock_temp_file.chmod.assert_called_once_with(0o600) + + # Verify env var was updated to point to temp file + assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == str(mock_temp_file) + + + def test_prepare_credentials_for_adc_with_inline_json_authorized_user( + self, mock_env, tmp_path, mock_auth_user_json + ): + """ + GIVEN inline authorized user JSON in GOOGLE_APPLICATION_CREDENTIALS + WHEN prepare_credentials_for_adc() called + THEN writes to temp file and updates env var (supports both credential types) + """ + # Arrange + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", mock_auth_user_json) + manager = CredentialManager() + + with patch("src.utils.configuration.Path") as mock_path_cls: + mock_temp_file = MagicMock() + mock_path_cls.return_value = mock_temp_file + + with patch("builtins.open", create=True) as mock_open: + # Act + manager.prepare_credentials_for_adc() + + # Assert + mock_open.assert_called_once() + mock_temp_file.chmod.assert_called_once_with(0o600) + + + def test_prepare_credentials_for_adc_with_file_path_existing(self, mock_env, tmp_path): + """ + GIVEN GOOGLE_APPLICATION_CREDENTIALS set to existing file path + WHEN prepare_credentials_for_adc() called + THEN env var unchanged, no new temp file created + """ + # Arrange + existing_file = tmp_path / "existing-credentials.json" + existing_file.write_text('{"type": "service_account", "project_id": "test"}') + original_path = str(existing_file) + + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", original_path) + manager = CredentialManager() + + # Act + manager.prepare_credentials_for_adc() + + # Assert - env var should remain unchanged + assert os.environ["GOOGLE_APPLICATION_CREDENTIALS"] == original_path + + + def test_prepare_credentials_for_adc_with_file_path_nonexistent(self, mock_env, caplog): + """ + GIVEN GOOGLE_APPLICATION_CREDENTIALS set to nonexistent file path + WHEN prepare_credentials_for_adc() called + THEN logs warning, no error raised (graceful degradation) + """ + # Arrange + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", "/nonexistent/path/creds.json") + manager = CredentialManager() + + # Act + manager.prepare_credentials_for_adc() + + # Assert + assert "not found" in caplog.text.lower() + + + def test_prepare_credentials_for_adc_with_invalid_json(self, mock_env): + """ + GIVEN GOOGLE_APPLICATION_CREDENTIALS with malformed JSON + WHEN prepare_credentials_for_adc() called + THEN raises ValueError without leaking credentials (Fail Fast) + """ + # Arrange + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", '{"invalid": json}') + manager = CredentialManager() + + # Act & Assert + with pytest.raises(ValueError) as exc_info: + manager.prepare_credentials_for_adc() + + error_message = str(exc_info.value) + + # Verify error message is generic and doesn't leak the malformed JSON + assert "Invalid credentials JSON format" in error_message + assert "json}" not in error_message + + + def test_prepare_credentials_for_adc_with_incomplete_json(self, mock_env): + """ + GIVEN valid JSON but missing required fields + WHEN prepare_credentials_for_adc() called + THEN raises ValueError from validation (Fail Fast) + """ + # Arrange - service account missing required fields + incomplete_json = '{"type": "service_account"}' + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", incomplete_json) + manager = CredentialManager() + + # Act & Assert + with pytest.raises(ValueError, match="Incomplete service_account credentials"): + manager.prepare_credentials_for_adc() + + + def test_prepare_credentials_for_adc_without_env_var(self, mock_env, caplog): + """ + GIVEN GOOGLE_APPLICATION_CREDENTIALS not set + WHEN prepare_credentials_for_adc() called + THEN logs warning, returns gracefully + """ + # Arrange + mock_env.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + manager = CredentialManager() + + # Act + manager.prepare_credentials_for_adc() + + # Assert + assert "not set" in caplog.text + + + def test_prepare_credentials_for_adc_clears_sensitive_data(self, mock_env, mock_service_account_json): + """ + GIVEN inline JSON credentials + WHEN prepare_credentials_for_adc() called + THEN credentials dict is cleared from memory (security) + """ + # Arrange + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", mock_service_account_json) + manager = CredentialManager() + + # Track if clear() was called on the dict + clear_called = False + + # Create a custom dict class that tracks clear() calls + + + class TrackableDict(dict): + + + def clear(self): + nonlocal clear_called + clear_called = True + super().clear() + + # Mock the validation to return our trackable dict + with ( + patch.object(manager, "_parse_and_validate_credentials_json") as mock_parse, + patch("src.utils.configuration.Path"), + patch("builtins.open", create=True), + ): + # Create trackable dict with expected data + tracked_dict = TrackableDict(json.loads(mock_service_account_json)) + mock_parse.return_value = tracked_dict + + # Act + manager.prepare_credentials_for_adc() + + # Assert + assert clear_called, "Credentials data should be cleared from memory" + + + def test_prepare_credentials_for_adc_temp_file_contents_valid( + self, mock_env, tmp_path, mock_service_account_json + ): + """ + GIVEN inline JSON credentials + WHEN prepare_credentials_for_adc() called + THEN temp file contains valid, parseable JSON matching input + """ + # Arrange + mock_env.setenv("GOOGLE_APPLICATION_CREDENTIALS", mock_service_account_json) + manager = CredentialManager() + + # Use tmp_path instead of /tmp for testing + temp_file_path = tmp_path / "gcp-credentials.json" + + # Mock Path to return our test path + with patch("src.utils.configuration.Path") as mock_path_cls: + mock_path_cls.return_value = temp_file_path + + # Act + manager.prepare_credentials_for_adc() + + # Assert - verify written data is valid JSON + assert temp_file_path.exists() + written_content = temp_file_path.read_text() + parsed = json.loads(written_content) + expected = json.loads(mock_service_account_json) + assert parsed == expected + + # Verify permissions were set correctly + assert oct(temp_file_path.stat().st_mode)[-3:] == "600" + + class TestLoadConfig: """Tests for the main load_config function."""