Add data completeness check

Author: MDUYN

investing_algorithm_framework/__init__.py

@@ -10,7 +10,7 @@
     get_ohlcv_data_completeness_chart, get_equity_curve_chart
 from .domain import ApiException, combine_backtests, PositionSize, \
     OrderType, OperationalException, OrderStatus, OrderSide, \
-    TimeUnit, TimeInterval, Order, Portfolio, Backtest, \
+    TimeUnit, TimeInterval, Order, Portfolio, Backtest, DataError, \
     Position, TimeFrame, INDEX_DATETIME, MarketCredential, \
     PortfolioConfiguration, RESOURCE_DIRECTORY, AWS_LAMBDA_LOGGING_CONFIG, \
     Trade, SYMBOLS, RESERVED_BALANCES, APP_MODE, AppMode, DATETIME_FORMAT, \
@@ -191,5 +191,6 @@
     "get_number_of_trades",
     "BacktestRun",
     "load_backtests_from_directory",
-    "save_backtests_to_directory"
+    "save_backtests_to_directory",
+    "DataError"
 ]

investing_algorithm_framework/app/app.py

@@ -5,6 +5,7 @@
 from datetime import datetime, timezone
 from typing import List, Optional, Any, Dict, Tuple
 
+import pandas as pd
 from flask import Flask
 
 from investing_algorithm_framework.app.algorithm import Algorithm
@@ -16,7 +17,7 @@
     SQLALCHEMY_DATABASE_URI, OperationalException, StateHandler, \
     BACKTESTING_START_DATE, BACKTESTING_END_DATE, APP_MODE, MarketCredential, \
     AppMode, BacktestDateRange, DATABASE_DIRECTORY_NAME, DataSource, \
-    BACKTESTING_INITIAL_AMOUNT, SNAPSHOT_INTERVAL, Backtest, \
+    BACKTESTING_INITIAL_AMOUNT, SNAPSHOT_INTERVAL, Backtest, DataError, \
     PortfolioConfiguration, SnapshotInterval, DataType, combine_backtests, \
     PortfolioProvider, OrderExecutor, ImproperlyConfigured, \
     DataProvider, INDEX_DATETIME, tqdm, BacktestPermutationTest, \
@@ -32,6 +33,7 @@
 from .eventloop import EventLoopService
 from .analysis import create_ohlcv_permutation
 
+
 logger = logging.getLogger("investing_algorithm_framework")
 COLOR_RESET = '\033[0m'
 COLOR_GREEN = '\033[92m'
@@ -789,6 +791,69 @@ def get_market_credentials(self) -> List[MarketCredential]:
             .market_credential_service()
         return market_credential_service.get_all()
 
+    def check_data_completeness(
+        self,
+        strategies: List[TradingStrategy],
+        backtest_date_range: BacktestDateRange
+    ) -> None:
+        """
+        Function to check the data completeness for a set of strategies
+        over a given backtest date range. This method checks if all data
+        sources required by the strategies have complete data for the
+        specified date range.
+
+        Args:
+            strategies (List[TradingStrategy]): List of strategy objects
+                to check data completeness for.
+            backtest_date_range (BacktestDateRange): The date range to
+                check data completeness for.
+        Returns:
+            None
+        """
+        data_sources = []
+
+        for strategy in strategies:
+            data_sources.extend(strategy.data_sources)
+
+        self.initialize_data_sources_backtest(
+            data_sources,
+            backtest_date_range,
+            show_progress=True
+        )
+        data_provider_service = self.container.data_provider_service()
+
+        for strategy in strategies:
+
+            for data_source in strategy.data_sources:
+
+                if DataType.OHLCV.equals(data_source.data_type):
+                    df = data_provider_service.get_ohlcv_data(
+                        symbol=data_source.symbol,
+                        start_date=backtest_date_range.start_date,
+                        end_date=backtest_date_range.end_date,
+                        pandas=True
+                    )
+                    df = df.copy()
+                    df['Datetime'] = pd.to_datetime(df['Datetime'])
+                    df = df.sort_values('Datetime').tail(data_source.window_size)
+                    start = df['Datetime'].iloc[0]
+                    end = df['Datetime'].iloc[-1]
+                    freq = pd.to_timedelta(data_source.time_frame.value)
+                    expected = pd.date_range(start, end, freq=freq)
+                    actual = df['Datetime']
+                    missing = expected.difference(actual)
+
+                    # Calculate the percentage completeness
+                    completeness = len(actual) / len(expected) * 100
+
+                    if completeness < 100:
+                        raise DataError(
+                            f"Data completeness for data source "
+                            f"{data_source.data_provider_identifier} "
+                            f"({data_source.symbol}) is {completeness:.2f}% "
+                            f"complete. Missing data points: {len(missing)}"
+                        )
+
     def run_vector_backtests(
         self,
         initial_amount,

investing_algorithm_framework/domain/__init__.py

@@ -15,7 +15,7 @@
 from .data_provider import DataProvider
 from .data_structures import PeekableQueue
 from .decimal_parsing import parse_decimal_to_string, parse_string_to_decimal
-from .exceptions import OperationalException, ApiException, \
+from .exceptions import OperationalException, ApiException, DataError, \
     PermissionDeniedApiException, ImproperlyConfigured, NetworkError
 from .models import OrderStatus, OrderSide, OrderType, TimeInterval, \
     TimeUnit, TimeFrame, PortfolioConfiguration, Portfolio, Position, \
@@ -143,5 +143,6 @@
     "BacktestEvaluationFocus",
     'combine_backtests',
     'PositionSize',
-    'generate_backtest_summary_metrics'
+    'generate_backtest_summary_metrics',
+    'DataError'
 ]

investing_algorithm_framework/domain/exceptions.py

@@ -84,3 +84,20 @@ def to_response(self):
             "status": "error",
             "message": self.error_message
         }
+
+
+class DataError(Exception):
+    """
+    Class DataError: Exception class indicating a problem occurred
+    during data retrieval or processing
+    """
+
+    def __init__(self, message) -> None:
+        super(DataError, self).__init__(message)
+        self.error_message = message
+
+    def to_response(self):
+        return {
+            "status": "error",
+            "message": self.error_message
+        }