Merge pull request #42 from salesforce/fix_torch

Fix the torch and transformers import issue

Author: yangwenzhuo08

logai/algorithms/anomaly_detection_algo/__init__.py

@@ -8,22 +8,30 @@
 from .dbl import DBLDetector
 from .distribution_divergence import DistributionDivergence
 from .ets import ETSDetector
-from .forecast_nn import ForecastBasedLSTM, ForecastBasedCNN, ForecastBasedTransformer
 from .isolation_forest import IsolationForestDetector
 from .local_outlier_factor import LOFDetector
-from .logbert import LogBERT
 from .one_class_svm import OneClassSVMDetector
+from logai.utils.misc import is_torch_available, \
+    is_transformers_available
 
-
-__all__ = [
+_MODULES = [
     "DBLDetector",
     "DistributionDivergence",
     "ETSDetector",
-    "ForecastBasedLSTM",
-    "ForecastBasedCNN",
-    "ForecastBasedTransformer",
     "IsolationForestDetector",
     "LOFDetector",
-    "LogBERT",
-    "OneClassSVMDetector",
+    "OneClassSVMDetector"
 ]
+
+if is_torch_available() and is_transformers_available():
+    from .forecast_nn import ForecastBasedLSTM, ForecastBasedCNN, ForecastBasedTransformer
+    from .logbert import LogBERT
+
+    _MODULES += [
+        "LogBERT",
+        "ForecastBasedLSTM",
+        "ForecastBasedCNN",
+        "ForecastBasedTransformer"
+    ]
+
+__all__ = _MODULES

logai/algorithms/anomaly_detection_algo/forecast_nn.py

@@ -76,11 +76,11 @@ def predict(self, test_data: ForecastNNVectorizedDataset):
 @factory.register("detection", "lstm", LSTMParams)
 class ForecastBasedLSTM(ForcastBasedNeuralAD):
     """Forecasting based lstm model for log anomaly detection.
+
     :param config: A config object containing parameters for LSTM based anomaly detection model.
     """
 
     def __init__(self, config: LSTMParams):
-        
         super().__init__(config)
         self.config = config
         self.model = LSTM(config=self.config)
@@ -89,6 +89,7 @@ def __init__(self, config: LSTMParams):
 @factory.register("detection", "cnn", CNNParams)
 class ForecastBasedCNN(ForcastBasedNeuralAD):
     """Forecasting based cnn model for log anomaly detection.
+
     :param config: A config object containing parameters for CNN based anomaly detection model.
     """
 
@@ -101,11 +102,11 @@ def __init__(self, config: CNNParams):
 @factory.register("detection", "transformer", TransformerParams)
 class ForecastBasedTransformer(ForcastBasedNeuralAD):
     """Forecasting based transformer model for log anomaly detection.
+
     :param config: A config object containing parameters for Transformer based anomaly detection model.
     """
 
     def __init__(self, config: TransformerParams):
-        
         super().__init__(config)
         self.config = config
         self.model = Transformer(config=self.config)

logai/algorithms/factory.py

@@ -5,6 +5,8 @@
 # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 #
 #
+from logai.utils.misc import is_torch_available, \
+    is_transformers_available
 
 
 class AlgorithmFactory:
@@ -18,6 +20,10 @@ class AlgorithmFactory:
         "clustering": {},
         "vectorization": {},
     }
+    _algorithms_with_torch = {
+        "lstm", "cnn", "transformer",
+        "logbert", "forecast_nn"
+    }
 
     def __new__(cls):
         if not hasattr(cls, "instance"):
@@ -58,14 +64,22 @@ def unregister(cls, task, name):
         """
         return cls._algorithms[task].pop(name, None)
 
+    def _check_algorithm(self, task, name):
+        if name in self._algorithms_with_torch:
+            if not is_torch_available() or not is_transformers_available():
+                raise ImportError("Some deep learning packages are missing. "
+                                  "Please install them via `pip install logai[deep-learning]`.")
+        assert name in self._algorithms[task], \
+            f"Unknown algorithm {name}, please choose from {self._algorithms[task].keys()}."
+
     def get_config_class(self, task, name):
         """
         Gets the corresponding configuration class given an algorithm name.
 
         :param task: The task name.
         :param name: The algorithm name.
         """
-        assert name in self._algorithms[task], f"Unknown algorithm {name}."
+        self._check_algorithm(task, name)
         return self._algorithms[task][name][0]
 
     def get_algorithm_class(self, task, name):
@@ -75,7 +89,7 @@ def get_algorithm_class(self, task, name):
         :param task: The task name.
         :param name: The algorithm name.
         """
-        assert name in self._algorithms[task], f"Unknown algorithm {name}."
+        self._check_algorithm(task, name)
         return self._algorithms[task][name][1]
 
     def get_config(self, task, name, config_dict):
@@ -86,7 +100,7 @@ def get_config(self, task, name, config_dict):
         :param name: The algorithm name.
         :param config_dict: The config dictionary.
         """
-        assert name in self._algorithms[task], f"Unknown algorithm {name}."
+        self._check_algorithm(task, name)
         return self._algorithms[task][name][0].from_dict(config_dict)
 
     def get_algorithm(self, task, name, config):
@@ -97,7 +111,7 @@ def get_algorithm(self, task, name, config):
         :param name: The algorithm name.
         :param config: The config instance.
         """
-        assert name in self._algorithms[task], f"Unknown algorithm {name}."
+        self._check_algorithm(task, name)
         config_class, algorithm_class = self._algorithms[task][name]
         if config and config.algo_params:
             assert isinstance(

logai/algorithms/vectorization_algo/__init__.py

@@ -6,19 +6,28 @@
 #
 #
 from .fasttext import FastText
-from .forecast_nn import ForecastNN
-from .logbert import LogBERT
 from .semantic import Semantic
 from .sequential import Sequential
 from .tfidf import TfIdf
 from .word2vec import Word2Vec
+from logai.utils.misc import is_torch_available, \
+    is_transformers_available
 
-__all__ = [
+_MODULES = [
     "FastText",
-    "ForecastNN",
-    "LogBERT",
     "Semantic",
     "Sequential",
     "TfIdf",
-    "Word2Vec",
+    "Word2Vec"
 ]
+
+if is_torch_available() and is_transformers_available():
+    from .forecast_nn import ForecastNN
+    from .logbert import LogBERT
+
+    _MODULES += [
+        "ForecastNN",
+        "LogBERT"
+    ]
+
+__all__ = _MODULES

logai/utils/misc.py

@@ -0,0 +1,83 @@
+#
+# Copyright (c) 2022 salesforce.com, inc.
+# All rights reserved.
+# SPDX-License-Identifier: BSD-3-Clause
+# For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+#
+import sys
+import importlib.util
+from packaging import version
+
+if sys.version_info < (3, 8):
+    import importlib_metadata
+else:
+    import importlib.metadata as importlib_metadata
+
+get_pkg_version = importlib_metadata.version
+
+
+def is_torch_available():
+    """
+    Checks if pytorch is available.
+    """
+    if importlib.util.find_spec("torch") is not None:
+        _torch_version = importlib_metadata.version("torch")
+        if version.parse(_torch_version) < version.parse("1.6"):
+            raise EnvironmentError(f"Torch found but with version {_torch_version}. " f"The minimum version is 1.6")
+        return True
+    else:
+        return False
+
+
+def is_tf_available():
+    """
+    Checks if tensorflow 2.0 is available.
+    """
+    candidates = (
+        "tensorflow",
+        "tensorflow-cpu",
+        "tensorflow-gpu",
+        "tf-nightly",
+        "tf-nightly-cpu",
+        "tf-nightly-gpu",
+        "intel-tensorflow",
+        "intel-tensorflow-avx512",
+        "tensorflow-rocm",
+        "tensorflow-macos",
+    )
+    _tf_version = None
+    for pkg in candidates:
+        try:
+            _tf_version = importlib_metadata.version(pkg)
+            break
+        except importlib_metadata.PackageNotFoundError:
+            pass
+    if _tf_version is not None:
+        if version.parse(_tf_version) < version.parse("2"):
+            raise EnvironmentError(f"Tensorflow found but with version {_tf_version}. " f"The minimum version is 2.0")
+        return True
+    else:
+        return False
+
+
+def is_transformers_available():
+    """
+    Checks if the `transformers` library is installed.
+    """
+    if importlib.util.find_spec("transformers") is not None:
+        _version = importlib_metadata.version("transformers")
+        if version.parse(_version) < version.parse("4.0"):
+            raise EnvironmentError(f"Transformers found but with version {_version}. " f"The minimum version is 4.0")
+        return True
+    else:
+        return False
+
+
+def is_nltk_available():
+    """
+        Checks if the `nltk` library is installed.
+        """
+    if importlib.util.find_spec("nltk") is not None:
+        return True
+    else:
+        return False