Add sf merlion for time-based and forecast AD

.github/workflows/run-forecast-unit-tests.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-operators-unit-tests.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

THIRD_PARTY_LICENSES.txt

@@ -483,6 +483,11 @@ rrcf
 * Source code: https://github.com/kLabUM/rrcf
 * Project home: https://github.com/kLabUM/rrcf
 
+Merlion
+* Copyright 2021 Salesforce.com Inc
+* License: BSD-3 Clause License
+* Source code: https://github.com/salesforce/Merlion
+* Project Home: https://github.com/salesforce/Merlion
 
 =============================== Licenses ===============================
 ------------------------------------------------------------------------

ads/opctl/operator/lowcode/anomaly/const.py

@@ -21,6 +21,23 @@ class SupportedModels(str, metaclass=ExtendedEnumMeta):
     EE = "ee"
     ISOLATIONFOREST = "isolationforest"
 
+    # point anomaly
+    DAGMM = "dagmm"
+    DEEP_POINT_ANOMALY_DETECTOR = "deep_point_anomaly_detector"
+    LSTM_ED = "lstm_ed"
+    SPECTRAL_RESIDUAL = "spectral_residual"
+    VAE = "vae"
+
+    # forecast_based
+    ARIMA = "arima"
+    ETS = "ets"
+    PROPHET = "prophet"
+    SARIMA = "sarima"
+
+    # changepoint
+    BOCPD = "bocpd"
+
+
 class NonTimeADSupportedModels(str, metaclass=ExtendedEnumMeta):
     """Supported non time-based anomaly detection models."""
 
@@ -29,7 +46,7 @@ class NonTimeADSupportedModels(str, metaclass=ExtendedEnumMeta):
     RandomCutForest = "randomcutforest"
     # TODO : Add DBScan
     # DBScan = "dbscan"
-    
+
 
 class TODSSubModels(str, metaclass=ExtendedEnumMeta):
     """Supported TODS sub models."""
@@ -61,6 +78,54 @@ class TODSSubModels(str, metaclass=ExtendedEnumMeta):
 }
 
 
+class MerlionADModels(str, metaclass=ExtendedEnumMeta):
+    """Supported Merlion AD sub models."""
+
+    # point anomaly
+    DAGMM = "dagmm"
+    DEEP_POINT_ANOMALY_DETECTOR = "deep_point_anomaly_detector"
+    LSTM_ED = "lstm_ed"
+    SPECTRAL_RESIDUAL = "spectral_residual"
+    VAE = "vae"
+
+    # forecast_based
+    ARIMA = "arima"
+    ETS = "ets"
+    PROPHET = "prophet"
+    SARIMA = "sarima"
+
+    # changepoint
+    BOCPD = "bocpd"
+
+
+MERLIONAD_IMPORT_MODEL_MAP = {
+    MerlionADModels.DAGMM: ".dagmm",
+    MerlionADModels.DEEP_POINT_ANOMALY_DETECTOR: ".deep_point_anomaly_detector",
+    MerlionADModels.LSTM_ED: ".lstm_ed",
+    MerlionADModels.SPECTRAL_RESIDUAL: ".spectral_residual",
+    MerlionADModels.VAE: ".vae",
+    MerlionADModels.ARIMA: ".forecast_based.arima",
+    MerlionADModels.ETS: ".forecast_based.ets",
+    MerlionADModels.PROPHET: ".forecast_based.prophet",
+    MerlionADModels.SARIMA: ".forecast_based.sarima",
+    MerlionADModels.BOCPD: ".change_point.bocpd",
+}
+
+
+MERLIONAD_MODEL_MAP = {
+    MerlionADModels.DAGMM: "DAGMM",
+    MerlionADModels.DEEP_POINT_ANOMALY_DETECTOR: "DeepPointAnomalyDetector",
+    MerlionADModels.LSTM_ED: "LSTMED",
+    MerlionADModels.SPECTRAL_RESIDUAL: "SpectralResidual",
+    MerlionADModels.VAE: "VAE",
+    MerlionADModels.ARIMA: "ArimaDetector",
+    MerlionADModels.ETS: "ETSDetector",
+    MerlionADModels.PROPHET: "ProphetDetector",
+    MerlionADModels.SARIMA: "SarimaDetector",
+    MerlionADModels.BOCPD: "BOCPD",
+}
+
+
 class SupportedMetrics(str, metaclass=ExtendedEnumMeta):
     UNSUPERVISED_UNIFY95 = "unsupervised_unify95"
     UNSUPERVISED_UNIFY95_LOG_LOSS = "unsupervised_unify95_log_loss"

ads/opctl/operator/lowcode/anomaly/model/anomaly_merlion.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import importlib
+
+import numpy as np
+import pandas as pd
+from merlion.post_process.threshold import AggregateAlarms
+from merlion.utils import TimeSeries
+
+from ads.common.decorator.runtime_dependency import runtime_dependency
+from ads.opctl.operator.lowcode.anomaly.const import (
+    MERLIONAD_IMPORT_MODEL_MAP,
+    MERLIONAD_MODEL_MAP,
+    OutputColumns,
+    SupportedModels,
+)
+
+from .anomaly_dataset import AnomalyOutput
+from .base_model import AnomalyOperatorBaseModel
+
+
+class AnomalyMerlionOperatorModel(AnomalyOperatorBaseModel):
+    """Class representing Merlion Anomaly Detection operator model."""
+
+    @runtime_dependency(
+        module="merlion",
+        err_msg=(
+            "Please run `pip3 install salesforce-merlion[all]` to "
+            "install the required packages."
+        ),
+    )
+    def _get_config_model(self, model_name):
+        """
+        Returns a dictionary with model names as keys and a list of model config and model object as values.
+
+        Parameters
+        ----------
+        model_name : str
+            model name from the Merlion model list.
+
+        Returns
+        -------
+        dict
+            A dictionary with model names as keys and a list of model config and model object as values.
+        """
+        model_config_map = {}
+        model_module = importlib.import_module(
+            name=MERLIONAD_IMPORT_MODEL_MAP.get(model_name),
+            package="merlion.models.anomaly",
+        )
+        model_config = getattr(
+            model_module, MERLIONAD_MODEL_MAP.get(model_name) + "Config"
+        )
+        model = getattr(model_module, MERLIONAD_MODEL_MAP.get(model_name))
+        model_config_map[model_name] = [model_config, model]
+        return model_config_map
+
+    def _build_model(self) -> AnomalyOutput:
+        """
+        Builds a Merlion anomaly detection model and trains it using the given data.
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        AnomalyOutput
+            An AnomalyOutput object containing the anomaly detection results.
+        """
+        model_kwargs = self.spec.model_kwargs
+        anomaly_output = AnomalyOutput(date_column="index")
+        anomaly_threshold = model_kwargs.get("anomaly_threshold", 95)
+        model_config_map = {}
+        model_config_map = self._get_config_model(self.spec.model)
+
+        date_column = self.spec.datetime_column.name
+
+        anomaly_output = AnomalyOutput(date_column=date_column)
+        # model_objects = defaultdict(list)
+        for target, df in self.datasets.full_data_dict.items():
+            data = df.set_index(date_column)
+            data = TimeSeries.from_pd(data)
+            for model_name, (model_config, model) in model_config_map.items():
+                if self.spec.model == SupportedModels.BOCPD:
+                    model_config = model_config(**self.spec.model_kwargs)
+                else:
+                    model_config = model_config(
+                        **{
+                            **self.spec.model_kwargs,
+                            "threshold": AggregateAlarms(
+                                alm_threshold=model_kwargs.get("alm_threshold")
+                                if model_kwargs.get("alm_threshold")
+                                else None
+                            ),
+                        }
+                    )
+                if hasattr(model_config, "target_seq_index"):
+                    model_config.target_seq_index = df.columns.get_loc(
+                        self.spec.target_column
+                    )
+                model = model(model_config)
+
+                scores = model.train(train_data=data, anomaly_labels=None)
+                scores = scores.to_pd().reset_index()
+                scores["anom_score"] = (
+                    scores["anom_score"] - scores["anom_score"].min()
+                ) / (scores["anom_score"].max() - scores["anom_score"].min())
+
+                try:
+                    y_pred = model.get_anomaly_label(data)
+                    y_pred = (y_pred.to_pd().reset_index()["anom_score"] > 0).astype(
+                        int
+                    )
+                except Exception as e:
+                    y_pred = (
+                        scores["anom_score"]
+                        > np.percentile(
+                            scores["anom_score"],
+                            anomaly_threshold,
+                        )
+                    ).astype(int)
+
+                index_col = df.columns[0]
+
+                anomaly = pd.DataFrame(
+                    {index_col: df[index_col], OutputColumns.ANOMALY_COL: y_pred}
+                ).reset_index(drop=True)
+                score = pd.DataFrame(
+                    {
+                        index_col: df[index_col],
+                        OutputColumns.SCORE_COL: scores["anom_score"],
+                    }
+                ).reset_index(drop=True)
+                # model_objects[model_name].append(model)
+
+                anomaly_output.add_output(target, anomaly, score)
+        return anomaly_output
+
+    def _generate_report(self):
+        """Genreates a report for the model."""
+        import report_creator as rc
+
+        other_sections = [
+            rc.Heading("Selected Models Overview", level=2),
+            rc.Text(
+                "The following tables provide information regarding the chosen model."
+            ),
+        ]
+
+        model_description = rc.Text(
+            "The Merlion anomaly detection model is a full-stack automated machine learning system for anomaly detection."
+        )
+
+        return (
+            model_description,
+            other_sections,
+        )

ads/opctl/operator/lowcode/anomaly/model/autots.py

@@ -5,15 +5,17 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from ads.common.decorator.runtime_dependency import runtime_dependency
+from ads.opctl import logger
 from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
+
+from ..const import SupportedModels
 from .anomaly_dataset import AnomalyOutput
 from .base_model import AnomalyOperatorBaseModel
-from ..const import SupportedModels
-from ads.opctl import logger
 
 
 class AutoTSOperatorModel(AnomalyOperatorBaseModel):
     """Class representing AutoTS Anomaly Detection operator model."""
+
     model_mapping = {
         "isolationforest": "IsolationForest",
         "lof": "LOF",
@@ -22,30 +24,43 @@ class AutoTSOperatorModel(AnomalyOperatorBaseModel):
         "rolling_zscore": "rolling_zscore",
         "mad": "mad",
         "minmax": "minmax",
-        "iqr": "IQR"
+        "iqr": "IQR",
     }
 
     @runtime_dependency(
         module="autots",
         err_msg=(
-                "Please run `pip3 install autots` to "
-                "install the required dependencies for AutoTS."
+            "Please run `pip3 install autots` to "
+            "install the required dependencies for AutoTS."
         ),
     )
     def _build_model(self) -> AnomalyOutput:
         from autots.evaluator.anomaly_detector import AnomalyDetector
 
-        method = SupportedModels.ISOLATIONFOREST if self.spec.model == SupportedModels.AutoTS else self.spec.model
-        model_params = {"method": self.model_mapping[method],
-                        "transform_dict": self.spec.model_kwargs.get("transform_dict", {}),
-                        "output": self.spec.model_kwargs.get("output", "univariate"), "method_params": {}}
+        method = (
+            SupportedModels.ISOLATIONFOREST
+            if self.spec.model == SupportedModels.AutoTS
+            else self.spec.model
+        )
+        model_params = {
+            "method": self.model_mapping[method],
+            "transform_dict": self.spec.model_kwargs.get("transform_dict", {}),
+            "output": self.spec.model_kwargs.get("output", "univariate"),
+            "method_params": {},
+        }
         # Supported methods with contamination param
-        if method in [SupportedModels.ISOLATIONFOREST, SupportedModels.LOF, SupportedModels.EE]:
-            model_params["method_params"][
-                "contamination"] = self.spec.contamination if self.spec.contamination else 0.01
-        else:
-            if self.spec.contamination:
-                raise ValueError(f"The contamination parameter is not supported for the selected model \"{method}\"")
+        if method in [
+            SupportedModels.ISOLATIONFOREST,
+            SupportedModels.LOF,
+            SupportedModels.EE,
+        ]:
+            model_params["method_params"]["contamination"] = (
+                self.spec.contamination if self.spec.contamination else 0.01
+            )
+        elif self.spec.contamination:
+            raise ValueError(
+                f'The contamination parameter is not supported for the selected model "{method}"'
+            )
         logger.info(f"model params: {model_params}")
 
         model = AnomalyDetector(**model_params)

ads/opctl/operator/lowcode/anomaly/model/factory.py

@@ -4,14 +4,16 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from ads.opctl.operator.lowcode.anomaly.utils import select_auto_model
+
+from ..const import NonTimeADSupportedModels, SupportedModels
+from ..operator_config import AnomalyOperatorConfig
 from .anomaly_dataset import AnomalyDatasets
+from .anomaly_merlion import AnomalyMerlionOperatorModel
 from .autots import AutoTSOperatorModel
 from .base_model import AnomalyOperatorBaseModel
 from .isolationforest import IsolationForestOperatorModel
 from .oneclasssvm import OneClassSVMOperatorModel
 from .randomcutforest import RandomCutForestOperatorModel
-from ..const import NonTimeADSupportedModels, SupportedModels
-from ..operator_config import AnomalyOperatorConfig
 
 
 class UnSupportedModelError(Exception):
@@ -48,7 +50,17 @@ class AnomalyOperatorModelFactory:
         SupportedModels.ZSCORE: AutoTSOperatorModel,
         SupportedModels.ROLLING_ZSCORE: AutoTSOperatorModel,
         SupportedModels.EE: AutoTSOperatorModel,
-        SupportedModels.MAD: AutoTSOperatorModel
+        SupportedModels.MAD: AutoTSOperatorModel,
+        SupportedModels.DAGMM: AnomalyMerlionOperatorModel,
+        SupportedModels.DEEP_POINT_ANOMALY_DETECTOR: AnomalyMerlionOperatorModel,
+        SupportedModels.LSTM_ED: AnomalyMerlionOperatorModel,
+        SupportedModels.SPECTRAL_RESIDUAL: AnomalyMerlionOperatorModel,
+        SupportedModels.VAE: AnomalyMerlionOperatorModel,
+        SupportedModels.ARIMA: AnomalyMerlionOperatorModel,
+        SupportedModels.ETS: AnomalyMerlionOperatorModel,
+        SupportedModels.PROPHET: AnomalyMerlionOperatorModel,
+        SupportedModels.SARIMA: AnomalyMerlionOperatorModel,
+        SupportedModels.BOCPD: AnomalyMerlionOperatorModel,
     }
 
     _NonTime_MAP = {