Skip to content

Evidently

zenml.integrations.evidently special

Initialization of the Evidently integration.

The Evidently integration provides a way to monitor your models in production. It includes a way to detect data drift and different kinds of model performance issues.

The results of Evidently calculations can either be exported as an interactive dashboard (visualized as an html file or in your Jupyter notebook), or as a JSON file.

EvidentlyIntegration (Integration)

Evidently integration for ZenML.

Source code in zenml/integrations/evidently/__init__.py
class EvidentlyIntegration(Integration):
    """[Evidently](https://github.com/evidentlyai/evidently) integration for ZenML."""

    NAME = EVIDENTLY
    REQUIREMENTS = [
        "evidently>=0.4.16,<=0.4.22",
        "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
    ]
    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity"]

    @classmethod
    def flavors(cls) -> List[Type[Flavor]]:
        """Declare the stack component flavors for the Great Expectations integration.

        Returns:
            List of stack component flavors for this integration.
        """
        from zenml.integrations.evidently.flavors import (
            EvidentlyDataValidatorFlavor,
        )

        return [EvidentlyDataValidatorFlavor]

flavors() classmethod

Declare the stack component flavors for the Great Expectations integration.

Returns:

Type Description
List[Type[zenml.stack.flavor.Flavor]]

List of stack component flavors for this integration.

Source code in zenml/integrations/evidently/__init__.py
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
    """Declare the stack component flavors for the Great Expectations integration.

    Returns:
        List of stack component flavors for this integration.
    """
    from zenml.integrations.evidently.flavors import (
        EvidentlyDataValidatorFlavor,
    )

    return [EvidentlyDataValidatorFlavor]

column_mapping

ZenML representation of an Evidently column mapping.

EvidentlyColumnMapping (BaseModel)

Column mapping configuration for Evidently.

This class is a 1-to-1 serializable analogue of Evidently's ColumnMapping data type that can be used as a step configuration field (see https://docs.evidentlyai.com/user-guide/input-data/column-mapping).

Attributes:

Name Type Description
target Optional[str]

target column

prediction Union[str, Sequence[str]]

target column

datetime Optional[str]

datetime column

id Optional[str]

id column

numerical_features Optional[List[str]]

numerical features

categorical_features Optional[List[str]]

categorical features

datetime_features Optional[List[str]]

datetime features

target_names Optional[List[str]]

target column names

task Optional[str]

model task

pos_label Union[str, int]

positive label

text_features Optional[List[str]]

text features

Source code in zenml/integrations/evidently/column_mapping.py
class EvidentlyColumnMapping(BaseModel):
    """Column mapping configuration for Evidently.

    This class is a 1-to-1 serializable analogue of Evidently's
    ColumnMapping data type that can be used as a step configuration field
    (see https://docs.evidentlyai.com/user-guide/input-data/column-mapping).

    Attributes:
        target: target column
        prediction: target column
        datetime: datetime column
        id: id column
        numerical_features: numerical features
        categorical_features: categorical features
        datetime_features: datetime features
        target_names: target column names
        task: model task
        pos_label: positive label
        text_features: text features
    """

    target: Optional[str] = None
    prediction: Optional[Union[str, Sequence[str]]] = Field(
        default="prediction", union_mode="left_to_right"
    )
    datetime: Optional[str] = None
    id: Optional[str] = None
    numerical_features: Optional[List[str]] = None
    categorical_features: Optional[List[str]] = None
    datetime_features: Optional[List[str]] = None
    target_names: Optional[List[str]] = None
    task: Optional[str] = None
    pos_label: Optional[Union[str, int]] = Field(
        default=1, union_mode="left_to_right"
    )
    text_features: Optional[List[str]] = None

    model_config = ConfigDict(
        validate_assignment=True,
    )

    def to_evidently_column_mapping(self) -> ColumnMapping:
        """Convert this Pydantic object to an Evidently ColumnMapping object.

        Returns:
            An Evidently column mapping converted from this Pydantic object.
        """
        column_mapping = ColumnMapping()

        # preserve the Evidently defaults where possible
        column_mapping.target = self.target or column_mapping.target
        column_mapping.prediction = (
            self.prediction or column_mapping.prediction
        )
        column_mapping.datetime = self.datetime or column_mapping.datetime
        column_mapping.id = self.id or column_mapping.id
        column_mapping.numerical_features = (
            self.numerical_features or column_mapping.numerical_features
        )
        column_mapping.datetime_features = (
            self.datetime_features or column_mapping.datetime_features
        )
        column_mapping.target_names = (
            self.target_names or column_mapping.target_names
        )
        column_mapping.task = self.task or column_mapping.task
        column_mapping.pos_label = self.pos_label or column_mapping.pos_label
        column_mapping.text_features = (
            self.text_features or column_mapping.text_features
        )

        return column_mapping
to_evidently_column_mapping(self)

Convert this Pydantic object to an Evidently ColumnMapping object.

Returns:

Type Description
evidently.ColumnMapping

An Evidently column mapping converted from this Pydantic object.

Source code in zenml/integrations/evidently/column_mapping.py
def to_evidently_column_mapping(self) -> ColumnMapping:
    """Convert this Pydantic object to an Evidently ColumnMapping object.

    Returns:
        An Evidently column mapping converted from this Pydantic object.
    """
    column_mapping = ColumnMapping()

    # preserve the Evidently defaults where possible
    column_mapping.target = self.target or column_mapping.target
    column_mapping.prediction = (
        self.prediction or column_mapping.prediction
    )
    column_mapping.datetime = self.datetime or column_mapping.datetime
    column_mapping.id = self.id or column_mapping.id
    column_mapping.numerical_features = (
        self.numerical_features or column_mapping.numerical_features
    )
    column_mapping.datetime_features = (
        self.datetime_features or column_mapping.datetime_features
    )
    column_mapping.target_names = (
        self.target_names or column_mapping.target_names
    )
    column_mapping.task = self.task or column_mapping.task
    column_mapping.pos_label = self.pos_label or column_mapping.pos_label
    column_mapping.text_features = (
        self.text_features or column_mapping.text_features
    )

    return column_mapping

data_validators special

Initialization of the Evidently data validator for ZenML.

evidently_data_validator

Implementation of the Evidently data validator.

EvidentlyDataValidator (BaseDataValidator)

Evidently data validator stack component.

Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
class EvidentlyDataValidator(BaseDataValidator):
    """Evidently data validator stack component."""

    NAME: ClassVar[str] = "Evidently"
    FLAVOR: ClassVar[Type[BaseDataValidatorFlavor]] = (
        EvidentlyDataValidatorFlavor
    )

    @classmethod
    def _unpack_options(
        cls, option_list: Sequence[Tuple[str, Dict[str, Any]]]
    ) -> Sequence[Any]:
        """Unpack Evidently options.

        Implements de-serialization for [Evidently options](https://docs.evidentlyai.com/user-guide/customization)
        that can be passed as constructor arguments when creating Profile and
        Dashboard objects. The convention used is that each item in the list
        consists of two elements:

        * a string containing the full class path of a `dataclass` based
        class with Evidently options
        * a dictionary with kwargs used as parameters for the option instance

        For example,

        ```python
            options = [
                (
                    "evidently.options.ColorOptions",{
                        "primary_color": "#5a86ad",
                        "fill_color": "#fff4f2",
                        "zero_line_color": "#016795",
                        "current_data_color": "#c292a1",
                        "reference_data_color": "#017b92",
                    }
                ),
            ]
        ```

        This is the same as saying:

        ```python
        from evidently.options import ColorOptions

        color_scheme = ColorOptions()
        color_scheme.primary_color = "#5a86ad"
        color_scheme.fill_color = "#fff4f2"
        color_scheme.zero_line_color = "#016795"
        color_scheme.current_data_color = "#c292a1"
        color_scheme.reference_data_color = "#017b92"
        ```

        Args:
            option_list: list of packed Evidently options

        Returns:
            A list of unpacked Evidently options

        Raises:
            ValueError: if one of the passed Evidently class paths cannot be
                resolved to an actual class.
        """
        options = []
        for option_clspath, option_args in option_list:
            try:
                option_cls = source_utils.load(option_clspath)
            except AttributeError:
                raise ValueError(
                    f"Could not map the `{option_clspath}` Evidently option "
                    f"class path to a valid class."
                )
            option = option_cls(**option_args)
            options.append(option)

        return options

    @staticmethod
    def _set_nltk_data_path() -> None:
        """Set the NLTK data path to the current working directory.

        This is necessary because the default download directory is not writable
        in some Docker containers.

        Raises:
            ImportError: if NLTK is not installed.
        """
        try:
            from nltk.data import (  # type: ignore[import-untyped]
                path as nltk_path,
            )
        except ImportError:
            raise ImportError(
                "NLTK is not installed. Please install NLTK to use "
                "Evidently text metrics and tests."
            )

        # Configure NLTK to use the current working directory to download and
        # lookup data. This is necessary because the default download directory
        # is not writable in some Docker containers.
        os.environ["NLTK_DATA"] = os.getcwd()
        nltk_path.append(os.getcwd())

    @staticmethod
    def _download_nltk_data() -> None:
        """Download NLTK data for text metrics and tests.

        Raises:
            ImportError: if NLTK is not installed.
        """
        try:
            import nltk  # type: ignore[import-untyped]
        except ImportError:
            raise ImportError(
                "NLTK is not installed. Please install NLTK to use "
                "Evidently text metrics and tests."
            )
        # Download NLTK data. We need this later on for the Evidently text report.
        nltk.download("words", download_dir=os.getcwd())
        nltk.download("wordnet", download_dir=os.getcwd())
        nltk.download("omw-1.4", download_dir=os.getcwd())

    def data_profiling(
        self,
        dataset: pd.DataFrame,
        comparison_dataset: Optional[pd.DataFrame] = None,
        profile_list: Optional[Sequence[EvidentlyMetricConfig]] = None,
        column_mapping: Optional[ColumnMapping] = None,
        report_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
        download_nltk_data: bool = False,
        **kwargs: Any,
    ) -> Report:
        """Analyze a dataset and generate a data report with Evidently.

        The method takes in an optional list of Evidently options to be passed
        to the report constructor (`report_options`). Each element in the list must be
        composed of two items: the first is a full class path of an Evidently
        option `dataclass`, the second is a dictionary of kwargs with the actual
        option parameters, e.g.:

        ```python
        options = [
            (
                "evidently.options.ColorOptions",{
                    "primary_color": "#5a86ad",
                    "fill_color": "#fff4f2",
                    "zero_line_color": "#016795",
                    "current_data_color": "#c292a1",
                    "reference_data_color": "#017b92",
                }
            ),
        ]
        ```

        Args:
            dataset: Target dataset to be profiled. When a comparison dataset
                is provided, this dataset is considered the reference dataset.
            comparison_dataset: Optional dataset to be used for data profiles
                that require a current dataset for comparison (e.g data drift
                profiles).
            profile_list: List of Evidently metric configurations to
                be included in the report. If not provided, all available
                metric presets will be included.
            column_mapping: Properties of the DataFrame columns used
            report_options: List of Evidently options to be passed to the
                report constructor.
            download_nltk_data: Whether to download NLTK data for text metrics.
                Defaults to False.
            **kwargs: Extra keyword arguments (unused).

        Returns:
            The Evidently Report as JSON object and as HTML.
        """
        self._set_nltk_data_path()
        if download_nltk_data:
            self._download_nltk_data()

        profile_list = profile_list or EvidentlyMetricConfig.default_metrics()
        metrics = [metric.to_evidently_metric() for metric in profile_list]

        unpacked_report_options = self._unpack_options(report_options)

        report = Report(metrics=metrics, options=unpacked_report_options)

        report.run(
            reference_data=dataset,
            current_data=comparison_dataset,
            column_mapping=column_mapping,
        )

        return report

    def data_validation(
        self,
        dataset: Any,
        comparison_dataset: Optional[Any] = None,
        check_list: Optional[Sequence[EvidentlyTestConfig]] = None,
        test_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
        column_mapping: Optional[ColumnMapping] = None,
        download_nltk_data: bool = False,
        **kwargs: Any,
    ) -> TestSuite:
        """Validate a dataset with Evidently.

        Args:
            dataset: Target dataset to be validated.
            comparison_dataset: Optional dataset to be used for data validation
                that require a baseline for comparison (e.g data drift
                validation).
            check_list: List of Evidently test configurations to be
                included in the test suite. If not provided, all available
                test presets will be included.
            test_options: List of Evidently options to be passed to the
                test suite constructor.
            column_mapping: Properties of the DataFrame columns used
            download_nltk_data: Whether to download NLTK data for text tests.
                Defaults to False.
            **kwargs: Extra keyword arguments (unused).

        Returns:
            The Evidently Test Suite as JSON object and as HTML.
        """
        if download_nltk_data:
            self._download_nltk_data()

        check_list = check_list or EvidentlyTestConfig.default_tests()
        tests = [test.to_evidently_test() for test in check_list]

        unpacked_test_options = self._unpack_options(test_options)

        test_suite = TestSuite(tests=tests, options=unpacked_test_options)
        test_suite.run(
            reference_data=dataset,
            current_data=comparison_dataset,
            column_mapping=column_mapping,
        )

        return test_suite
FLAVOR (BaseDataValidatorFlavor)

Evidently data validator flavor.

Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
class EvidentlyDataValidatorFlavor(BaseDataValidatorFlavor):
    """Evidently data validator flavor."""

    @property
    def name(self) -> str:
        """Name of the flavor.

        Returns:
            The name of the flavor.
        """
        return EVIDENTLY_DATA_VALIDATOR_FLAVOR

    @property
    def docs_url(self) -> Optional[str]:
        """A url to point at docs explaining this flavor.

        Returns:
            A flavor docs url.
        """
        return self.generate_default_docs_url()

    @property
    def sdk_docs_url(self) -> Optional[str]:
        """A url to point at SDK docs explaining this flavor.

        Returns:
            A flavor SDK docs url.
        """
        return self.generate_default_sdk_docs_url()

    @property
    def logo_url(self) -> str:
        """A url to represent the flavor in the dashboard.

        Returns:
            The flavor logo.
        """
        return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/data_validator/evidently.png"

    @property
    def implementation_class(self) -> Type["EvidentlyDataValidator"]:
        """Implementation class.

        Returns:
            The implementation class.
        """
        from zenml.integrations.evidently.data_validators import (
            EvidentlyDataValidator,
        )

        return EvidentlyDataValidator
docs_url: Optional[str] property readonly

A url to point at docs explaining this flavor.

Returns:

Type Description
Optional[str]

A flavor docs url.

implementation_class: Type[EvidentlyDataValidator] property readonly

Implementation class.

Returns:

Type Description
Type[EvidentlyDataValidator]

The implementation class.

logo_url: str property readonly

A url to represent the flavor in the dashboard.

Returns:

Type Description
str

The flavor logo.

name: str property readonly

Name of the flavor.

Returns:

Type Description
str

The name of the flavor.

sdk_docs_url: Optional[str] property readonly

A url to point at SDK docs explaining this flavor.

Returns:

Type Description
Optional[str]

A flavor SDK docs url.

data_profiling(self, dataset, comparison_dataset=None, profile_list=None, column_mapping=None, report_options=[], download_nltk_data=False, **kwargs)

Analyze a dataset and generate a data report with Evidently.

The method takes in an optional list of Evidently options to be passed to the report constructor (report_options). Each element in the list must be composed of two items: the first is a full class path of an Evidently option dataclass, the second is a dictionary of kwargs with the actual option parameters, e.g.:

options = [
    (
        "evidently.options.ColorOptions",{
            "primary_color": "#5a86ad",
            "fill_color": "#fff4f2",
            "zero_line_color": "#016795",
            "current_data_color": "#c292a1",
            "reference_data_color": "#017b92",
        }
    ),
]

Parameters:

Name Type Description Default
dataset DataFrame

Target dataset to be profiled. When a comparison dataset is provided, this dataset is considered the reference dataset.

required
comparison_dataset Optional[pandas.core.frame.DataFrame]

Optional dataset to be used for data profiles that require a current dataset for comparison (e.g data drift profiles).

None
profile_list Optional[Sequence[zenml.integrations.evidently.metrics.EvidentlyMetricConfig]]

List of Evidently metric configurations to be included in the report. If not provided, all available metric presets will be included.

None
column_mapping Optional[evidently.pipeline.column_mapping.ColumnMapping]

Properties of the DataFrame columns used

None
report_options Sequence[Tuple[str, Dict[str, Any]]]

List of Evidently options to be passed to the report constructor.

[]
download_nltk_data bool

Whether to download NLTK data for text metrics. Defaults to False.

False
**kwargs Any

Extra keyword arguments (unused).

{}

Returns:

Type Description
evidently.report.Report

The Evidently Report as JSON object and as HTML.

Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
def data_profiling(
    self,
    dataset: pd.DataFrame,
    comparison_dataset: Optional[pd.DataFrame] = None,
    profile_list: Optional[Sequence[EvidentlyMetricConfig]] = None,
    column_mapping: Optional[ColumnMapping] = None,
    report_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
    download_nltk_data: bool = False,
    **kwargs: Any,
) -> Report:
    """Analyze a dataset and generate a data report with Evidently.

    The method takes in an optional list of Evidently options to be passed
    to the report constructor (`report_options`). Each element in the list must be
    composed of two items: the first is a full class path of an Evidently
    option `dataclass`, the second is a dictionary of kwargs with the actual
    option parameters, e.g.:

    ```python
    options = [
        (
            "evidently.options.ColorOptions",{
                "primary_color": "#5a86ad",
                "fill_color": "#fff4f2",
                "zero_line_color": "#016795",
                "current_data_color": "#c292a1",
                "reference_data_color": "#017b92",
            }
        ),
    ]
    ```

    Args:
        dataset: Target dataset to be profiled. When a comparison dataset
            is provided, this dataset is considered the reference dataset.
        comparison_dataset: Optional dataset to be used for data profiles
            that require a current dataset for comparison (e.g data drift
            profiles).
        profile_list: List of Evidently metric configurations to
            be included in the report. If not provided, all available
            metric presets will be included.
        column_mapping: Properties of the DataFrame columns used
        report_options: List of Evidently options to be passed to the
            report constructor.
        download_nltk_data: Whether to download NLTK data for text metrics.
            Defaults to False.
        **kwargs: Extra keyword arguments (unused).

    Returns:
        The Evidently Report as JSON object and as HTML.
    """
    self._set_nltk_data_path()
    if download_nltk_data:
        self._download_nltk_data()

    profile_list = profile_list or EvidentlyMetricConfig.default_metrics()
    metrics = [metric.to_evidently_metric() for metric in profile_list]

    unpacked_report_options = self._unpack_options(report_options)

    report = Report(metrics=metrics, options=unpacked_report_options)

    report.run(
        reference_data=dataset,
        current_data=comparison_dataset,
        column_mapping=column_mapping,
    )

    return report
data_validation(self, dataset, comparison_dataset=None, check_list=None, test_options=[], column_mapping=None, download_nltk_data=False, **kwargs)

Validate a dataset with Evidently.

Parameters:

Name Type Description Default
dataset Any

Target dataset to be validated.

required
comparison_dataset Optional[Any]

Optional dataset to be used for data validation that require a baseline for comparison (e.g data drift validation).

None
check_list Optional[Sequence[zenml.integrations.evidently.tests.EvidentlyTestConfig]]

List of Evidently test configurations to be included in the test suite. If not provided, all available test presets will be included.

None
test_options Sequence[Tuple[str, Dict[str, Any]]]

List of Evidently options to be passed to the test suite constructor.

[]
column_mapping Optional[evidently.pipeline.column_mapping.ColumnMapping]

Properties of the DataFrame columns used

None
download_nltk_data bool

Whether to download NLTK data for text tests. Defaults to False.

False
**kwargs Any

Extra keyword arguments (unused).

{}

Returns:

Type Description
evidently.test_suite.TestSuite

The Evidently Test Suite as JSON object and as HTML.

Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
def data_validation(
    self,
    dataset: Any,
    comparison_dataset: Optional[Any] = None,
    check_list: Optional[Sequence[EvidentlyTestConfig]] = None,
    test_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
    column_mapping: Optional[ColumnMapping] = None,
    download_nltk_data: bool = False,
    **kwargs: Any,
) -> TestSuite:
    """Validate a dataset with Evidently.

    Args:
        dataset: Target dataset to be validated.
        comparison_dataset: Optional dataset to be used for data validation
            that require a baseline for comparison (e.g data drift
            validation).
        check_list: List of Evidently test configurations to be
            included in the test suite. If not provided, all available
            test presets will be included.
        test_options: List of Evidently options to be passed to the
            test suite constructor.
        column_mapping: Properties of the DataFrame columns used
        download_nltk_data: Whether to download NLTK data for text tests.
            Defaults to False.
        **kwargs: Extra keyword arguments (unused).

    Returns:
        The Evidently Test Suite as JSON object and as HTML.
    """
    if download_nltk_data:
        self._download_nltk_data()

    check_list = check_list or EvidentlyTestConfig.default_tests()
    tests = [test.to_evidently_test() for test in check_list]

    unpacked_test_options = self._unpack_options(test_options)

    test_suite = TestSuite(tests=tests, options=unpacked_test_options)
    test_suite.run(
        reference_data=dataset,
        current_data=comparison_dataset,
        column_mapping=column_mapping,
    )

    return test_suite

flavors special

Evidently integration flavors.

evidently_data_validator_flavor

Evidently data validator flavor.

EvidentlyDataValidatorFlavor (BaseDataValidatorFlavor)

Evidently data validator flavor.

Source code in zenml/integrations/evidently/flavors/evidently_data_validator_flavor.py
class EvidentlyDataValidatorFlavor(BaseDataValidatorFlavor):
    """Evidently data validator flavor."""

    @property
    def name(self) -> str:
        """Name of the flavor.

        Returns:
            The name of the flavor.
        """
        return EVIDENTLY_DATA_VALIDATOR_FLAVOR

    @property
    def docs_url(self) -> Optional[str]:
        """A url to point at docs explaining this flavor.

        Returns:
            A flavor docs url.
        """
        return self.generate_default_docs_url()

    @property
    def sdk_docs_url(self) -> Optional[str]:
        """A url to point at SDK docs explaining this flavor.

        Returns:
            A flavor SDK docs url.
        """
        return self.generate_default_sdk_docs_url()

    @property
    def logo_url(self) -> str:
        """A url to represent the flavor in the dashboard.

        Returns:
            The flavor logo.
        """
        return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/data_validator/evidently.png"

    @property
    def implementation_class(self) -> Type["EvidentlyDataValidator"]:
        """Implementation class.

        Returns:
            The implementation class.
        """
        from zenml.integrations.evidently.data_validators import (
            EvidentlyDataValidator,
        )

        return EvidentlyDataValidator
docs_url: Optional[str] property readonly

A url to point at docs explaining this flavor.

Returns:

Type Description
Optional[str]

A flavor docs url.

implementation_class: Type[EvidentlyDataValidator] property readonly

Implementation class.

Returns:

Type Description
Type[EvidentlyDataValidator]

The implementation class.

logo_url: str property readonly

A url to represent the flavor in the dashboard.

Returns:

Type Description
str

The flavor logo.

name: str property readonly

Name of the flavor.

Returns:

Type Description
str

The name of the flavor.

sdk_docs_url: Optional[str] property readonly

A url to point at SDK docs explaining this flavor.

Returns:

Type Description
Optional[str]

A flavor SDK docs url.

metrics

ZenML declarative representation of Evidently Metrics.

EvidentlyMetricConfig (BaseModel)

Declarative Evidently Metric configuration.

This is a declarative representation of the configuration that goes into an Evidently Metric, MetricPreset or Metric generator instance. We need this to be able to store the configuration as part of a ZenML step parameter and later instantiate the Evidently Metric from it.

This representation covers all 3 possible ways of configuring an Evidently Metric or Metric-like object that can later be used in an Evidently Report:

  1. A Metric (derived from the Metric class).
  2. A MetricPreset (derived from the MetricPreset class).
  3. A column Metric generator (derived from the BaseGenerator class).

Ideally, it should be possible to just pass a Metric or Metric-like object to this class and have it automatically derive the configuration used to instantiate it. Unfortunately, this is not possible because the Evidently Metric classes are not designed in a way that allows us to extract the constructor parameters from them in a generic way.

Attributes:

Name Type Description
class_path str

The full class path of the Evidently Metric class.

parameters Dict[str, Any]

The parameters of the Evidently Metric.

is_generator bool

Whether this is an Evidently column Metric generator.

columns Union[str, List[str]]

The columns that the Evidently column Metric generator is applied to. Only used if generator is True.

skip_id_column bool

Whether to skip the ID column when applying the Evidently Metric generator. Only used if generator is True.

Source code in zenml/integrations/evidently/metrics.py
class EvidentlyMetricConfig(BaseModel):
    """Declarative Evidently Metric configuration.

    This is a declarative representation of the configuration that goes into an
    Evidently Metric, MetricPreset or Metric generator instance. We need this to
    be able to store the configuration as part of a ZenML step parameter and
    later instantiate the Evidently Metric from it.

    This representation covers all 3 possible ways of configuring an Evidently
    Metric or Metric-like object that can later be used in an Evidently Report:

    1. A Metric (derived from the Metric class).
    2. A MetricPreset (derived from the MetricPreset class).
    3. A column Metric generator (derived from the BaseGenerator class).

    Ideally, it should be possible to just pass a Metric or Metric-like
    object to this class and have it automatically derive the configuration used
    to instantiate it. Unfortunately, this is not possible because the Evidently
    Metric classes are not designed in a way that allows us to extract the
    constructor parameters from them in a generic way.

    Attributes:
        class_path: The full class path of the Evidently Metric class.
        parameters: The parameters of the Evidently Metric.
        is_generator: Whether this is an Evidently column Metric generator.
        columns: The columns that the Evidently column Metric generator is
            applied to. Only used if `generator` is True.
        skip_id_column: Whether to skip the ID column when applying the
            Evidently Metric generator. Only used if `generator` is True.
    """

    class_path: str
    parameters: Dict[str, Any] = Field(default_factory=dict)
    is_generator: bool = False
    columns: Optional[Union[str, List[str]]] = Field(
        default=None, union_mode="left_to_right"
    )
    skip_id_column: bool = False

    @staticmethod
    def get_metric_class(metric_name: str) -> Union[Metric, MetricPreset]:
        """Get the Evidently metric or metric preset class from a string.

        Args:
            metric_name: The metric or metric preset class or full class
                path.

        Returns:
            The Evidently metric or metric preset class.

        Raises:
            ValueError: If the name cannot be converted into a valid Evidently
                metric or metric preset class.
        """
        # First, try to interpret the metric name as a full class path.
        if "." in metric_name:
            try:
                metric_class = source_utils.load(metric_name)
            except (ImportError, AttributeError) as e:
                raise ValueError(
                    f"Could not import Evidently Metric or MetricPreset "
                    f"`{metric_name}`: {str(e)}"
                )

        else:
            # Next, try to interpret the metric as a Metric or MetricPreset
            # class name
            if hasattr(metrics, metric_name):
                metric_class = getattr(metrics, metric_name)
            elif hasattr(metric_preset, metric_name):
                metric_class = getattr(metric_preset, metric_name)
            else:
                raise ValueError(
                    f"Could not import Evidently Metric or MetricPreset "
                    f"`{metric_name}`"
                )

        if not issubclass(metric_class, (Metric, MetricPreset)):
            raise ValueError(
                f"Class `{metric_name}` is not a valid Evidently "
                f"Metric or MetricPreset."
            )

        return metric_class

    @classmethod
    def metric_generator(
        cls,
        metric: Union[Type[Metric], str],
        columns: Optional[Union[str, List[str]]] = None,
        skip_id_column: bool = False,
        **parameters: Any,
    ) -> "EvidentlyMetricConfig":
        """Create a declarative configuration for an Evidently column Metric generator.

        Call this method to get a declarative representation for the
        configuration of an Evidently column Metric generator.

        The `columns`, `skip_id_column` and `parameters` arguments will be
        passed to the Evidently `generate_column_metrics` function:

        - if `columns` is a list, it is interpreted as a list of column names.
        - if `columns` is a string, it can be one of values:
            - "all" - use all columns, including target/prediction columns
            - "num" - for numeric features
            - "cat" - for category features
            - "text" - for text features
            - "features" - for all features, not target/prediction columns.
        - a None value is the same as "all".

        Some examples
        -------------

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyMetric

        # Configure an Evidently Metric generator using a Metric class name
        # and pass additional parameters
        config = EvidentlyMetric.metric_generator(
            "ColumnQuantileMetric", columns="num", quantile=0.5
        )
        ```

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyMetric

        # Configure an Evidently Metric generator using a full Metric class
        # path
        config = EvidentlyMetric.metric_generator(
            "evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
        )
        ```

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyMetric

        # Configure an Evidently Metric generator using a Metric class
        from evidently.metrics import ColumnDriftMetric
        config = EvidentlyMetric.metric_generator(
            ColumnDriftMetric, columns="all", skip_id_column=True
        )
        ```

        Args:
            metric: The Evidently Metric class, class name or class path to use
                for the generator.
            columns: The columns to apply the generator to. Takes the same
                values that the Evidently `generate_column_metrics` function
                takes.
            skip_id_column: Whether to skip the ID column when applying the
                generator.
            parameters: Additional optional parameters needed to instantiate the
                Evidently Metric. These will be passed to the Evidently
                `generate_column_metrics` function.

        Returns:
            The EvidentlyMetric declarative representation of the Evidently
            Metric generator configuration.

        Raises:
            ValueError: If `metric` does not point to a valid Evidently Metric
                or MetricPreset class.
        """
        if isinstance(metric, str):
            metric_class = cls.get_metric_class(metric)
        elif issubclass(metric, (Metric, MetricPreset)):
            metric_class = metric
        else:
            raise ValueError(f"Invalid Evidently Metric class: {metric}")

        class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"

        config = cls(
            class_path=class_path,
            parameters=parameters,
            columns=columns,
            skip_id_column=skip_id_column,
            is_generator=True,
        )

        # Try to instantiate the configuration to check if the parameters are
        # valid
        config.to_evidently_metric()

        return config

    @classmethod
    def metric(
        cls,
        metric: Union[Type[Metric], Type[MetricPreset], str],
        **parameters: Any,
    ) -> "EvidentlyMetricConfig":
        """Create a declarative configuration for an Evidently Metric.

        Call this method to get a declarative representation for the
        configuration of an Evidently Metric.

        Some examples
        -------------

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyMetric

        # Configure an Evidently MetricPreset using its class name
        config = EvidentlyMetric.metric("DataDriftPreset")
        ```

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyMetric

        # Configure an Evidently MetricPreset using its full class path
        config = EvidentlyMetric.metric(
            "evidently.metric_preset.DataDriftPreset"
        )
        ```

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyMetric

        # Configure an Evidently Metric using its class and pass additional
        # parameters
        from evidently.metrics import ColumnSummaryMetric
        config = EvidentlyMetric.metric(
            ColumnSummaryMetric, column_name="age"
        )
        ```

        Args:
            metric: The Evidently Metric or MetricPreset class, class name or
                class path.
            parameters: Additional optional parameters needed to instantiate the
                Evidently Metric or MetricPreset.

        Returns:
            The EvidentlyMetric declarative representation of the Evidently
            Metric configuration.

        Raises:
            ValueError: If `metric` does not point to a valid Evidently Metric
                or MetricPreset class.
        """
        if isinstance(metric, str):
            metric_class = cls.get_metric_class(metric)
        elif issubclass(metric, (Metric, MetricPreset)):
            metric_class = metric
        else:
            raise ValueError(
                f"Invalid Evidently Metric or MetricPreset class: {metric}"
            )

        class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"

        config = cls(class_path=class_path, parameters=parameters)

        # Try to instantiate the configuration to check if the parameters are
        # valid
        config.to_evidently_metric()

        return config

    @classmethod
    def default_metrics(cls) -> List["EvidentlyMetricConfig"]:
        """Default Evidently metric configurations.

        Call this to fetch a default list of Evidently metrics to use in cases
        where no metrics are explicitly configured for a data validator.
        All available Evidently MetricPreset classes are used, except for the
        `TextOverviewPreset` which requires a text column, which we don't have
        by default.

        Returns:
            A list of EvidentlyMetricConfig objects to use as default metrics.
        """
        return [
            cls.metric(metric=metric_preset_class_name)
            for metric_preset_class_name in metric_preset.__all__
            # TextOverviewPreset requires a text column, which we don't
            # have by default
            if metric_preset_class_name != "TextOverviewPreset"
        ]

    def to_evidently_metric(
        self,
    ) -> Union[Metric, MetricPreset, BaseGenerator]:
        """Create an Evidently Metric, MetricPreset or metric generator object.

        Call this method to create an Evidently Metric, MetricPreset or metric
        generator instance from its declarative representation.

        Returns:
            The Evidently Metric, MetricPreset or metric generator object.

        Raises:
            ValueError: If the Evidently Metric, MetricPreset or column metric
                generator could not be instantiated.
        """
        metric_class = self.get_metric_class(self.class_path)

        if self.is_generator:
            try:
                return generate_column_metrics(
                    metric_class=metric_class,
                    columns=self.columns,
                    skip_id_column=self.skip_id_column,
                    parameters=self.parameters,
                )
            except Exception as e:
                raise ValueError(
                    f"Could not instantiate Evidently column Metric generator "
                    f"`{self.class_path}`: {str(e)}"
                )

        try:
            return metric_class(**self.parameters)
        except Exception as e:
            raise ValueError(
                f"Could not instantiate Evidently Metric or MetricPreset "
                f"`{self.class_path}`: {str(e)}"
            )

    model_config = ConfigDict(extra="forbid")
default_metrics() classmethod

Default Evidently metric configurations.

Call this to fetch a default list of Evidently metrics to use in cases where no metrics are explicitly configured for a data validator. All available Evidently MetricPreset classes are used, except for the TextOverviewPreset which requires a text column, which we don't have by default.

Returns:

Type Description
List[EvidentlyMetricConfig]

A list of EvidentlyMetricConfig objects to use as default metrics.

Source code in zenml/integrations/evidently/metrics.py
@classmethod
def default_metrics(cls) -> List["EvidentlyMetricConfig"]:
    """Default Evidently metric configurations.

    Call this to fetch a default list of Evidently metrics to use in cases
    where no metrics are explicitly configured for a data validator.
    All available Evidently MetricPreset classes are used, except for the
    `TextOverviewPreset` which requires a text column, which we don't have
    by default.

    Returns:
        A list of EvidentlyMetricConfig objects to use as default metrics.
    """
    return [
        cls.metric(metric=metric_preset_class_name)
        for metric_preset_class_name in metric_preset.__all__
        # TextOverviewPreset requires a text column, which we don't
        # have by default
        if metric_preset_class_name != "TextOverviewPreset"
    ]
get_metric_class(metric_name) staticmethod

Get the Evidently metric or metric preset class from a string.

Parameters:

Name Type Description Default
metric_name str

The metric or metric preset class or full class path.

required

Returns:

Type Description
Union[evidently.metrics.base_metric.Metric, evidently.metric_preset.metric_preset.MetricPreset]

The Evidently metric or metric preset class.

Exceptions:

Type Description
ValueError

If the name cannot be converted into a valid Evidently metric or metric preset class.

Source code in zenml/integrations/evidently/metrics.py
@staticmethod
def get_metric_class(metric_name: str) -> Union[Metric, MetricPreset]:
    """Get the Evidently metric or metric preset class from a string.

    Args:
        metric_name: The metric or metric preset class or full class
            path.

    Returns:
        The Evidently metric or metric preset class.

    Raises:
        ValueError: If the name cannot be converted into a valid Evidently
            metric or metric preset class.
    """
    # First, try to interpret the metric name as a full class path.
    if "." in metric_name:
        try:
            metric_class = source_utils.load(metric_name)
        except (ImportError, AttributeError) as e:
            raise ValueError(
                f"Could not import Evidently Metric or MetricPreset "
                f"`{metric_name}`: {str(e)}"
            )

    else:
        # Next, try to interpret the metric as a Metric or MetricPreset
        # class name
        if hasattr(metrics, metric_name):
            metric_class = getattr(metrics, metric_name)
        elif hasattr(metric_preset, metric_name):
            metric_class = getattr(metric_preset, metric_name)
        else:
            raise ValueError(
                f"Could not import Evidently Metric or MetricPreset "
                f"`{metric_name}`"
            )

    if not issubclass(metric_class, (Metric, MetricPreset)):
        raise ValueError(
            f"Class `{metric_name}` is not a valid Evidently "
            f"Metric or MetricPreset."
        )

    return metric_class
metric(metric, **parameters) classmethod

Create a declarative configuration for an Evidently Metric.

Call this method to get a declarative representation for the configuration of an Evidently Metric.

Some examples
from zenml.integrations.evidently.data_validators import EvidentlyMetric

# Configure an Evidently MetricPreset using its class name
config = EvidentlyMetric.metric("DataDriftPreset")
from zenml.integrations.evidently.data_validators import EvidentlyMetric

# Configure an Evidently MetricPreset using its full class path
config = EvidentlyMetric.metric(
    "evidently.metric_preset.DataDriftPreset"
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric

# Configure an Evidently Metric using its class and pass additional
# parameters
from evidently.metrics import ColumnSummaryMetric
config = EvidentlyMetric.metric(
    ColumnSummaryMetric, column_name="age"
)

Parameters:

Name Type Description Default
metric Union[Type[evidently.metrics.base_metric.Metric], Type[evidently.metric_preset.metric_preset.MetricPreset], str]

The Evidently Metric or MetricPreset class, class name or class path.

required
parameters Any

Additional optional parameters needed to instantiate the Evidently Metric or MetricPreset.

{}

Returns:

Type Description
EvidentlyMetricConfig

The EvidentlyMetric declarative representation of the Evidently Metric configuration.

Exceptions:

Type Description
ValueError

If metric does not point to a valid Evidently Metric or MetricPreset class.

Source code in zenml/integrations/evidently/metrics.py
@classmethod
def metric(
    cls,
    metric: Union[Type[Metric], Type[MetricPreset], str],
    **parameters: Any,
) -> "EvidentlyMetricConfig":
    """Create a declarative configuration for an Evidently Metric.

    Call this method to get a declarative representation for the
    configuration of an Evidently Metric.

    Some examples
    -------------

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyMetric

    # Configure an Evidently MetricPreset using its class name
    config = EvidentlyMetric.metric("DataDriftPreset")
    ```

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyMetric

    # Configure an Evidently MetricPreset using its full class path
    config = EvidentlyMetric.metric(
        "evidently.metric_preset.DataDriftPreset"
    )
    ```

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyMetric

    # Configure an Evidently Metric using its class and pass additional
    # parameters
    from evidently.metrics import ColumnSummaryMetric
    config = EvidentlyMetric.metric(
        ColumnSummaryMetric, column_name="age"
    )
    ```

    Args:
        metric: The Evidently Metric or MetricPreset class, class name or
            class path.
        parameters: Additional optional parameters needed to instantiate the
            Evidently Metric or MetricPreset.

    Returns:
        The EvidentlyMetric declarative representation of the Evidently
        Metric configuration.

    Raises:
        ValueError: If `metric` does not point to a valid Evidently Metric
            or MetricPreset class.
    """
    if isinstance(metric, str):
        metric_class = cls.get_metric_class(metric)
    elif issubclass(metric, (Metric, MetricPreset)):
        metric_class = metric
    else:
        raise ValueError(
            f"Invalid Evidently Metric or MetricPreset class: {metric}"
        )

    class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"

    config = cls(class_path=class_path, parameters=parameters)

    # Try to instantiate the configuration to check if the parameters are
    # valid
    config.to_evidently_metric()

    return config
metric_generator(metric, columns=None, skip_id_column=False, **parameters) classmethod

Create a declarative configuration for an Evidently column Metric generator.

Call this method to get a declarative representation for the configuration of an Evidently column Metric generator.

The columns, skip_id_column and parameters arguments will be passed to the Evidently generate_column_metrics function:

  • if columns is a list, it is interpreted as a list of column names.
  • if columns is a string, it can be one of values:
    • "all" - use all columns, including target/prediction columns
    • "num" - for numeric features
    • "cat" - for category features
    • "text" - for text features
    • "features" - for all features, not target/prediction columns.
  • a None value is the same as "all".
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyMetric

# Configure an Evidently Metric generator using a Metric class name
# and pass additional parameters
config = EvidentlyMetric.metric_generator(
    "ColumnQuantileMetric", columns="num", quantile=0.5
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric

# Configure an Evidently Metric generator using a full Metric class
# path
config = EvidentlyMetric.metric_generator(
    "evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric

# Configure an Evidently Metric generator using a Metric class
from evidently.metrics import ColumnDriftMetric
config = EvidentlyMetric.metric_generator(
    ColumnDriftMetric, columns="all", skip_id_column=True
)

Parameters:

Name Type Description Default
metric Union[Type[evidently.metrics.base_metric.Metric], str]

The Evidently Metric class, class name or class path to use for the generator.

required
columns Union[str, List[str]]

The columns to apply the generator to. Takes the same values that the Evidently generate_column_metrics function takes.

None
skip_id_column bool

Whether to skip the ID column when applying the generator.

False
parameters Any

Additional optional parameters needed to instantiate the Evidently Metric. These will be passed to the Evidently generate_column_metrics function.

{}

Returns:

Type Description
EvidentlyMetricConfig

The EvidentlyMetric declarative representation of the Evidently Metric generator configuration.

Exceptions:

Type Description
ValueError

If metric does not point to a valid Evidently Metric or MetricPreset class.

Source code in zenml/integrations/evidently/metrics.py
@classmethod
def metric_generator(
    cls,
    metric: Union[Type[Metric], str],
    columns: Optional[Union[str, List[str]]] = None,
    skip_id_column: bool = False,
    **parameters: Any,
) -> "EvidentlyMetricConfig":
    """Create a declarative configuration for an Evidently column Metric generator.

    Call this method to get a declarative representation for the
    configuration of an Evidently column Metric generator.

    The `columns`, `skip_id_column` and `parameters` arguments will be
    passed to the Evidently `generate_column_metrics` function:

    - if `columns` is a list, it is interpreted as a list of column names.
    - if `columns` is a string, it can be one of values:
        - "all" - use all columns, including target/prediction columns
        - "num" - for numeric features
        - "cat" - for category features
        - "text" - for text features
        - "features" - for all features, not target/prediction columns.
    - a None value is the same as "all".

    Some examples
    -------------

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyMetric

    # Configure an Evidently Metric generator using a Metric class name
    # and pass additional parameters
    config = EvidentlyMetric.metric_generator(
        "ColumnQuantileMetric", columns="num", quantile=0.5
    )
    ```

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyMetric

    # Configure an Evidently Metric generator using a full Metric class
    # path
    config = EvidentlyMetric.metric_generator(
        "evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
    )
    ```

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyMetric

    # Configure an Evidently Metric generator using a Metric class
    from evidently.metrics import ColumnDriftMetric
    config = EvidentlyMetric.metric_generator(
        ColumnDriftMetric, columns="all", skip_id_column=True
    )
    ```

    Args:
        metric: The Evidently Metric class, class name or class path to use
            for the generator.
        columns: The columns to apply the generator to. Takes the same
            values that the Evidently `generate_column_metrics` function
            takes.
        skip_id_column: Whether to skip the ID column when applying the
            generator.
        parameters: Additional optional parameters needed to instantiate the
            Evidently Metric. These will be passed to the Evidently
            `generate_column_metrics` function.

    Returns:
        The EvidentlyMetric declarative representation of the Evidently
        Metric generator configuration.

    Raises:
        ValueError: If `metric` does not point to a valid Evidently Metric
            or MetricPreset class.
    """
    if isinstance(metric, str):
        metric_class = cls.get_metric_class(metric)
    elif issubclass(metric, (Metric, MetricPreset)):
        metric_class = metric
    else:
        raise ValueError(f"Invalid Evidently Metric class: {metric}")

    class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"

    config = cls(
        class_path=class_path,
        parameters=parameters,
        columns=columns,
        skip_id_column=skip_id_column,
        is_generator=True,
    )

    # Try to instantiate the configuration to check if the parameters are
    # valid
    config.to_evidently_metric()

    return config
to_evidently_metric(self)

Create an Evidently Metric, MetricPreset or metric generator object.

Call this method to create an Evidently Metric, MetricPreset or metric generator instance from its declarative representation.

Returns:

Type Description
Union[evidently.metrics.base_metric.Metric, evidently.metric_preset.metric_preset.MetricPreset, evidently.utils.generators.BaseGenerator]

The Evidently Metric, MetricPreset or metric generator object.

Exceptions:

Type Description
ValueError

If the Evidently Metric, MetricPreset or column metric generator could not be instantiated.

Source code in zenml/integrations/evidently/metrics.py
def to_evidently_metric(
    self,
) -> Union[Metric, MetricPreset, BaseGenerator]:
    """Create an Evidently Metric, MetricPreset or metric generator object.

    Call this method to create an Evidently Metric, MetricPreset or metric
    generator instance from its declarative representation.

    Returns:
        The Evidently Metric, MetricPreset or metric generator object.

    Raises:
        ValueError: If the Evidently Metric, MetricPreset or column metric
            generator could not be instantiated.
    """
    metric_class = self.get_metric_class(self.class_path)

    if self.is_generator:
        try:
            return generate_column_metrics(
                metric_class=metric_class,
                columns=self.columns,
                skip_id_column=self.skip_id_column,
                parameters=self.parameters,
            )
        except Exception as e:
            raise ValueError(
                f"Could not instantiate Evidently column Metric generator "
                f"`{self.class_path}`: {str(e)}"
            )

    try:
        return metric_class(**self.parameters)
    except Exception as e:
        raise ValueError(
            f"Could not instantiate Evidently Metric or MetricPreset "
            f"`{self.class_path}`: {str(e)}"
        )

steps special

Initialization of the Evidently Standard Steps.

evidently_report

Implementation of the Evidently Report Step.

evidently_test

Implementation of the Evidently Test Step.

tests

ZenML declarative representation of Evidently Tests.

EvidentlyTestConfig (BaseModel)

Declarative Evidently Test configuration.

This is a declarative representation of the configuration that goes into an Evidently Test, TestPreset or Test generator instance. We need this to be able to store the configuration as part of a ZenML step parameter and later instantiate the Evidently Test from it.

This representation covers all 3 possible ways of configuring an Evidently Test or Test-like object that can later be used in an Evidently TestSuite:

  1. A Test (derived from the Test class).
  2. A TestPreset (derived from the TestPreset class).
  3. A column Test generator (derived from the BaseGenerator class).

Ideally, it should be possible to just pass a Test or Test-like object to this class and have it automatically derive the configuration used to instantiate it. Unfortunately, this is not possible because the Evidently Test classes are not designed in a way that allows us to extract the constructor parameters from them in a generic way.

Attributes:

Name Type Description
class_path str

The full class path of the Evidently Test class.

parameters Dict[str, Any]

The parameters of the Evidently Test.

is_generator bool

Whether this is an Evidently column Test generator.

columns Union[str, List[str]]

The columns that the Evidently column Test generator is applied to. Only used if generator is True.

Source code in zenml/integrations/evidently/tests.py
class EvidentlyTestConfig(BaseModel):
    """Declarative Evidently Test configuration.

    This is a declarative representation of the configuration that goes into an
    Evidently Test, TestPreset or Test generator instance. We need this to
    be able to store the configuration as part of a ZenML step parameter and
    later instantiate the Evidently Test from it.

    This representation covers all 3 possible ways of configuring an Evidently
    Test or Test-like object that can later be used in an Evidently TestSuite:

    1. A Test (derived from the Test class).
    2. A TestPreset (derived from the TestPreset class).
    3. A column Test generator (derived from the BaseGenerator class).

    Ideally, it should be possible to just pass a Test or Test-like
    object to this class and have it automatically derive the configuration used
    to instantiate it. Unfortunately, this is not possible because the Evidently
    Test classes are not designed in a way that allows us to extract the
    constructor parameters from them in a generic way.

    Attributes:
        class_path: The full class path of the Evidently Test class.
        parameters: The parameters of the Evidently Test.
        is_generator: Whether this is an Evidently column Test generator.
        columns: The columns that the Evidently column Test generator is
            applied to. Only used if `generator` is True.
    """

    class_path: str
    parameters: Dict[str, Any] = Field(default_factory=dict)
    is_generator: bool = False
    columns: Optional[Union[str, List[str]]] = Field(
        default=None, union_mode="left_to_right"
    )

    @staticmethod
    def get_test_class(test_name: str) -> Union[Test, TestPreset]:
        """Get the Evidently test or test preset class from a string.

        Args:
            test_name: The test or test preset class or full class
                path.

        Returns:
            The Evidently test or test preset class.

        Raises:
            ValueError: If the name cannot be converted into a valid Evidently
                test or test preset class.
        """
        # First, try to interpret the test name as a full class path.
        if "." in test_name:
            try:
                test_class = source_utils.load(test_name)
            except (ImportError, AttributeError) as e:
                raise ValueError(
                    f"Could not import Evidently Test or TestPreset "
                    f"`{test_name}`: {str(e)}"
                )

        else:
            # Next, try to interpret the test as a Test or TestPreset
            # class name
            if hasattr(tests, test_name):
                test_class = getattr(tests, test_name)
            elif hasattr(test_preset, test_name):
                test_class = getattr(test_preset, test_name)
            else:
                raise ValueError(
                    f"Could not import Evidently Test or TestPreset "
                    f"`{test_name}`"
                )

        if not issubclass(test_class, (Test, TestPreset)):
            raise ValueError(
                f"Class `{test_name}` is not a valid Evidently "
                f"Test or TestPreset."
            )

        return test_class

    @classmethod
    def test_generator(
        cls,
        test: Union[Type[Test], str],
        columns: Optional[Union[str, List[str]]] = None,
        **parameters: Any,
    ) -> "EvidentlyTestConfig":
        """Create a declarative configuration for an Evidently column Test generator.

        Call this method to get a declarative representation for the
        configuration of an Evidently column Test generator.

        The `columns`, `parameters` arguments will be
        passed to the Evidently `generate_column_tests` function:

        - if `columns` is a list, it is interpreted as a list of column names.
        - if `columns` is a string, it can be one of values:
            - "all" - use all columns, including target/prediction columns
            - "num" - for numeric features
            - "cat" - for category features
            - "text" - for text features
            - "features" - for all features, not target/prediction columns.
        - a None value is the same as "all".

        Some examples
        -------------

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyTest

        # Configure an Evidently Test generator using a Test class name
        # and pass additional parameters
        config = EvidentlyTest.test_generator(
            "TestColumnValueMin", columns="num", gt=0.5
        )
        ```

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyTest

        # Configure an Evidently Test generator using a full Test class
        # path
        config = EvidentlyTest.test_generator(
            "evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
        )
        ```

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyTest

        # Configure an Evidently Test generator using a Test class
        from evidently.tests import TestColumnQuantile
        config = EvidentlyTest.test_generator(
            TestColumnQuantile, columns="all", quantile=0.5
        )
        ```

        Args:
            test: The Evidently Test class, class name or class path to use
                for the generator.
            columns: The columns to apply the generator to. Takes the same
                values that the Evidently `generate_column_tests` function
                takes.
            parameters: Additional optional parameters needed to instantiate the
                Evidently Test. These will be passed to the Evidently
                `generate_column_tests` function.

        Returns:
            The EvidentlyTest declarative representation of the Evidently
            Test generator configuration.

        Raises:
            ValueError: If `test` does not point to a valid Evidently Test
                or TestPreset class.
        """
        if isinstance(test, str):
            test_class = cls.get_test_class(test)
        elif issubclass(test, (Test, TestPreset)):
            test_class = test
        else:
            raise ValueError(f"Invalid Evidently Test class: {test}")

        class_path = f"{test_class.__module__}." f"{test_class.__name__}"

        config = cls(
            class_path=class_path,
            parameters=parameters,
            columns=columns,
            is_generator=True,
        )

        # Try to instantiate the configuration to check if the parameters are
        # valid
        config.to_evidently_test()

        return config

    @classmethod
    def test(
        cls,
        test: Union[Type[Test], Type[TestPreset], str],
        **parameters: Any,
    ) -> "EvidentlyTestConfig":
        """Create a declarative configuration for an Evidently Test.

        Call this method to get a declarative representation for the
        configuration of an Evidently Test.

        Some examples
        -------------

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyTest

        # Configure an Evidently TestPreset using its class name
        config = EvidentlyTest.test("DataDriftPreset")
        ```

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyTest

        # Configure an Evidently TestPreset using its full class path
        config = EvidentlyTest.test(
            "evidently.test_preset.DataDriftPreset"
        )
        ```

        ```python
        from zenml.integrations.evidently.data_validators import EvidentlyTest

        # Configure an Evidently Test using its class and pass additional
        # parameters
        from evidently.tests import ColumnSummaryTest
        config = EvidentlyTest.test(
            ColumnSummaryTest, column_name="age"
        )
        ```

        Args:
            test: The Evidently Test or TestPreset class, class name or
                class path.
            parameters: Additional optional parameters needed to instantiate the
                Evidently Test or TestPreset.

        Returns:
            The EvidentlyTest declarative representation of the Evidently
            Test configuration.

        Raises:
            ValueError: If `test` does not point to a valid Evidently Test
                or TestPreset class.
        """
        if isinstance(test, str):
            test_class = cls.get_test_class(test)
        elif issubclass(test, (Test, TestPreset)):
            test_class = test
        else:
            raise ValueError(
                f"Invalid Evidently Test or TestPreset class: {test}"
            )

        class_path = f"{test_class.__module__}." f"{test_class.__name__}"
        config = cls(class_path=class_path, parameters=parameters)

        # Try to instantiate the configuration to check if the parameters are
        # valid
        config.to_evidently_test()

        return config

    @classmethod
    def default_tests(cls) -> List["EvidentlyTestConfig"]:
        """Default Evidently test configurations.

        Call this to fetch a default list of Evidently tests to use in cases
        where no tests are explicitly configured for a data validator.
        All available Evidently TestPreset classes are used.

        Returns:
            A list of EvidentlyTestConfig objects to use as default tests.
        """
        return [
            cls.test(test=test_preset_class_name)
            for test_preset_class_name in test_preset.__all__
        ]

    def to_evidently_test(self) -> Union[Test, TestPreset, BaseGenerator]:
        """Create an Evidently Test, TestPreset or test generator object.

        Call this method to create an Evidently Test, TestPreset or test
        generator instance from its declarative representation.

        Returns:
            The Evidently Test, TestPreset or test generator object.

        Raises:
            ValueError: If the Evidently Test, TestPreset or column test
                generator could not be instantiated.
        """
        test_class = self.get_test_class(self.class_path)

        if self.is_generator:
            try:
                return generate_column_tests(
                    test_class=test_class,
                    columns=self.columns,
                    parameters=self.parameters,
                )
            except Exception as e:
                raise ValueError(
                    f"Could not instantiate Evidently column Test generator "
                    f"`{self.class_path}`: {str(e)}"
                )

        try:
            return test_class(**self.parameters)
        except Exception as e:
            raise ValueError(
                f"Could not instantiate Evidently Test or TestPreset "
                f"`{self.class_path}`: {str(e)}"
            )

    model_config = ConfigDict(extra="forbid")
default_tests() classmethod

Default Evidently test configurations.

Call this to fetch a default list of Evidently tests to use in cases where no tests are explicitly configured for a data validator. All available Evidently TestPreset classes are used.

Returns:

Type Description
List[EvidentlyTestConfig]

A list of EvidentlyTestConfig objects to use as default tests.

Source code in zenml/integrations/evidently/tests.py
@classmethod
def default_tests(cls) -> List["EvidentlyTestConfig"]:
    """Default Evidently test configurations.

    Call this to fetch a default list of Evidently tests to use in cases
    where no tests are explicitly configured for a data validator.
    All available Evidently TestPreset classes are used.

    Returns:
        A list of EvidentlyTestConfig objects to use as default tests.
    """
    return [
        cls.test(test=test_preset_class_name)
        for test_preset_class_name in test_preset.__all__
    ]
get_test_class(test_name) staticmethod

Get the Evidently test or test preset class from a string.

Parameters:

Name Type Description Default
test_name str

The test or test preset class or full class path.

required

Returns:

Type Description
Union[evidently.tests.base_test.Test, evidently.test_preset.test_preset.TestPreset]

The Evidently test or test preset class.

Exceptions:

Type Description
ValueError

If the name cannot be converted into a valid Evidently test or test preset class.

Source code in zenml/integrations/evidently/tests.py
@staticmethod
def get_test_class(test_name: str) -> Union[Test, TestPreset]:
    """Get the Evidently test or test preset class from a string.

    Args:
        test_name: The test or test preset class or full class
            path.

    Returns:
        The Evidently test or test preset class.

    Raises:
        ValueError: If the name cannot be converted into a valid Evidently
            test or test preset class.
    """
    # First, try to interpret the test name as a full class path.
    if "." in test_name:
        try:
            test_class = source_utils.load(test_name)
        except (ImportError, AttributeError) as e:
            raise ValueError(
                f"Could not import Evidently Test or TestPreset "
                f"`{test_name}`: {str(e)}"
            )

    else:
        # Next, try to interpret the test as a Test or TestPreset
        # class name
        if hasattr(tests, test_name):
            test_class = getattr(tests, test_name)
        elif hasattr(test_preset, test_name):
            test_class = getattr(test_preset, test_name)
        else:
            raise ValueError(
                f"Could not import Evidently Test or TestPreset "
                f"`{test_name}`"
            )

    if not issubclass(test_class, (Test, TestPreset)):
        raise ValueError(
            f"Class `{test_name}` is not a valid Evidently "
            f"Test or TestPreset."
        )

    return test_class
test(test, **parameters) classmethod

Create a declarative configuration for an Evidently Test.

Call this method to get a declarative representation for the configuration of an Evidently Test.

Some examples
from zenml.integrations.evidently.data_validators import EvidentlyTest

# Configure an Evidently TestPreset using its class name
config = EvidentlyTest.test("DataDriftPreset")
from zenml.integrations.evidently.data_validators import EvidentlyTest

# Configure an Evidently TestPreset using its full class path
config = EvidentlyTest.test(
    "evidently.test_preset.DataDriftPreset"
)
from zenml.integrations.evidently.data_validators import EvidentlyTest

# Configure an Evidently Test using its class and pass additional
# parameters
from evidently.tests import ColumnSummaryTest
config = EvidentlyTest.test(
    ColumnSummaryTest, column_name="age"
)

Parameters:

Name Type Description Default
test Union[Type[evidently.tests.base_test.Test], Type[evidently.test_preset.test_preset.TestPreset], str]

The Evidently Test or TestPreset class, class name or class path.

required
parameters Any

Additional optional parameters needed to instantiate the Evidently Test or TestPreset.

{}

Returns:

Type Description
EvidentlyTestConfig

The EvidentlyTest declarative representation of the Evidently Test configuration.

Exceptions:

Type Description
ValueError

If test does not point to a valid Evidently Test or TestPreset class.

Source code in zenml/integrations/evidently/tests.py
@classmethod
def test(
    cls,
    test: Union[Type[Test], Type[TestPreset], str],
    **parameters: Any,
) -> "EvidentlyTestConfig":
    """Create a declarative configuration for an Evidently Test.

    Call this method to get a declarative representation for the
    configuration of an Evidently Test.

    Some examples
    -------------

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyTest

    # Configure an Evidently TestPreset using its class name
    config = EvidentlyTest.test("DataDriftPreset")
    ```

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyTest

    # Configure an Evidently TestPreset using its full class path
    config = EvidentlyTest.test(
        "evidently.test_preset.DataDriftPreset"
    )
    ```

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyTest

    # Configure an Evidently Test using its class and pass additional
    # parameters
    from evidently.tests import ColumnSummaryTest
    config = EvidentlyTest.test(
        ColumnSummaryTest, column_name="age"
    )
    ```

    Args:
        test: The Evidently Test or TestPreset class, class name or
            class path.
        parameters: Additional optional parameters needed to instantiate the
            Evidently Test or TestPreset.

    Returns:
        The EvidentlyTest declarative representation of the Evidently
        Test configuration.

    Raises:
        ValueError: If `test` does not point to a valid Evidently Test
            or TestPreset class.
    """
    if isinstance(test, str):
        test_class = cls.get_test_class(test)
    elif issubclass(test, (Test, TestPreset)):
        test_class = test
    else:
        raise ValueError(
            f"Invalid Evidently Test or TestPreset class: {test}"
        )

    class_path = f"{test_class.__module__}." f"{test_class.__name__}"
    config = cls(class_path=class_path, parameters=parameters)

    # Try to instantiate the configuration to check if the parameters are
    # valid
    config.to_evidently_test()

    return config
test_generator(test, columns=None, **parameters) classmethod

Create a declarative configuration for an Evidently column Test generator.

Call this method to get a declarative representation for the configuration of an Evidently column Test generator.

The columns, parameters arguments will be passed to the Evidently generate_column_tests function:

  • if columns is a list, it is interpreted as a list of column names.
  • if columns is a string, it can be one of values:
    • "all" - use all columns, including target/prediction columns
    • "num" - for numeric features
    • "cat" - for category features
    • "text" - for text features
    • "features" - for all features, not target/prediction columns.
  • a None value is the same as "all".
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyTest

# Configure an Evidently Test generator using a Test class name
# and pass additional parameters
config = EvidentlyTest.test_generator(
    "TestColumnValueMin", columns="num", gt=0.5
)
from zenml.integrations.evidently.data_validators import EvidentlyTest

# Configure an Evidently Test generator using a full Test class
# path
config = EvidentlyTest.test_generator(
    "evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
)
from zenml.integrations.evidently.data_validators import EvidentlyTest

# Configure an Evidently Test generator using a Test class
from evidently.tests import TestColumnQuantile
config = EvidentlyTest.test_generator(
    TestColumnQuantile, columns="all", quantile=0.5
)

Parameters:

Name Type Description Default
test Union[Type[evidently.tests.base_test.Test], str]

The Evidently Test class, class name or class path to use for the generator.

required
columns Union[str, List[str]]

The columns to apply the generator to. Takes the same values that the Evidently generate_column_tests function takes.

None
parameters Any

Additional optional parameters needed to instantiate the Evidently Test. These will be passed to the Evidently generate_column_tests function.

{}

Returns:

Type Description
EvidentlyTestConfig

The EvidentlyTest declarative representation of the Evidently Test generator configuration.

Exceptions:

Type Description
ValueError

If test does not point to a valid Evidently Test or TestPreset class.

Source code in zenml/integrations/evidently/tests.py
@classmethod
def test_generator(
    cls,
    test: Union[Type[Test], str],
    columns: Optional[Union[str, List[str]]] = None,
    **parameters: Any,
) -> "EvidentlyTestConfig":
    """Create a declarative configuration for an Evidently column Test generator.

    Call this method to get a declarative representation for the
    configuration of an Evidently column Test generator.

    The `columns`, `parameters` arguments will be
    passed to the Evidently `generate_column_tests` function:

    - if `columns` is a list, it is interpreted as a list of column names.
    - if `columns` is a string, it can be one of values:
        - "all" - use all columns, including target/prediction columns
        - "num" - for numeric features
        - "cat" - for category features
        - "text" - for text features
        - "features" - for all features, not target/prediction columns.
    - a None value is the same as "all".

    Some examples
    -------------

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyTest

    # Configure an Evidently Test generator using a Test class name
    # and pass additional parameters
    config = EvidentlyTest.test_generator(
        "TestColumnValueMin", columns="num", gt=0.5
    )
    ```

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyTest

    # Configure an Evidently Test generator using a full Test class
    # path
    config = EvidentlyTest.test_generator(
        "evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
    )
    ```

    ```python
    from zenml.integrations.evidently.data_validators import EvidentlyTest

    # Configure an Evidently Test generator using a Test class
    from evidently.tests import TestColumnQuantile
    config = EvidentlyTest.test_generator(
        TestColumnQuantile, columns="all", quantile=0.5
    )
    ```

    Args:
        test: The Evidently Test class, class name or class path to use
            for the generator.
        columns: The columns to apply the generator to. Takes the same
            values that the Evidently `generate_column_tests` function
            takes.
        parameters: Additional optional parameters needed to instantiate the
            Evidently Test. These will be passed to the Evidently
            `generate_column_tests` function.

    Returns:
        The EvidentlyTest declarative representation of the Evidently
        Test generator configuration.

    Raises:
        ValueError: If `test` does not point to a valid Evidently Test
            or TestPreset class.
    """
    if isinstance(test, str):
        test_class = cls.get_test_class(test)
    elif issubclass(test, (Test, TestPreset)):
        test_class = test
    else:
        raise ValueError(f"Invalid Evidently Test class: {test}")

    class_path = f"{test_class.__module__}." f"{test_class.__name__}"

    config = cls(
        class_path=class_path,
        parameters=parameters,
        columns=columns,
        is_generator=True,
    )

    # Try to instantiate the configuration to check if the parameters are
    # valid
    config.to_evidently_test()

    return config
to_evidently_test(self)

Create an Evidently Test, TestPreset or test generator object.

Call this method to create an Evidently Test, TestPreset or test generator instance from its declarative representation.

Returns:

Type Description
Union[evidently.tests.base_test.Test, evidently.test_preset.test_preset.TestPreset, evidently.utils.generators.BaseGenerator]

The Evidently Test, TestPreset or test generator object.

Exceptions:

Type Description
ValueError

If the Evidently Test, TestPreset or column test generator could not be instantiated.

Source code in zenml/integrations/evidently/tests.py
def to_evidently_test(self) -> Union[Test, TestPreset, BaseGenerator]:
    """Create an Evidently Test, TestPreset or test generator object.

    Call this method to create an Evidently Test, TestPreset or test
    generator instance from its declarative representation.

    Returns:
        The Evidently Test, TestPreset or test generator object.

    Raises:
        ValueError: If the Evidently Test, TestPreset or column test
            generator could not be instantiated.
    """
    test_class = self.get_test_class(self.class_path)

    if self.is_generator:
        try:
            return generate_column_tests(
                test_class=test_class,
                columns=self.columns,
                parameters=self.parameters,
            )
        except Exception as e:
            raise ValueError(
                f"Could not instantiate Evidently column Test generator "
                f"`{self.class_path}`: {str(e)}"
            )

    try:
        return test_class(**self.parameters)
    except Exception as e:
        raise ValueError(
            f"Could not instantiate Evidently Test or TestPreset "
            f"`{self.class_path}`: {str(e)}"
        )