Evidently
zenml.integrations.evidently
special
Initialization of the Evidently integration.
The Evidently integration provides a way to monitor your models in production. It includes a way to detect data drift and different kinds of model performance issues.
The results of Evidently calculations can either be exported as an interactive dashboard (visualized as an html file or in your Jupyter notebook), or as a JSON file.
EvidentlyIntegration (Integration)
Evidently integration for ZenML.
Source code in zenml/integrations/evidently/__init__.py
class EvidentlyIntegration(Integration):
"""[Evidently](https://github.com/evidentlyai/evidently) integration for ZenML."""
NAME = EVIDENTLY
REQUIREMENTS = ["evidently>0.2.6,<0.4.5"] # supports pyyaml 6
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
"""Declare the stack component flavors for the Great Expectations integration.
Returns:
List of stack component flavors for this integration.
"""
from zenml.integrations.evidently.flavors import (
EvidentlyDataValidatorFlavor,
)
return [EvidentlyDataValidatorFlavor]
flavors()
classmethod
Declare the stack component flavors for the Great Expectations integration.
Returns:
| Type | Description |
|---|---|
List[Type[zenml.stack.flavor.Flavor]] |
List of stack component flavors for this integration. |
Source code in zenml/integrations/evidently/__init__.py
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
"""Declare the stack component flavors for the Great Expectations integration.
Returns:
List of stack component flavors for this integration.
"""
from zenml.integrations.evidently.flavors import (
EvidentlyDataValidatorFlavor,
)
return [EvidentlyDataValidatorFlavor]
column_mapping
ZenML representation of an Evidently column mapping.
EvidentlyColumnMapping (BaseModel)
pydantic-model
Column mapping configuration for Evidently.
This class is a 1-to-1 serializable analogue of Evidently's ColumnMapping data type that can be used as a step configuration field (see https://docs.evidentlyai.com/user-guide/input-data/column-mapping).
Attributes:
| Name | Type | Description |
|---|---|---|
target |
Optional[str] |
target column |
prediction |
Union[str, Sequence[str]] |
target column |
datetime |
Optional[str] |
datetime column |
id |
Optional[str] |
id column |
numerical_features |
Optional[List[str]] |
numerical features |
categorical_features |
Optional[List[str]] |
categorical features |
datetime_features |
Optional[List[str]] |
datetime features |
target_names |
Optional[List[str]] |
target column names |
task |
Optional[str] |
model task |
pos_label |
Union[str, int] |
positive label |
text_features |
Optional[List[str]] |
text features |
Source code in zenml/integrations/evidently/column_mapping.py
class EvidentlyColumnMapping(BaseModel):
"""Column mapping configuration for Evidently.
This class is a 1-to-1 serializable analogue of Evidently's
ColumnMapping data type that can be used as a step configuration field
(see https://docs.evidentlyai.com/user-guide/input-data/column-mapping).
Attributes:
target: target column
prediction: target column
datetime: datetime column
id: id column
numerical_features: numerical features
categorical_features: categorical features
datetime_features: datetime features
target_names: target column names
task: model task
pos_label: positive label
text_features: text features
"""
target: Optional[str] = None
prediction: Optional[Union[str, Sequence[str]]] = "prediction"
datetime: Optional[str] = None
id: Optional[str] = None
numerical_features: Optional[List[str]] = None
categorical_features: Optional[List[str]] = None
datetime_features: Optional[List[str]] = None
target_names: Optional[List[str]] = None
task: Optional[str] = None
pos_label: Optional[Union[str, int]] = 1
text_features: Optional[List[str]] = None
def to_evidently_column_mapping(self) -> ColumnMapping:
"""Convert this Pydantic object to an Evidently ColumnMapping object.
Returns:
An Evidently column mapping converted from this Pydantic object.
"""
column_mapping = ColumnMapping()
# preserve the Evidently defaults where possible
column_mapping.target = self.target or column_mapping.target
column_mapping.prediction = (
self.prediction or column_mapping.prediction
)
column_mapping.datetime = self.datetime or column_mapping.datetime
column_mapping.id = self.id or column_mapping.id
column_mapping.numerical_features = (
self.numerical_features or column_mapping.numerical_features
)
column_mapping.datetime_features = (
self.datetime_features or column_mapping.datetime_features
)
column_mapping.target_names = (
self.target_names or column_mapping.target_names
)
column_mapping.task = self.task or column_mapping.task
column_mapping.pos_label = self.pos_label or column_mapping.pos_label
column_mapping.text_features = (
self.text_features or column_mapping.text_features
)
return column_mapping
to_evidently_column_mapping(self)
Convert this Pydantic object to an Evidently ColumnMapping object.
Returns:
| Type | Description |
|---|---|
evidently.ColumnMapping |
An Evidently column mapping converted from this Pydantic object. |
Source code in zenml/integrations/evidently/column_mapping.py
def to_evidently_column_mapping(self) -> ColumnMapping:
"""Convert this Pydantic object to an Evidently ColumnMapping object.
Returns:
An Evidently column mapping converted from this Pydantic object.
"""
column_mapping = ColumnMapping()
# preserve the Evidently defaults where possible
column_mapping.target = self.target or column_mapping.target
column_mapping.prediction = (
self.prediction or column_mapping.prediction
)
column_mapping.datetime = self.datetime or column_mapping.datetime
column_mapping.id = self.id or column_mapping.id
column_mapping.numerical_features = (
self.numerical_features or column_mapping.numerical_features
)
column_mapping.datetime_features = (
self.datetime_features or column_mapping.datetime_features
)
column_mapping.target_names = (
self.target_names or column_mapping.target_names
)
column_mapping.task = self.task or column_mapping.task
column_mapping.pos_label = self.pos_label or column_mapping.pos_label
column_mapping.text_features = (
self.text_features or column_mapping.text_features
)
return column_mapping
data_validators
special
Initialization of the Evidently data validator for ZenML.
evidently_data_validator
Implementation of the Evidently data validator.
EvidentlyDataValidator (BaseDataValidator)
Evidently data validator stack component.
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
class EvidentlyDataValidator(BaseDataValidator):
"""Evidently data validator stack component."""
NAME: ClassVar[str] = "Evidently"
FLAVOR: ClassVar[Type[BaseDataValidatorFlavor]] = (
EvidentlyDataValidatorFlavor
)
@classmethod
def _unpack_options(
cls, option_list: Sequence[Tuple[str, Dict[str, Any]]]
) -> Sequence[Any]:
"""Unpack Evidently options.
Implements de-serialization for [Evidently options](https://docs.evidentlyai.com/user-guide/customization)
that can be passed as constructor arguments when creating Profile and
Dashboard objects. The convention used is that each item in the list
consists of two elements:
* a string containing the full class path of a `dataclass` based
class with Evidently options
* a dictionary with kwargs used as parameters for the option instance
For example,
```python
options = [
(
"evidently.options.ColorOptions",{
"primary_color": "#5a86ad",
"fill_color": "#fff4f2",
"zero_line_color": "#016795",
"current_data_color": "#c292a1",
"reference_data_color": "#017b92",
}
),
]
```
This is the same as saying:
```python
from evidently.options import ColorOptions
color_scheme = ColorOptions()
color_scheme.primary_color = "#5a86ad"
color_scheme.fill_color = "#fff4f2"
color_scheme.zero_line_color = "#016795"
color_scheme.current_data_color = "#c292a1"
color_scheme.reference_data_color = "#017b92"
```
Args:
option_list: list of packed Evidently options
Returns:
A list of unpacked Evidently options
Raises:
ValueError: if one of the passed Evidently class paths cannot be
resolved to an actual class.
"""
options = []
for option_clspath, option_args in option_list:
try:
option_cls = source_utils.load(option_clspath)
except AttributeError:
raise ValueError(
f"Could not map the `{option_clspath}` Evidently option "
f"class path to a valid class."
)
option = option_cls(**option_args)
options.append(option)
return options
@staticmethod
def _download_nltk_data() -> None:
"""Download NLTK data for text metrics and tests.
Raises:
ImportError: if NLTK is not installed.
"""
try:
import nltk # type: ignore[import-untyped]
from nltk.data import ( # type: ignore[import-untyped]
path as nltk_path,
)
except ImportError:
raise ImportError(
"NLTK is not installed. Please install NLTK to use "
"Evidently text metrics and tests."
)
# Configure NLTK to use the current working directory to download and
# lookup data. This is necessary because the default download directory
# is not writable in some Docker containers.
nltk_path.append(os.getcwd())
# Download NLTK data. We need this later on for the Evidently text report.
nltk.download("words", download_dir=os.getcwd())
nltk.download("wordnet", download_dir=os.getcwd())
nltk.download("omw-1.4", download_dir=os.getcwd())
def data_profiling(
self,
dataset: pd.DataFrame,
comparison_dataset: Optional[pd.DataFrame] = None,
profile_list: Optional[Sequence[EvidentlyMetricConfig]] = None,
column_mapping: Optional[ColumnMapping] = None,
report_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
download_nltk_data: bool = False,
**kwargs: Any,
) -> Report:
"""Analyze a dataset and generate a data report with Evidently.
The method takes in an optional list of Evidently options to be passed
to the report constructor (`report_options`). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option `dataclass`, the second is a dictionary of kwargs with the actual
option parameters, e.g.:
```python
options = [
(
"evidently.options.ColorOptions",{
"primary_color": "#5a86ad",
"fill_color": "#fff4f2",
"zero_line_color": "#016795",
"current_data_color": "#c292a1",
"reference_data_color": "#017b92",
}
),
]
```
Args:
dataset: Target dataset to be profiled. When a comparison dataset
is provided, this dataset is considered the reference dataset.
comparison_dataset: Optional dataset to be used for data profiles
that require a current dataset for comparison (e.g data drift
profiles).
profile_list: List of Evidently metric configurations to
be included in the report. If not provided, all available
metric presets will be included.
column_mapping: Properties of the DataFrame columns used
report_options: List of Evidently options to be passed to the
report constructor.
download_nltk_data: Whether to download NLTK data for text metrics.
Defaults to False.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Report as JSON object and as HTML.
"""
if download_nltk_data:
self._download_nltk_data()
profile_list = profile_list or EvidentlyMetricConfig.default_metrics()
metrics = [metric.to_evidently_metric() for metric in profile_list]
unpacked_report_options = self._unpack_options(report_options)
report = Report(metrics=metrics, options=unpacked_report_options)
report.run(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return report
def data_validation(
self,
dataset: Any,
comparison_dataset: Optional[Any] = None,
check_list: Optional[Sequence[EvidentlyTestConfig]] = None,
test_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
column_mapping: Optional[ColumnMapping] = None,
download_nltk_data: bool = False,
**kwargs: Any,
) -> TestSuite:
"""Validate a dataset with Evidently.
Args:
dataset: Target dataset to be validated.
comparison_dataset: Optional dataset to be used for data validation
that require a baseline for comparison (e.g data drift
validation).
check_list: List of Evidently test configurations to be
included in the test suite. If not provided, all available
test presets will be included.
test_options: List of Evidently options to be passed to the
test suite constructor.
column_mapping: Properties of the DataFrame columns used
download_nltk_data: Whether to download NLTK data for text tests.
Defaults to False.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Test Suite as JSON object and as HTML.
"""
if download_nltk_data:
self._download_nltk_data()
check_list = check_list or EvidentlyTestConfig.default_tests()
tests = [test.to_evidently_test() for test in check_list]
unpacked_test_options = self._unpack_options(test_options)
test_suite = TestSuite(tests=tests, options=unpacked_test_options)
test_suite.run(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return test_suite
FLAVOR (BaseDataValidatorFlavor)
Evidently data validator flavor.
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
class EvidentlyDataValidatorFlavor(BaseDataValidatorFlavor):
"""Evidently data validator flavor."""
@property
def name(self) -> str:
"""Name of the flavor.
Returns:
The name of the flavor.
"""
return EVIDENTLY_DATA_VALIDATOR_FLAVOR
@property
def docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_docs_url()
@property
def sdk_docs_url(self) -> Optional[str]:
"""A url to point at SDK docs explaining this flavor.
Returns:
A flavor SDK docs url.
"""
return self.generate_default_sdk_docs_url()
@property
def logo_url(self) -> str:
"""A url to represent the flavor in the dashboard.
Returns:
The flavor logo.
"""
return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/data_validator/evidently.png"
@property
def implementation_class(self) -> Type["EvidentlyDataValidator"]:
"""Implementation class.
Returns:
The implementation class.
"""
from zenml.integrations.evidently.data_validators import (
EvidentlyDataValidator,
)
return EvidentlyDataValidator
docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
| Type | Description |
|---|---|
Optional[str] |
A flavor docs url. |
implementation_class: Type[EvidentlyDataValidator]
property
readonly
Implementation class.
Returns:
| Type | Description |
|---|---|
Type[EvidentlyDataValidator] |
The implementation class. |
logo_url: str
property
readonly
A url to represent the flavor in the dashboard.
Returns:
| Type | Description |
|---|---|
str |
The flavor logo. |
name: str
property
readonly
Name of the flavor.
Returns:
| Type | Description |
|---|---|
str |
The name of the flavor. |
sdk_docs_url: Optional[str]
property
readonly
A url to point at SDK docs explaining this flavor.
Returns:
| Type | Description |
|---|---|
Optional[str] |
A flavor SDK docs url. |
data_profiling(self, dataset, comparison_dataset=None, profile_list=None, column_mapping=None, report_options=[], download_nltk_data=False, **kwargs)
Analyze a dataset and generate a data report with Evidently.
The method takes in an optional list of Evidently options to be passed
to the report constructor (report_options). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option dataclass, the second is a dictionary of kwargs with the actual
option parameters, e.g.:
options = [
(
"evidently.options.ColorOptions",{
"primary_color": "#5a86ad",
"fill_color": "#fff4f2",
"zero_line_color": "#016795",
"current_data_color": "#c292a1",
"reference_data_color": "#017b92",
}
),
]
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
DataFrame |
Target dataset to be profiled. When a comparison dataset is provided, this dataset is considered the reference dataset. |
required |
comparison_dataset |
Optional[pandas.core.frame.DataFrame] |
Optional dataset to be used for data profiles that require a current dataset for comparison (e.g data drift profiles). |
None |
profile_list |
Optional[Sequence[zenml.integrations.evidently.metrics.EvidentlyMetricConfig]] |
List of Evidently metric configurations to be included in the report. If not provided, all available metric presets will be included. |
None |
column_mapping |
Optional[evidently.pipeline.column_mapping.ColumnMapping] |
Properties of the DataFrame columns used |
None |
report_options |
Sequence[Tuple[str, Dict[str, Any]]] |
List of Evidently options to be passed to the report constructor. |
[] |
download_nltk_data |
bool |
Whether to download NLTK data for text metrics. Defaults to False. |
False |
**kwargs |
Any |
Extra keyword arguments (unused). |
{} |
Returns:
| Type | Description |
|---|---|
evidently.report.Report |
The Evidently Report as JSON object and as HTML. |
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
def data_profiling(
self,
dataset: pd.DataFrame,
comparison_dataset: Optional[pd.DataFrame] = None,
profile_list: Optional[Sequence[EvidentlyMetricConfig]] = None,
column_mapping: Optional[ColumnMapping] = None,
report_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
download_nltk_data: bool = False,
**kwargs: Any,
) -> Report:
"""Analyze a dataset and generate a data report with Evidently.
The method takes in an optional list of Evidently options to be passed
to the report constructor (`report_options`). Each element in the list must be
composed of two items: the first is a full class path of an Evidently
option `dataclass`, the second is a dictionary of kwargs with the actual
option parameters, e.g.:
```python
options = [
(
"evidently.options.ColorOptions",{
"primary_color": "#5a86ad",
"fill_color": "#fff4f2",
"zero_line_color": "#016795",
"current_data_color": "#c292a1",
"reference_data_color": "#017b92",
}
),
]
```
Args:
dataset: Target dataset to be profiled. When a comparison dataset
is provided, this dataset is considered the reference dataset.
comparison_dataset: Optional dataset to be used for data profiles
that require a current dataset for comparison (e.g data drift
profiles).
profile_list: List of Evidently metric configurations to
be included in the report. If not provided, all available
metric presets will be included.
column_mapping: Properties of the DataFrame columns used
report_options: List of Evidently options to be passed to the
report constructor.
download_nltk_data: Whether to download NLTK data for text metrics.
Defaults to False.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Report as JSON object and as HTML.
"""
if download_nltk_data:
self._download_nltk_data()
profile_list = profile_list or EvidentlyMetricConfig.default_metrics()
metrics = [metric.to_evidently_metric() for metric in profile_list]
unpacked_report_options = self._unpack_options(report_options)
report = Report(metrics=metrics, options=unpacked_report_options)
report.run(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return report
data_validation(self, dataset, comparison_dataset=None, check_list=None, test_options=[], column_mapping=None, download_nltk_data=False, **kwargs)
Validate a dataset with Evidently.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
dataset |
Any |
Target dataset to be validated. |
required |
comparison_dataset |
Optional[Any] |
Optional dataset to be used for data validation that require a baseline for comparison (e.g data drift validation). |
None |
check_list |
Optional[Sequence[zenml.integrations.evidently.tests.EvidentlyTestConfig]] |
List of Evidently test configurations to be included in the test suite. If not provided, all available test presets will be included. |
None |
test_options |
Sequence[Tuple[str, Dict[str, Any]]] |
List of Evidently options to be passed to the test suite constructor. |
[] |
column_mapping |
Optional[evidently.pipeline.column_mapping.ColumnMapping] |
Properties of the DataFrame columns used |
None |
download_nltk_data |
bool |
Whether to download NLTK data for text tests. Defaults to False. |
False |
**kwargs |
Any |
Extra keyword arguments (unused). |
{} |
Returns:
| Type | Description |
|---|---|
evidently.test_suite.TestSuite |
The Evidently Test Suite as JSON object and as HTML. |
Source code in zenml/integrations/evidently/data_validators/evidently_data_validator.py
def data_validation(
self,
dataset: Any,
comparison_dataset: Optional[Any] = None,
check_list: Optional[Sequence[EvidentlyTestConfig]] = None,
test_options: Sequence[Tuple[str, Dict[str, Any]]] = [],
column_mapping: Optional[ColumnMapping] = None,
download_nltk_data: bool = False,
**kwargs: Any,
) -> TestSuite:
"""Validate a dataset with Evidently.
Args:
dataset: Target dataset to be validated.
comparison_dataset: Optional dataset to be used for data validation
that require a baseline for comparison (e.g data drift
validation).
check_list: List of Evidently test configurations to be
included in the test suite. If not provided, all available
test presets will be included.
test_options: List of Evidently options to be passed to the
test suite constructor.
column_mapping: Properties of the DataFrame columns used
download_nltk_data: Whether to download NLTK data for text tests.
Defaults to False.
**kwargs: Extra keyword arguments (unused).
Returns:
The Evidently Test Suite as JSON object and as HTML.
"""
if download_nltk_data:
self._download_nltk_data()
check_list = check_list or EvidentlyTestConfig.default_tests()
tests = [test.to_evidently_test() for test in check_list]
unpacked_test_options = self._unpack_options(test_options)
test_suite = TestSuite(tests=tests, options=unpacked_test_options)
test_suite.run(
reference_data=dataset,
current_data=comparison_dataset,
column_mapping=column_mapping,
)
return test_suite
flavors
special
Evidently integration flavors.
evidently_data_validator_flavor
Evidently data validator flavor.
EvidentlyDataValidatorFlavor (BaseDataValidatorFlavor)
Evidently data validator flavor.
Source code in zenml/integrations/evidently/flavors/evidently_data_validator_flavor.py
class EvidentlyDataValidatorFlavor(BaseDataValidatorFlavor):
"""Evidently data validator flavor."""
@property
def name(self) -> str:
"""Name of the flavor.
Returns:
The name of the flavor.
"""
return EVIDENTLY_DATA_VALIDATOR_FLAVOR
@property
def docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_docs_url()
@property
def sdk_docs_url(self) -> Optional[str]:
"""A url to point at SDK docs explaining this flavor.
Returns:
A flavor SDK docs url.
"""
return self.generate_default_sdk_docs_url()
@property
def logo_url(self) -> str:
"""A url to represent the flavor in the dashboard.
Returns:
The flavor logo.
"""
return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/data_validator/evidently.png"
@property
def implementation_class(self) -> Type["EvidentlyDataValidator"]:
"""Implementation class.
Returns:
The implementation class.
"""
from zenml.integrations.evidently.data_validators import (
EvidentlyDataValidator,
)
return EvidentlyDataValidator
docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
| Type | Description |
|---|---|
Optional[str] |
A flavor docs url. |
implementation_class: Type[EvidentlyDataValidator]
property
readonly
Implementation class.
Returns:
| Type | Description |
|---|---|
Type[EvidentlyDataValidator] |
The implementation class. |
logo_url: str
property
readonly
A url to represent the flavor in the dashboard.
Returns:
| Type | Description |
|---|---|
str |
The flavor logo. |
name: str
property
readonly
Name of the flavor.
Returns:
| Type | Description |
|---|---|
str |
The name of the flavor. |
sdk_docs_url: Optional[str]
property
readonly
A url to point at SDK docs explaining this flavor.
Returns:
| Type | Description |
|---|---|
Optional[str] |
A flavor SDK docs url. |
metrics
ZenML declarative representation of Evidently Metrics.
EvidentlyMetricConfig (BaseModel)
pydantic-model
Declarative Evidently Metric configuration.
This is a declarative representation of the configuration that goes into an Evidently Metric, MetricPreset or Metric generator instance. We need this to be able to store the configuration as part of a ZenML step parameter and later instantiate the Evidently Metric from it.
This representation covers all 3 possible ways of configuring an Evidently Metric or Metric-like object that can later be used in an Evidently Report:
- A Metric (derived from the Metric class).
- A MetricPreset (derived from the MetricPreset class).
- A column Metric generator (derived from the BaseGenerator class).
Ideally, it should be possible to just pass a Metric or Metric-like object to this class and have it automatically derive the configuration used to instantiate it. Unfortunately, this is not possible because the Evidently Metric classes are not designed in a way that allows us to extract the constructor parameters from them in a generic way.
Attributes:
| Name | Type | Description |
|---|---|---|
class_path |
str |
The full class path of the Evidently Metric class. |
parameters |
Dict[str, Any] |
The parameters of the Evidently Metric. |
is_generator |
bool |
Whether this is an Evidently column Metric generator. |
columns |
Union[str, List[str]] |
The columns that the Evidently column Metric generator is
applied to. Only used if |
skip_id_column |
bool |
Whether to skip the ID column when applying the
Evidently Metric generator. Only used if |
Source code in zenml/integrations/evidently/metrics.py
class EvidentlyMetricConfig(BaseModel):
"""Declarative Evidently Metric configuration.
This is a declarative representation of the configuration that goes into an
Evidently Metric, MetricPreset or Metric generator instance. We need this to
be able to store the configuration as part of a ZenML step parameter and
later instantiate the Evidently Metric from it.
This representation covers all 3 possible ways of configuring an Evidently
Metric or Metric-like object that can later be used in an Evidently Report:
1. A Metric (derived from the Metric class).
2. A MetricPreset (derived from the MetricPreset class).
3. A column Metric generator (derived from the BaseGenerator class).
Ideally, it should be possible to just pass a Metric or Metric-like
object to this class and have it automatically derive the configuration used
to instantiate it. Unfortunately, this is not possible because the Evidently
Metric classes are not designed in a way that allows us to extract the
constructor parameters from them in a generic way.
Attributes:
class_path: The full class path of the Evidently Metric class.
parameters: The parameters of the Evidently Metric.
is_generator: Whether this is an Evidently column Metric generator.
columns: The columns that the Evidently column Metric generator is
applied to. Only used if `generator` is True.
skip_id_column: Whether to skip the ID column when applying the
Evidently Metric generator. Only used if `generator` is True.
"""
class_path: str
parameters: Dict[str, Any] = Field(default_factory=dict)
is_generator: bool = False
columns: Optional[Union[str, List[str]]] = None
skip_id_column: bool = False
@staticmethod
def get_metric_class(metric_name: str) -> Union[Metric, MetricPreset]:
"""Get the Evidently metric or metric preset class from a string.
Args:
metric_name: The metric or metric preset class or full class
path.
Returns:
The Evidently metric or metric preset class.
Raises:
ValueError: If the name cannot be converted into a valid Evidently
metric or metric preset class.
"""
# First, try to interpret the metric name as a full class path.
if "." in metric_name:
try:
metric_class = source_utils.load(metric_name)
except (ImportError, AttributeError) as e:
raise ValueError(
f"Could not import Evidently Metric or MetricPreset "
f"`{metric_name}`: {str(e)}"
)
else:
# Next, try to interpret the metric as a Metric or MetricPreset
# class name
if hasattr(metrics, metric_name):
metric_class = getattr(metrics, metric_name)
elif hasattr(metric_preset, metric_name):
metric_class = getattr(metric_preset, metric_name)
else:
raise ValueError(
f"Could not import Evidently Metric or MetricPreset "
f"`{metric_name}`"
)
if not issubclass(metric_class, (Metric, MetricPreset)):
raise ValueError(
f"Class `{metric_name}` is not a valid Evidently "
f"Metric or MetricPreset."
)
return metric_class
@classmethod
def metric_generator(
cls,
metric: Union[Type[Metric], str],
columns: Optional[Union[str, List[str]]] = None,
skip_id_column: bool = False,
**parameters: Any,
) -> "EvidentlyMetricConfig":
"""Create a declarative configuration for an Evidently column Metric generator.
Call this method to get a declarative representation for the
configuration of an Evidently column Metric generator.
The `columns`, `skip_id_column` and `parameters` arguments will be
passed to the Evidently `generate_column_metrics` function:
- if `columns` is a list, it is interpreted as a list of column names.
- if `columns` is a string, it can be one of values:
- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class name
# and pass additional parameters
config = EvidentlyMetric.metric_generator(
"ColumnQuantileMetric", columns="num", quantile=0.5
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a full Metric class
# path
config = EvidentlyMetric.metric_generator(
"evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class
from evidently.metrics import ColumnDriftMetric
config = EvidentlyMetric.metric_generator(
ColumnDriftMetric, columns="all", skip_id_column=True
)
```
Args:
metric: The Evidently Metric class, class name or class path to use
for the generator.
columns: The columns to apply the generator to. Takes the same
values that the Evidently `generate_column_metrics` function
takes.
skip_id_column: Whether to skip the ID column when applying the
generator.
parameters: Additional optional parameters needed to instantiate the
Evidently Metric. These will be passed to the Evidently
`generate_column_metrics` function.
Returns:
The EvidentlyMetric declarative representation of the Evidently
Metric generator configuration.
Raises:
ValueError: If `metric` does not point to a valid Evidently Metric
or MetricPreset class.
"""
if isinstance(metric, str):
metric_class = cls.get_metric_class(metric)
elif issubclass(metric, (Metric, MetricPreset)):
metric_class = metric
else:
raise ValueError(f"Invalid Evidently Metric class: {metric}")
class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
config = cls(
class_path=class_path,
parameters=parameters,
columns=columns,
skip_id_column=skip_id_column,
is_generator=True,
)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_metric()
return config
@classmethod
def metric(
cls,
metric: Union[Type[Metric], Type[MetricPreset], str],
**parameters: Any,
) -> "EvidentlyMetricConfig":
"""Create a declarative configuration for an Evidently Metric.
Call this method to get a declarative representation for the
configuration of an Evidently Metric.
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its class name
config = EvidentlyMetric.metric("DataDriftPreset")
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its full class path
config = EvidentlyMetric.metric(
"evidently.metric_preset.DataDriftPreset"
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric using its class and pass additional
# parameters
from evidently.metrics import ColumnSummaryMetric
config = EvidentlyMetric.metric(
ColumnSummaryMetric, column_name="age"
)
```
Args:
metric: The Evidently Metric or MetricPreset class, class name or
class path.
parameters: Additional optional parameters needed to instantiate the
Evidently Metric or MetricPreset.
Returns:
The EvidentlyMetric declarative representation of the Evidently
Metric configuration.
Raises:
ValueError: If `metric` does not point to a valid Evidently Metric
or MetricPreset class.
"""
if isinstance(metric, str):
metric_class = cls.get_metric_class(metric)
elif issubclass(metric, (Metric, MetricPreset)):
metric_class = metric
else:
raise ValueError(
f"Invalid Evidently Metric or MetricPreset class: {metric}"
)
class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
config = cls(class_path=class_path, parameters=parameters)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_metric()
return config
@classmethod
def default_metrics(cls) -> List["EvidentlyMetricConfig"]:
"""Default Evidently metric configurations.
Call this to fetch a default list of Evidently metrics to use in cases
where no metrics are explicitly configured for a data validator.
All available Evidently MetricPreset classes are used, except for the
`TextOverviewPreset` which requires a text column, which we don't have
by default.
Returns:
A list of EvidentlyMetricConfig objects to use as default metrics.
"""
return [
cls.metric(metric=metric_preset_class_name)
for metric_preset_class_name in metric_preset.__all__
# TextOverviewPreset requires a text column, which we don't
# have by default
if metric_preset_class_name != "TextOverviewPreset"
]
def to_evidently_metric(
self,
) -> Union[Metric, MetricPreset, BaseGenerator]:
"""Create an Evidently Metric, MetricPreset or metric generator object.
Call this method to create an Evidently Metric, MetricPreset or metric
generator instance from its declarative representation.
Returns:
The Evidently Metric, MetricPreset or metric generator object.
Raises:
ValueError: If the Evidently Metric, MetricPreset or column metric
generator could not be instantiated.
"""
metric_class = self.get_metric_class(self.class_path)
if self.is_generator:
try:
return generate_column_metrics(
metric_class=metric_class,
columns=self.columns,
skip_id_column=self.skip_id_column,
parameters=self.parameters,
)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently column Metric generator "
f"`{self.class_path}`: {str(e)}"
)
try:
return metric_class(**self.parameters)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently Metric or MetricPreset "
f"`{self.class_path}`: {str(e)}"
)
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/metrics.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_metrics()
classmethod
Default Evidently metric configurations.
Call this to fetch a default list of Evidently metrics to use in cases
where no metrics are explicitly configured for a data validator.
All available Evidently MetricPreset classes are used, except for the
TextOverviewPreset which requires a text column, which we don't have
by default.
Returns:
| Type | Description |
|---|---|
List[EvidentlyMetricConfig] |
A list of EvidentlyMetricConfig objects to use as default metrics. |
Source code in zenml/integrations/evidently/metrics.py
@classmethod
def default_metrics(cls) -> List["EvidentlyMetricConfig"]:
"""Default Evidently metric configurations.
Call this to fetch a default list of Evidently metrics to use in cases
where no metrics are explicitly configured for a data validator.
All available Evidently MetricPreset classes are used, except for the
`TextOverviewPreset` which requires a text column, which we don't have
by default.
Returns:
A list of EvidentlyMetricConfig objects to use as default metrics.
"""
return [
cls.metric(metric=metric_preset_class_name)
for metric_preset_class_name in metric_preset.__all__
# TextOverviewPreset requires a text column, which we don't
# have by default
if metric_preset_class_name != "TextOverviewPreset"
]
get_metric_class(metric_name)
staticmethod
Get the Evidently metric or metric preset class from a string.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
metric_name |
str |
The metric or metric preset class or full class path. |
required |
Returns:
| Type | Description |
|---|---|
Union[evidently.metrics.base_metric.Metric, evidently.metric_preset.metric_preset.MetricPreset] |
The Evidently metric or metric preset class. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the name cannot be converted into a valid Evidently metric or metric preset class. |
Source code in zenml/integrations/evidently/metrics.py
@staticmethod
def get_metric_class(metric_name: str) -> Union[Metric, MetricPreset]:
"""Get the Evidently metric or metric preset class from a string.
Args:
metric_name: The metric or metric preset class or full class
path.
Returns:
The Evidently metric or metric preset class.
Raises:
ValueError: If the name cannot be converted into a valid Evidently
metric or metric preset class.
"""
# First, try to interpret the metric name as a full class path.
if "." in metric_name:
try:
metric_class = source_utils.load(metric_name)
except (ImportError, AttributeError) as e:
raise ValueError(
f"Could not import Evidently Metric or MetricPreset "
f"`{metric_name}`: {str(e)}"
)
else:
# Next, try to interpret the metric as a Metric or MetricPreset
# class name
if hasattr(metrics, metric_name):
metric_class = getattr(metrics, metric_name)
elif hasattr(metric_preset, metric_name):
metric_class = getattr(metric_preset, metric_name)
else:
raise ValueError(
f"Could not import Evidently Metric or MetricPreset "
f"`{metric_name}`"
)
if not issubclass(metric_class, (Metric, MetricPreset)):
raise ValueError(
f"Class `{metric_name}` is not a valid Evidently "
f"Metric or MetricPreset."
)
return metric_class
metric(metric, **parameters)
classmethod
Create a declarative configuration for an Evidently Metric.
Call this method to get a declarative representation for the configuration of an Evidently Metric.
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its class name
config = EvidentlyMetric.metric("DataDriftPreset")
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its full class path
config = EvidentlyMetric.metric(
"evidently.metric_preset.DataDriftPreset"
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric using its class and pass additional
# parameters
from evidently.metrics import ColumnSummaryMetric
config = EvidentlyMetric.metric(
ColumnSummaryMetric, column_name="age"
)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
metric |
Union[Type[evidently.metrics.base_metric.Metric], Type[evidently.metric_preset.metric_preset.MetricPreset], str] |
The Evidently Metric or MetricPreset class, class name or class path. |
required |
parameters |
Any |
Additional optional parameters needed to instantiate the Evidently Metric or MetricPreset. |
{} |
Returns:
| Type | Description |
|---|---|
EvidentlyMetricConfig |
The EvidentlyMetric declarative representation of the Evidently Metric configuration. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in zenml/integrations/evidently/metrics.py
@classmethod
def metric(
cls,
metric: Union[Type[Metric], Type[MetricPreset], str],
**parameters: Any,
) -> "EvidentlyMetricConfig":
"""Create a declarative configuration for an Evidently Metric.
Call this method to get a declarative representation for the
configuration of an Evidently Metric.
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its class name
config = EvidentlyMetric.metric("DataDriftPreset")
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently MetricPreset using its full class path
config = EvidentlyMetric.metric(
"evidently.metric_preset.DataDriftPreset"
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric using its class and pass additional
# parameters
from evidently.metrics import ColumnSummaryMetric
config = EvidentlyMetric.metric(
ColumnSummaryMetric, column_name="age"
)
```
Args:
metric: The Evidently Metric or MetricPreset class, class name or
class path.
parameters: Additional optional parameters needed to instantiate the
Evidently Metric or MetricPreset.
Returns:
The EvidentlyMetric declarative representation of the Evidently
Metric configuration.
Raises:
ValueError: If `metric` does not point to a valid Evidently Metric
or MetricPreset class.
"""
if isinstance(metric, str):
metric_class = cls.get_metric_class(metric)
elif issubclass(metric, (Metric, MetricPreset)):
metric_class = metric
else:
raise ValueError(
f"Invalid Evidently Metric or MetricPreset class: {metric}"
)
class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
config = cls(class_path=class_path, parameters=parameters)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_metric()
return config
metric_generator(metric, columns=None, skip_id_column=False, **parameters)
classmethod
Create a declarative configuration for an Evidently column Metric generator.
Call this method to get a declarative representation for the configuration of an Evidently column Metric generator.
The columns, skip_id_column and parameters arguments will be
passed to the Evidently generate_column_metrics function:
- if
columnsis a list, it is interpreted as a list of column names. - if
columnsis a string, it can be one of values:- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class name
# and pass additional parameters
config = EvidentlyMetric.metric_generator(
"ColumnQuantileMetric", columns="num", quantile=0.5
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a full Metric class
# path
config = EvidentlyMetric.metric_generator(
"evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
)
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class
from evidently.metrics import ColumnDriftMetric
config = EvidentlyMetric.metric_generator(
ColumnDriftMetric, columns="all", skip_id_column=True
)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
metric |
Union[Type[evidently.metrics.base_metric.Metric], str] |
The Evidently Metric class, class name or class path to use for the generator. |
required |
columns |
Union[str, List[str]] |
The columns to apply the generator to. Takes the same
values that the Evidently |
None |
skip_id_column |
bool |
Whether to skip the ID column when applying the generator. |
False |
parameters |
Any |
Additional optional parameters needed to instantiate the
Evidently Metric. These will be passed to the Evidently
|
{} |
Returns:
| Type | Description |
|---|---|
EvidentlyMetricConfig |
The EvidentlyMetric declarative representation of the Evidently Metric generator configuration. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in zenml/integrations/evidently/metrics.py
@classmethod
def metric_generator(
cls,
metric: Union[Type[Metric], str],
columns: Optional[Union[str, List[str]]] = None,
skip_id_column: bool = False,
**parameters: Any,
) -> "EvidentlyMetricConfig":
"""Create a declarative configuration for an Evidently column Metric generator.
Call this method to get a declarative representation for the
configuration of an Evidently column Metric generator.
The `columns`, `skip_id_column` and `parameters` arguments will be
passed to the Evidently `generate_column_metrics` function:
- if `columns` is a list, it is interpreted as a list of column names.
- if `columns` is a string, it can be one of values:
- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class name
# and pass additional parameters
config = EvidentlyMetric.metric_generator(
"ColumnQuantileMetric", columns="num", quantile=0.5
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a full Metric class
# path
config = EvidentlyMetric.metric_generator(
"evidently.metrics.ColumnSummaryMetric", columns=["age", "name"]
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyMetric
# Configure an Evidently Metric generator using a Metric class
from evidently.metrics import ColumnDriftMetric
config = EvidentlyMetric.metric_generator(
ColumnDriftMetric, columns="all", skip_id_column=True
)
```
Args:
metric: The Evidently Metric class, class name or class path to use
for the generator.
columns: The columns to apply the generator to. Takes the same
values that the Evidently `generate_column_metrics` function
takes.
skip_id_column: Whether to skip the ID column when applying the
generator.
parameters: Additional optional parameters needed to instantiate the
Evidently Metric. These will be passed to the Evidently
`generate_column_metrics` function.
Returns:
The EvidentlyMetric declarative representation of the Evidently
Metric generator configuration.
Raises:
ValueError: If `metric` does not point to a valid Evidently Metric
or MetricPreset class.
"""
if isinstance(metric, str):
metric_class = cls.get_metric_class(metric)
elif issubclass(metric, (Metric, MetricPreset)):
metric_class = metric
else:
raise ValueError(f"Invalid Evidently Metric class: {metric}")
class_path = f"{metric_class.__module__}." f"{metric_class.__name__}"
config = cls(
class_path=class_path,
parameters=parameters,
columns=columns,
skip_id_column=skip_id_column,
is_generator=True,
)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_metric()
return config
to_evidently_metric(self)
Create an Evidently Metric, MetricPreset or metric generator object.
Call this method to create an Evidently Metric, MetricPreset or metric generator instance from its declarative representation.
Returns:
| Type | Description |
|---|---|
Union[evidently.metrics.base_metric.Metric, evidently.metric_preset.metric_preset.MetricPreset, evidently.utils.generators.BaseGenerator] |
The Evidently Metric, MetricPreset or metric generator object. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the Evidently Metric, MetricPreset or column metric generator could not be instantiated. |
Source code in zenml/integrations/evidently/metrics.py
def to_evidently_metric(
self,
) -> Union[Metric, MetricPreset, BaseGenerator]:
"""Create an Evidently Metric, MetricPreset or metric generator object.
Call this method to create an Evidently Metric, MetricPreset or metric
generator instance from its declarative representation.
Returns:
The Evidently Metric, MetricPreset or metric generator object.
Raises:
ValueError: If the Evidently Metric, MetricPreset or column metric
generator could not be instantiated.
"""
metric_class = self.get_metric_class(self.class_path)
if self.is_generator:
try:
return generate_column_metrics(
metric_class=metric_class,
columns=self.columns,
skip_id_column=self.skip_id_column,
parameters=self.parameters,
)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently column Metric generator "
f"`{self.class_path}`: {str(e)}"
)
try:
return metric_class(**self.parameters)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently Metric or MetricPreset "
f"`{self.class_path}`: {str(e)}"
)
steps
special
Initialization of the Evidently Standard Steps.
evidently_report
Implementation of the Evidently Report Step.
evidently_test
Implementation of the Evidently Test Step.
tests
ZenML declarative representation of Evidently Tests.
EvidentlyTestConfig (BaseModel)
pydantic-model
Declarative Evidently Test configuration.
This is a declarative representation of the configuration that goes into an Evidently Test, TestPreset or Test generator instance. We need this to be able to store the configuration as part of a ZenML step parameter and later instantiate the Evidently Test from it.
This representation covers all 3 possible ways of configuring an Evidently Test or Test-like object that can later be used in an Evidently TestSuite:
- A Test (derived from the Test class).
- A TestPreset (derived from the TestPreset class).
- A column Test generator (derived from the BaseGenerator class).
Ideally, it should be possible to just pass a Test or Test-like object to this class and have it automatically derive the configuration used to instantiate it. Unfortunately, this is not possible because the Evidently Test classes are not designed in a way that allows us to extract the constructor parameters from them in a generic way.
Attributes:
| Name | Type | Description |
|---|---|---|
class_path |
str |
The full class path of the Evidently Test class. |
parameters |
Dict[str, Any] |
The parameters of the Evidently Test. |
is_generator |
bool |
Whether this is an Evidently column Test generator. |
columns |
Union[str, List[str]] |
The columns that the Evidently column Test generator is
applied to. Only used if |
Source code in zenml/integrations/evidently/tests.py
class EvidentlyTestConfig(BaseModel):
"""Declarative Evidently Test configuration.
This is a declarative representation of the configuration that goes into an
Evidently Test, TestPreset or Test generator instance. We need this to
be able to store the configuration as part of a ZenML step parameter and
later instantiate the Evidently Test from it.
This representation covers all 3 possible ways of configuring an Evidently
Test or Test-like object that can later be used in an Evidently TestSuite:
1. A Test (derived from the Test class).
2. A TestPreset (derived from the TestPreset class).
3. A column Test generator (derived from the BaseGenerator class).
Ideally, it should be possible to just pass a Test or Test-like
object to this class and have it automatically derive the configuration used
to instantiate it. Unfortunately, this is not possible because the Evidently
Test classes are not designed in a way that allows us to extract the
constructor parameters from them in a generic way.
Attributes:
class_path: The full class path of the Evidently Test class.
parameters: The parameters of the Evidently Test.
is_generator: Whether this is an Evidently column Test generator.
columns: The columns that the Evidently column Test generator is
applied to. Only used if `generator` is True.
"""
class_path: str
parameters: Dict[str, Any] = Field(default_factory=dict)
is_generator: bool = False
columns: Optional[Union[str, List[str]]] = None
@staticmethod
def get_test_class(test_name: str) -> Union[Test, TestPreset]:
"""Get the Evidently test or test preset class from a string.
Args:
test_name: The test or test preset class or full class
path.
Returns:
The Evidently test or test preset class.
Raises:
ValueError: If the name cannot be converted into a valid Evidently
test or test preset class.
"""
# First, try to interpret the test name as a full class path.
if "." in test_name:
try:
test_class = source_utils.load(test_name)
except (ImportError, AttributeError) as e:
raise ValueError(
f"Could not import Evidently Test or TestPreset "
f"`{test_name}`: {str(e)}"
)
else:
# Next, try to interpret the test as a Test or TestPreset
# class name
if hasattr(tests, test_name):
test_class = getattr(tests, test_name)
elif hasattr(test_preset, test_name):
test_class = getattr(test_preset, test_name)
else:
raise ValueError(
f"Could not import Evidently Test or TestPreset "
f"`{test_name}`"
)
if not issubclass(test_class, (Test, TestPreset)):
raise ValueError(
f"Class `{test_name}` is not a valid Evidently "
f"Test or TestPreset."
)
return test_class
@classmethod
def test_generator(
cls,
test: Union[Type[Test], str],
columns: Optional[Union[str, List[str]]] = None,
**parameters: Any,
) -> "EvidentlyTestConfig":
"""Create a declarative configuration for an Evidently column Test generator.
Call this method to get a declarative representation for the
configuration of an Evidently column Test generator.
The `columns`, `parameters` arguments will be
passed to the Evidently `generate_column_tests` function:
- if `columns` is a list, it is interpreted as a list of column names.
- if `columns` is a string, it can be one of values:
- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class name
# and pass additional parameters
config = EvidentlyTest.test_generator(
"TestColumnValueMin", columns="num", gt=0.5
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a full Test class
# path
config = EvidentlyTest.test_generator(
"evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class
from evidently.tests import TestColumnQuantile
config = EvidentlyTest.test_generator(
TestColumnQuantile, columns="all", quantile=0.5
)
```
Args:
test: The Evidently Test class, class name or class path to use
for the generator.
columns: The columns to apply the generator to. Takes the same
values that the Evidently `generate_column_tests` function
takes.
parameters: Additional optional parameters needed to instantiate the
Evidently Test. These will be passed to the Evidently
`generate_column_tests` function.
Returns:
The EvidentlyTest declarative representation of the Evidently
Test generator configuration.
Raises:
ValueError: If `test` does not point to a valid Evidently Test
or TestPreset class.
"""
if isinstance(test, str):
test_class = cls.get_test_class(test)
elif issubclass(test, (Test, TestPreset)):
test_class = test
else:
raise ValueError(f"Invalid Evidently Test class: {test}")
class_path = f"{test_class.__module__}." f"{test_class.__name__}"
config = cls(
class_path=class_path,
parameters=parameters,
columns=columns,
is_generator=True,
)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_test()
return config
@classmethod
def test(
cls,
test: Union[Type[Test], Type[TestPreset], str],
**parameters: Any,
) -> "EvidentlyTestConfig":
"""Create a declarative configuration for an Evidently Test.
Call this method to get a declarative representation for the
configuration of an Evidently Test.
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its class name
config = EvidentlyTest.test("DataDriftPreset")
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its full class path
config = EvidentlyTest.test(
"evidently.test_preset.DataDriftPreset"
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test using its class and pass additional
# parameters
from evidently.tests import ColumnSummaryTest
config = EvidentlyTest.test(
ColumnSummaryTest, column_name="age"
)
```
Args:
test: The Evidently Test or TestPreset class, class name or
class path.
parameters: Additional optional parameters needed to instantiate the
Evidently Test or TestPreset.
Returns:
The EvidentlyTest declarative representation of the Evidently
Test configuration.
Raises:
ValueError: If `test` does not point to a valid Evidently Test
or TestPreset class.
"""
if isinstance(test, str):
test_class = cls.get_test_class(test)
elif issubclass(test, (Test, TestPreset)):
test_class = test
else:
raise ValueError(
f"Invalid Evidently Test or TestPreset class: {test}"
)
class_path = f"{test_class.__module__}." f"{test_class.__name__}"
config = cls(class_path=class_path, parameters=parameters)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_test()
return config
@classmethod
def default_tests(cls) -> List["EvidentlyTestConfig"]:
"""Default Evidently test configurations.
Call this to fetch a default list of Evidently tests to use in cases
where no tests are explicitly configured for a data validator.
All available Evidently TestPreset classes are used.
Returns:
A list of EvidentlyTestConfig objects to use as default tests.
"""
return [
cls.test(test=test_preset_class_name)
for test_preset_class_name in test_preset.__all__
]
def to_evidently_test(self) -> Union[Test, TestPreset, BaseGenerator]:
"""Create an Evidently Test, TestPreset or test generator object.
Call this method to create an Evidently Test, TestPreset or test
generator instance from its declarative representation.
Returns:
The Evidently Test, TestPreset or test generator object.
Raises:
ValueError: If the Evidently Test, TestPreset or column test
generator could not be instantiated.
"""
test_class = self.get_test_class(self.class_path)
if self.is_generator:
try:
return generate_column_tests(
test_class=test_class,
columns=self.columns,
parameters=self.parameters,
)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently column Test generator "
f"`{self.class_path}`: {str(e)}"
)
try:
return test_class(**self.parameters)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently Test or TestPreset "
f"`{self.class_path}`: {str(e)}"
)
class Config:
"""Pydantic config class."""
extra = "forbid"
Config
Pydantic config class.
Source code in zenml/integrations/evidently/tests.py
class Config:
"""Pydantic config class."""
extra = "forbid"
default_tests()
classmethod
Default Evidently test configurations.
Call this to fetch a default list of Evidently tests to use in cases where no tests are explicitly configured for a data validator. All available Evidently TestPreset classes are used.
Returns:
| Type | Description |
|---|---|
List[EvidentlyTestConfig] |
A list of EvidentlyTestConfig objects to use as default tests. |
Source code in zenml/integrations/evidently/tests.py
@classmethod
def default_tests(cls) -> List["EvidentlyTestConfig"]:
"""Default Evidently test configurations.
Call this to fetch a default list of Evidently tests to use in cases
where no tests are explicitly configured for a data validator.
All available Evidently TestPreset classes are used.
Returns:
A list of EvidentlyTestConfig objects to use as default tests.
"""
return [
cls.test(test=test_preset_class_name)
for test_preset_class_name in test_preset.__all__
]
get_test_class(test_name)
staticmethod
Get the Evidently test or test preset class from a string.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
test_name |
str |
The test or test preset class or full class path. |
required |
Returns:
| Type | Description |
|---|---|
Union[evidently.tests.base_test.Test, evidently.test_preset.test_preset.TestPreset] |
The Evidently test or test preset class. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the name cannot be converted into a valid Evidently test or test preset class. |
Source code in zenml/integrations/evidently/tests.py
@staticmethod
def get_test_class(test_name: str) -> Union[Test, TestPreset]:
"""Get the Evidently test or test preset class from a string.
Args:
test_name: The test or test preset class or full class
path.
Returns:
The Evidently test or test preset class.
Raises:
ValueError: If the name cannot be converted into a valid Evidently
test or test preset class.
"""
# First, try to interpret the test name as a full class path.
if "." in test_name:
try:
test_class = source_utils.load(test_name)
except (ImportError, AttributeError) as e:
raise ValueError(
f"Could not import Evidently Test or TestPreset "
f"`{test_name}`: {str(e)}"
)
else:
# Next, try to interpret the test as a Test or TestPreset
# class name
if hasattr(tests, test_name):
test_class = getattr(tests, test_name)
elif hasattr(test_preset, test_name):
test_class = getattr(test_preset, test_name)
else:
raise ValueError(
f"Could not import Evidently Test or TestPreset "
f"`{test_name}`"
)
if not issubclass(test_class, (Test, TestPreset)):
raise ValueError(
f"Class `{test_name}` is not a valid Evidently "
f"Test or TestPreset."
)
return test_class
test(test, **parameters)
classmethod
Create a declarative configuration for an Evidently Test.
Call this method to get a declarative representation for the configuration of an Evidently Test.
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its class name
config = EvidentlyTest.test("DataDriftPreset")
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its full class path
config = EvidentlyTest.test(
"evidently.test_preset.DataDriftPreset"
)
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test using its class and pass additional
# parameters
from evidently.tests import ColumnSummaryTest
config = EvidentlyTest.test(
ColumnSummaryTest, column_name="age"
)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
test |
Union[Type[evidently.tests.base_test.Test], Type[evidently.test_preset.test_preset.TestPreset], str] |
The Evidently Test or TestPreset class, class name or class path. |
required |
parameters |
Any |
Additional optional parameters needed to instantiate the Evidently Test or TestPreset. |
{} |
Returns:
| Type | Description |
|---|---|
EvidentlyTestConfig |
The EvidentlyTest declarative representation of the Evidently Test configuration. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in zenml/integrations/evidently/tests.py
@classmethod
def test(
cls,
test: Union[Type[Test], Type[TestPreset], str],
**parameters: Any,
) -> "EvidentlyTestConfig":
"""Create a declarative configuration for an Evidently Test.
Call this method to get a declarative representation for the
configuration of an Evidently Test.
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its class name
config = EvidentlyTest.test("DataDriftPreset")
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently TestPreset using its full class path
config = EvidentlyTest.test(
"evidently.test_preset.DataDriftPreset"
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test using its class and pass additional
# parameters
from evidently.tests import ColumnSummaryTest
config = EvidentlyTest.test(
ColumnSummaryTest, column_name="age"
)
```
Args:
test: The Evidently Test or TestPreset class, class name or
class path.
parameters: Additional optional parameters needed to instantiate the
Evidently Test or TestPreset.
Returns:
The EvidentlyTest declarative representation of the Evidently
Test configuration.
Raises:
ValueError: If `test` does not point to a valid Evidently Test
or TestPreset class.
"""
if isinstance(test, str):
test_class = cls.get_test_class(test)
elif issubclass(test, (Test, TestPreset)):
test_class = test
else:
raise ValueError(
f"Invalid Evidently Test or TestPreset class: {test}"
)
class_path = f"{test_class.__module__}." f"{test_class.__name__}"
config = cls(class_path=class_path, parameters=parameters)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_test()
return config
test_generator(test, columns=None, **parameters)
classmethod
Create a declarative configuration for an Evidently column Test generator.
Call this method to get a declarative representation for the configuration of an Evidently column Test generator.
The columns, parameters arguments will be
passed to the Evidently generate_column_tests function:
- if
columnsis a list, it is interpreted as a list of column names. - if
columnsis a string, it can be one of values:- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class name
# and pass additional parameters
config = EvidentlyTest.test_generator(
"TestColumnValueMin", columns="num", gt=0.5
)
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a full Test class
# path
config = EvidentlyTest.test_generator(
"evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
)
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class
from evidently.tests import TestColumnQuantile
config = EvidentlyTest.test_generator(
TestColumnQuantile, columns="all", quantile=0.5
)
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
test |
Union[Type[evidently.tests.base_test.Test], str] |
The Evidently Test class, class name or class path to use for the generator. |
required |
columns |
Union[str, List[str]] |
The columns to apply the generator to. Takes the same
values that the Evidently |
None |
parameters |
Any |
Additional optional parameters needed to instantiate the
Evidently Test. These will be passed to the Evidently
|
{} |
Returns:
| Type | Description |
|---|---|
EvidentlyTestConfig |
The EvidentlyTest declarative representation of the Evidently Test generator configuration. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If |
Source code in zenml/integrations/evidently/tests.py
@classmethod
def test_generator(
cls,
test: Union[Type[Test], str],
columns: Optional[Union[str, List[str]]] = None,
**parameters: Any,
) -> "EvidentlyTestConfig":
"""Create a declarative configuration for an Evidently column Test generator.
Call this method to get a declarative representation for the
configuration of an Evidently column Test generator.
The `columns`, `parameters` arguments will be
passed to the Evidently `generate_column_tests` function:
- if `columns` is a list, it is interpreted as a list of column names.
- if `columns` is a string, it can be one of values:
- "all" - use all columns, including target/prediction columns
- "num" - for numeric features
- "cat" - for category features
- "text" - for text features
- "features" - for all features, not target/prediction columns.
- a None value is the same as "all".
Some examples
-------------
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class name
# and pass additional parameters
config = EvidentlyTest.test_generator(
"TestColumnValueMin", columns="num", gt=0.5
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a full Test class
# path
config = EvidentlyTest.test_generator(
"evidently.tests.TestColumnShareOfMissingValues", columns=["age", "name"]
)
```
```python
from zenml.integrations.evidently.data_validators import EvidentlyTest
# Configure an Evidently Test generator using a Test class
from evidently.tests import TestColumnQuantile
config = EvidentlyTest.test_generator(
TestColumnQuantile, columns="all", quantile=0.5
)
```
Args:
test: The Evidently Test class, class name or class path to use
for the generator.
columns: The columns to apply the generator to. Takes the same
values that the Evidently `generate_column_tests` function
takes.
parameters: Additional optional parameters needed to instantiate the
Evidently Test. These will be passed to the Evidently
`generate_column_tests` function.
Returns:
The EvidentlyTest declarative representation of the Evidently
Test generator configuration.
Raises:
ValueError: If `test` does not point to a valid Evidently Test
or TestPreset class.
"""
if isinstance(test, str):
test_class = cls.get_test_class(test)
elif issubclass(test, (Test, TestPreset)):
test_class = test
else:
raise ValueError(f"Invalid Evidently Test class: {test}")
class_path = f"{test_class.__module__}." f"{test_class.__name__}"
config = cls(
class_path=class_path,
parameters=parameters,
columns=columns,
is_generator=True,
)
# Try to instantiate the configuration to check if the parameters are
# valid
config.to_evidently_test()
return config
to_evidently_test(self)
Create an Evidently Test, TestPreset or test generator object.
Call this method to create an Evidently Test, TestPreset or test generator instance from its declarative representation.
Returns:
| Type | Description |
|---|---|
Union[evidently.tests.base_test.Test, evidently.test_preset.test_preset.TestPreset, evidently.utils.generators.BaseGenerator] |
The Evidently Test, TestPreset or test generator object. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the Evidently Test, TestPreset or column test generator could not be instantiated. |
Source code in zenml/integrations/evidently/tests.py
def to_evidently_test(self) -> Union[Test, TestPreset, BaseGenerator]:
"""Create an Evidently Test, TestPreset or test generator object.
Call this method to create an Evidently Test, TestPreset or test
generator instance from its declarative representation.
Returns:
The Evidently Test, TestPreset or test generator object.
Raises:
ValueError: If the Evidently Test, TestPreset or column test
generator could not be instantiated.
"""
test_class = self.get_test_class(self.class_path)
if self.is_generator:
try:
return generate_column_tests(
test_class=test_class,
columns=self.columns,
parameters=self.parameters,
)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently column Test generator "
f"`{self.class_path}`: {str(e)}"
)
try:
return test_class(**self.parameters)
except Exception as e:
raise ValueError(
f"Could not instantiate Evidently Test or TestPreset "
f"`{self.class_path}`: {str(e)}"
)