Skip to content

Data Validators

Data validators are stack components responsible for data profiling and validation.

BaseDataValidator

Bases: StackComponent

Base class for all ZenML data validators.

Source code in src/zenml/data_validators/base_data_validator.py 
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
class BaseDataValidator(StackComponent):
    """Base class for all ZenML data validators."""

    NAME: ClassVar[str]
    FLAVOR: ClassVar[Type["BaseDataValidatorFlavor"]]

    @property
    def config(self) -> BaseDataValidatorConfig:
        """Returns the config of this data validator.

        Returns:
            The config of this data validator.
        """
        return cast(BaseDataValidatorConfig, self._config)

    @classmethod
    def get_active_data_validator(cls) -> "BaseDataValidator":
        """Get the data validator registered in the active stack.

        Returns:
            The data validator registered in the active stack.

        Raises:
            TypeError: if a data validator is not part of the
                active stack.
        """
        flavor: BaseDataValidatorFlavor = cls.FLAVOR()
        client = Client()
        data_validator = client.active_stack.data_validator
        if not data_validator or not isinstance(data_validator, cls):
            raise TypeError(
                f"The active stack needs to have a {cls.NAME} data "
                f"validator component registered to be able to run data validation "
                f"actions with {cls.NAME}. You can create a new stack with "
                f"a {cls.NAME} data validator component or update your "
                f"active stack to add this component, e.g.:\n\n"
                f"  `zenml data-validator register {flavor.name} "
                f"--flavor={flavor.name} ...`\n"
                f"  `zenml stack register <STACK-NAME> -dv {flavor.name} ...`\n"
                f"  or:\n"
                f"  `zenml stack update -dv {flavor.name}`\n\n"
            )

        return data_validator

    def data_profiling(
        self,
        dataset: Any,
        comparison_dataset: Optional[Any] = None,
        profile_list: Optional[Sequence[Any]] = None,
        **kwargs: Any,
    ) -> Any:
        """Analyze one or more datasets and generate a data profile.

        This method should be implemented by data validators that support
        analyzing a dataset and generating a data profile (e.g. schema,
        statistical summary, data distribution profile, validation
        rules, data drift reports etc.).
        The method should return a data profile object.

        This method also accepts an optional second dataset argument to
        accommodate different categories of data profiling, e.g.:

        * profiles generated from a single dataset: schema inference, validation
        rules inference, statistical profiles, data integrity reports
        * differential profiles that need a second dataset for comparison:
        differential statistical profiles, data drift reports

        Data validators that support generating multiple categories of data
        profiles should also take in a `profile_list` argument that lists the
        subset of profiles to be generated. If not supplied, the behavior is
        implementation specific, but it is recommended to provide a good default
        (e.g. a single default data profile type may be generated and returned,
        or all available data profiles may be generated and returned as a single
        result).

        Args:
            dataset: Target dataset to be profiled.
            comparison_dataset: Optional second dataset to be used for data
                comparison profiles (e.g data drift reports).
            profile_list: Optional list identifying the categories of data
                profiles to be generated.
            **kwargs: Implementation specific keyword arguments.

        Raises:
            NotImplementedError: if data profiling is not supported by this
                data validator.
        """
        raise NotImplementedError(
            f"Data profiling is not supported by the {self.__class__} data "
            f"validator."
        )

    def data_validation(
        self,
        dataset: Any,
        comparison_dataset: Optional[Any] = None,
        check_list: Optional[Sequence[Any]] = None,
        **kwargs: Any,
    ) -> Any:
        """Run data validation checks on a dataset.

        This method should be implemented by data validators that support
        running data quality checks an input dataset (e.g. data integrity
        checks, data drift checks).

        This method also accepts an optional second dataset argument to
        accommodate different categories of data validation tests, e.g.:

        * single dataset checks: data integrity checks (e.g. missing
        values, conflicting labels, mixed data types etc.)
        * checks that compare two datasets: data drift checks (e.g. new labels,
        feature drift, label drift etc.)

        Data validators that support running multiple categories of data
        integrity checks should also take in a `check_list` argument that
        lists the subset of checks to be performed. If not supplied, the
        behavior is implementation specific, but it is recommended to provide a
        good default (e.g. a single default validation check may be performed,
        or all available validation checks may be performed and their results
        returned as a list of objects).

        Args:
            dataset: Target dataset to be validated.
            comparison_dataset: Optional second dataset to be used for data
                comparison checks (e.g data drift checks).
            check_list: Optional list identifying the data checks to
                be performed.
            **kwargs: Implementation specific keyword arguments.

        Raises:
            NotImplementedError: if data validation is not
                supported by this data validator.
        """
        raise NotImplementedError(
            f"Data validation not implemented for {self}."
        )

    def model_validation(
        self,
        dataset: Any,
        model: Any,
        comparison_dataset: Optional[Any] = None,
        check_list: Optional[Sequence[Any]] = None,
        **kwargs: Any,
    ) -> Any:
        """Run model validation checks.

        This method should be implemented by data validators that support
        running model validation checks (e.g. confusion matrix validation,
        performance reports, model error analyzes, etc).

        Unlike `data_validation`, model validation checks require that a model
        be present as an active component during the validation process.

        This method also accepts an optional second dataset argument to
        accommodate different categories of data validation tests, e.g.:

        * single dataset tests: confusion matrix validation,
        performance reports, model error analyzes, etc
        * model comparison tests: tests that identify changes in a model
        behavior by comparing how it performs on two different datasets.

        Data validators that support running multiple categories of model
        validation checks should also take in a `check_list` argument that
        lists the subset of checks to be performed. If not supplied, the
        behavior is implementation specific, but it is recommended to provide a
        good default (e.g. a single default validation check may be performed,
        or all available validation checks may be performed and their results
        returned as a list of objects).

        Args:
            dataset: Target dataset to be validated.
            model: Target model to be validated.
            comparison_dataset: Optional second dataset to be used for model
                comparison checks (e.g model performance comparison checks).
            check_list: Optional list identifying the model validation checks to
                be performed.
            **kwargs: Implementation specific keyword arguments.

        Raises:
            NotImplementedError: if model validation is not supported by this
                data validator.
        """
        raise NotImplementedError(
            f"Model validation not implemented for {self}."
        )

config property

Returns the config of this data validator.

Returns:

Type Description
BaseDataValidatorConfig

The config of this data validator.

data_profiling(dataset, comparison_dataset=None, profile_list=None, **kwargs)

Analyze one or more datasets and generate a data profile.

This method should be implemented by data validators that support analyzing a dataset and generating a data profile (e.g. schema, statistical summary, data distribution profile, validation rules, data drift reports etc.). The method should return a data profile object.

This method also accepts an optional second dataset argument to accommodate different categories of data profiling, e.g.:

  • profiles generated from a single dataset: schema inference, validation rules inference, statistical profiles, data integrity reports
  • differential profiles that need a second dataset for comparison: differential statistical profiles, data drift reports

Data validators that support generating multiple categories of data profiles should also take in a profile_list argument that lists the subset of profiles to be generated. If not supplied, the behavior is implementation specific, but it is recommended to provide a good default (e.g. a single default data profile type may be generated and returned, or all available data profiles may be generated and returned as a single result).

Parameters:

Name Type Description Default
dataset Any

Target dataset to be profiled.

 required
comparison_dataset Optional[Any]

Optional second dataset to be used for data comparison profiles (e.g data drift reports).

 None
profile_list Optional[Sequence[Any]]

Optional list identifying the categories of data profiles to be generated.

 None
**kwargs Any

Implementation specific keyword arguments.

 {}

Raises:

Type Description
NotImplementedError

if data profiling is not supported by this data validator.
Source code in src/zenml/data_validators/base_data_validator.py 
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
def data_profiling(
    self,
    dataset: Any,
    comparison_dataset: Optional[Any] = None,
    profile_list: Optional[Sequence[Any]] = None,
    **kwargs: Any,
) -> Any:
    """Analyze one or more datasets and generate a data profile.

    This method should be implemented by data validators that support
    analyzing a dataset and generating a data profile (e.g. schema,
    statistical summary, data distribution profile, validation
    rules, data drift reports etc.).
    The method should return a data profile object.

    This method also accepts an optional second dataset argument to
    accommodate different categories of data profiling, e.g.:

    * profiles generated from a single dataset: schema inference, validation
    rules inference, statistical profiles, data integrity reports
    * differential profiles that need a second dataset for comparison:
    differential statistical profiles, data drift reports

    Data validators that support generating multiple categories of data
    profiles should also take in a `profile_list` argument that lists the
    subset of profiles to be generated. If not supplied, the behavior is
    implementation specific, but it is recommended to provide a good default
    (e.g. a single default data profile type may be generated and returned,
    or all available data profiles may be generated and returned as a single
    result).

    Args:
        dataset: Target dataset to be profiled.
        comparison_dataset: Optional second dataset to be used for data
            comparison profiles (e.g data drift reports).
        profile_list: Optional list identifying the categories of data
            profiles to be generated.
        **kwargs: Implementation specific keyword arguments.

    Raises:
        NotImplementedError: if data profiling is not supported by this
            data validator.
    """
    raise NotImplementedError(
        f"Data profiling is not supported by the {self.__class__} data "
        f"validator."
    )

data_validation(dataset, comparison_dataset=None, check_list=None, **kwargs)

Run data validation checks on a dataset.

This method should be implemented by data validators that support running data quality checks an input dataset (e.g. data integrity checks, data drift checks).

This method also accepts an optional second dataset argument to accommodate different categories of data validation tests, e.g.:

  • single dataset checks: data integrity checks (e.g. missing values, conflicting labels, mixed data types etc.)
  • checks that compare two datasets: data drift checks (e.g. new labels, feature drift, label drift etc.)

Data validators that support running multiple categories of data integrity checks should also take in a check_list argument that lists the subset of checks to be performed. If not supplied, the behavior is implementation specific, but it is recommended to provide a good default (e.g. a single default validation check may be performed, or all available validation checks may be performed and their results returned as a list of objects).

Parameters:

Name Type Description Default
dataset Any

Target dataset to be validated.

 required
comparison_dataset Optional[Any]

Optional second dataset to be used for data comparison checks (e.g data drift checks).

 None
check_list Optional[Sequence[Any]]

Optional list identifying the data checks to be performed.

 None
**kwargs Any

Implementation specific keyword arguments.

 {}

Raises:

Type Description
NotImplementedError

if data validation is not supported by this data validator.
Source code in src/zenml/data_validators/base_data_validator.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
def data_validation(
    self,
    dataset: Any,
    comparison_dataset: Optional[Any] = None,
    check_list: Optional[Sequence[Any]] = None,
    **kwargs: Any,
) -> Any:
    """Run data validation checks on a dataset.

    This method should be implemented by data validators that support
    running data quality checks an input dataset (e.g. data integrity
    checks, data drift checks).

    This method also accepts an optional second dataset argument to
    accommodate different categories of data validation tests, e.g.:

    * single dataset checks: data integrity checks (e.g. missing
    values, conflicting labels, mixed data types etc.)
    * checks that compare two datasets: data drift checks (e.g. new labels,
    feature drift, label drift etc.)

    Data validators that support running multiple categories of data
    integrity checks should also take in a `check_list` argument that
    lists the subset of checks to be performed. If not supplied, the
    behavior is implementation specific, but it is recommended to provide a
    good default (e.g. a single default validation check may be performed,
    or all available validation checks may be performed and their results
    returned as a list of objects).

    Args:
        dataset: Target dataset to be validated.
        comparison_dataset: Optional second dataset to be used for data
            comparison checks (e.g data drift checks).
        check_list: Optional list identifying the data checks to
            be performed.
        **kwargs: Implementation specific keyword arguments.

    Raises:
        NotImplementedError: if data validation is not
            supported by this data validator.
    """
    raise NotImplementedError(
        f"Data validation not implemented for {self}."
    )

get_active_data_validator() classmethod

Get the data validator registered in the active stack.

Returns:

Type Description
BaseDataValidator

The data validator registered in the active stack.

Raises:

Type Description
TypeError

if a data validator is not part of the active stack.
Source code in src/zenml/data_validators/base_data_validator.py 
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
@classmethod
def get_active_data_validator(cls) -> "BaseDataValidator":
    """Get the data validator registered in the active stack.

    Returns:
        The data validator registered in the active stack.

    Raises:
        TypeError: if a data validator is not part of the
            active stack.
    """
    flavor: BaseDataValidatorFlavor = cls.FLAVOR()
    client = Client()
    data_validator = client.active_stack.data_validator
    if not data_validator or not isinstance(data_validator, cls):
        raise TypeError(
            f"The active stack needs to have a {cls.NAME} data "
            f"validator component registered to be able to run data validation "
            f"actions with {cls.NAME}. You can create a new stack with "
            f"a {cls.NAME} data validator component or update your "
            f"active stack to add this component, e.g.:\n\n"
            f"  `zenml data-validator register {flavor.name} "
            f"--flavor={flavor.name} ...`\n"
            f"  `zenml stack register <STACK-NAME> -dv {flavor.name} ...`\n"
            f"  or:\n"
            f"  `zenml stack update -dv {flavor.name}`\n\n"
        )

    return data_validator

model_validation(dataset, model, comparison_dataset=None, check_list=None, **kwargs)

Run model validation checks.

This method should be implemented by data validators that support running model validation checks (e.g. confusion matrix validation, performance reports, model error analyzes, etc).

Unlike data_validation, model validation checks require that a model be present as an active component during the validation process.

This method also accepts an optional second dataset argument to accommodate different categories of data validation tests, e.g.:

  • single dataset tests: confusion matrix validation, performance reports, model error analyzes, etc
  • model comparison tests: tests that identify changes in a model behavior by comparing how it performs on two different datasets.

Data validators that support running multiple categories of model validation checks should also take in a check_list argument that lists the subset of checks to be performed. If not supplied, the behavior is implementation specific, but it is recommended to provide a good default (e.g. a single default validation check may be performed, or all available validation checks may be performed and their results returned as a list of objects).

Parameters:

Name Type Description Default
dataset Any

Target dataset to be validated.

 required
model Any

Target model to be validated.

 required
comparison_dataset Optional[Any]

Optional second dataset to be used for model comparison checks (e.g model performance comparison checks).

 None
check_list Optional[Sequence[Any]]

Optional list identifying the model validation checks to be performed.

 None
**kwargs Any

Implementation specific keyword arguments.

 {}

Raises:

Type Description
NotImplementedError

if model validation is not supported by this data validator.
Source code in src/zenml/data_validators/base_data_validator.py 
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
def model_validation(
    self,
    dataset: Any,
    model: Any,
    comparison_dataset: Optional[Any] = None,
    check_list: Optional[Sequence[Any]] = None,
    **kwargs: Any,
) -> Any:
    """Run model validation checks.

    This method should be implemented by data validators that support
    running model validation checks (e.g. confusion matrix validation,
    performance reports, model error analyzes, etc).

    Unlike `data_validation`, model validation checks require that a model
    be present as an active component during the validation process.

    This method also accepts an optional second dataset argument to
    accommodate different categories of data validation tests, e.g.:

    * single dataset tests: confusion matrix validation,
    performance reports, model error analyzes, etc
    * model comparison tests: tests that identify changes in a model
    behavior by comparing how it performs on two different datasets.

    Data validators that support running multiple categories of model
    validation checks should also take in a `check_list` argument that
    lists the subset of checks to be performed. If not supplied, the
    behavior is implementation specific, but it is recommended to provide a
    good default (e.g. a single default validation check may be performed,
    or all available validation checks may be performed and their results
    returned as a list of objects).

    Args:
        dataset: Target dataset to be validated.
        model: Target model to be validated.
        comparison_dataset: Optional second dataset to be used for model
            comparison checks (e.g model performance comparison checks).
        check_list: Optional list identifying the model validation checks to
            be performed.
        **kwargs: Implementation specific keyword arguments.

    Raises:
        NotImplementedError: if model validation is not supported by this
            data validator.
    """
    raise NotImplementedError(
        f"Model validation not implemented for {self}."
    )

BaseDataValidatorFlavor

Bases: Flavor

Base class for data validator flavors.

Source code in src/zenml/data_validators/base_data_validator.py 
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
class BaseDataValidatorFlavor(Flavor):
    """Base class for data validator flavors."""

    @property
    def type(self) -> StackComponentType:
        """The type of the component.

        Returns:
            The type of the component.
        """
        return StackComponentType.DATA_VALIDATOR

    @property
    def config_class(self) -> Type[BaseDataValidatorConfig]:
        """Config class for data validator.

        Returns:
            Config class for data validator.
        """
        return BaseDataValidatorConfig

    @property
    def implementation_class(self) -> Type[BaseDataValidator]:
        """Implementation for data validator.

        Returns:
            Implementation for data validator.
        """
        return BaseDataValidator

config_class property

Config class for data validator.

Returns:

Type Description
Type[BaseDataValidatorConfig]

Config class for data validator.

implementation_class property

Implementation for data validator.

Returns:

Type Description
Type[BaseDataValidator]

Implementation for data validator.

type property

The type of the component.

Returns:

Type Description
StackComponentType

The type of the component.