Skip to content

Zen Stores

zenml.zen_stores special

ZenStores define ways to store ZenML relevant data locally or remotely.

base_zen_store

Base Zen Store implementation.

BaseZenStore (BaseModel, ZenStoreInterface, ABC) pydantic-model

Base class for accessing and persisting ZenML core objects.

Attributes:

Name Type Description
config StoreConfiguration

The configuration of the store.

Source code in zenml/zen_stores/base_zen_store.py
class BaseZenStore(
    BaseModel,
    ZenStoreInterface,
    ABC,
):
    """Base class for accessing and persisting ZenML core objects.

    Attributes:
        config: The configuration of the store.
    """

    config: StoreConfiguration

    TYPE: ClassVar[StoreType]
    CONFIG_TYPE: ClassVar[Type[StoreConfiguration]]

    # ---------------------------------
    # Initialization and configuration
    # ---------------------------------

    def __init__(
        self,
        skip_default_registrations: bool = False,
        **kwargs: Any,
    ) -> None:
        """Create and initialize a store.

        Args:
            skip_default_registrations: If `True`, the creation of the default
                stack and user in the store will be skipped.
            **kwargs: Additional keyword arguments to pass to the Pydantic
                constructor.

        Raises:
            RuntimeError: If the store cannot be initialized.
            AuthorizationException: If the store cannot be initialized due to
                authentication errors.
        """
        super().__init__(**kwargs)

        try:
            self._initialize()

        # Handle cases where the ZenML server is not available
        except ConnectionError as e:
            error_message = (
                "Cannot connect to the ZenML database because the ZenML server "
                f"at {self.url} is not running."
            )
            if urlparse(self.url).hostname in ["localhost", "127.0.0.1"]:
                recommendation = (
                    "Please run `zenml down` and `zenml up` to restart the "
                    "server."
                )
            else:
                recommendation = (
                    "Please run `zenml disconnect` and `zenml connect --url "
                    f"{self.url}` to reconnect to the server."
                )
            raise RuntimeError(f"{error_message}\n{recommendation}") from e

        except AuthorizationException as e:
            raise AuthorizationException(
                f"Authorization failed for store at '{self.url}'. Please check "
                f"your credentials: {str(e)}"
            )

        except Exception as e:
            raise RuntimeError(
                f"Error initializing {self.type.value} store with URL "
                f"'{self.url}': {str(e)}"
            ) from e

        if not skip_default_registrations:
            logger.debug("Initializing database")
            self._initialize_database()
        else:
            logger.debug("Skipping database initialization")

    @staticmethod
    def get_store_class(store_type: StoreType) -> Type["BaseZenStore"]:
        """Returns the class of the given store type.

        Args:
            store_type: The type of the store to get the class for.

        Returns:
            The class of the given store type or None if the type is unknown.

        Raises:
            TypeError: If the store type is unsupported.
        """
        if store_type == StoreType.SQL:
            from zenml.zen_stores.sql_zen_store import SqlZenStore

            return SqlZenStore
        elif store_type == StoreType.REST:
            from zenml.zen_stores.rest_zen_store import RestZenStore

            return RestZenStore
        else:
            raise TypeError(
                f"No store implementation found for store type "
                f"`{store_type.value}`."
            )

    @staticmethod
    def get_store_config_class(
        store_type: StoreType,
    ) -> Type["StoreConfiguration"]:
        """Returns the store config class of the given store type.

        Args:
            store_type: The type of the store to get the class for.

        Returns:
            The config class of the given store type.
        """
        store_class = BaseZenStore.get_store_class(store_type)
        return store_class.CONFIG_TYPE

    @staticmethod
    def get_store_type(url: str) -> StoreType:
        """Returns the store type associated with a URL schema.

        Args:
            url: The store URL.

        Returns:
            The store type associated with the supplied URL schema.

        Raises:
            TypeError: If no store type was found to support the supplied URL.
        """
        from zenml.zen_stores.rest_zen_store import RestZenStoreConfiguration
        from zenml.zen_stores.sql_zen_store import SqlZenStoreConfiguration

        if SqlZenStoreConfiguration.supports_url_scheme(url):
            return StoreType.SQL
        elif RestZenStoreConfiguration.supports_url_scheme(url):
            return StoreType.REST
        else:
            raise TypeError(f"No store implementation found for URL: {url}.")

    @staticmethod
    def create_store(
        config: StoreConfiguration,
        skip_default_registrations: bool = False,
        **kwargs: Any,
    ) -> "BaseZenStore":
        """Create and initialize a store from a store configuration.

        Args:
            config: The store configuration to use.
            skip_default_registrations: If `True`, the creation of the default
                stack and user in the store will be skipped.
            **kwargs: Additional keyword arguments to pass to the store class

        Returns:
            The initialized store.
        """
        logger.debug(f"Creating store with config '{config}'...")
        store_class = BaseZenStore.get_store_class(config.type)
        store = store_class(
            config=config,
            skip_default_registrations=skip_default_registrations,
            **kwargs,
        )

        return store

    @staticmethod
    def get_default_store_config(path: str) -> StoreConfiguration:
        """Get the default store configuration.

        The default store is a SQLite store that saves the DB contents on the
        local filesystem.

        Args:
            path: The local path where the store DB will be stored.

        Returns:
            The default store configuration.
        """
        from zenml.zen_stores.sql_zen_store import SqlZenStoreConfiguration

        config = SqlZenStoreConfiguration(
            type=StoreType.SQL,
            url=SqlZenStoreConfiguration.get_local_url(path),
            secrets_store=SqlSecretsStoreConfiguration(
                type=SecretsStoreType.SQL,
            ),
        )
        return config

    def _initialize_database(self) -> None:
        """Initialize the database on first use."""

    @property
    def url(self) -> str:
        """The URL of the store.

        Returns:
            The URL of the store.
        """
        return self.config.url

    @property
    def type(self) -> StoreType:
        """The type of the store.

        Returns:
            The type of the store.
        """
        return self.TYPE

    def validate_active_config(
        self,
        active_workspace_name_or_id: Optional[Union[str, UUID]] = None,
        active_stack_id: Optional[UUID] = None,
        config_name: str = "",
    ) -> Tuple[WorkspaceResponse, StackResponse]:
        """Validate the active configuration.

        Call this method to validate the supplied active workspace and active
        stack values.

        This method is guaranteed to return valid workspace ID and stack ID
        values. If the supplied workspace and stack are not set or are not valid
        (e.g. they do not exist or are not accessible), the default workspace and
        default workspace stack will be returned in their stead.

        Args:
            active_workspace_name_or_id: The name or ID of the active workspace.
            active_stack_id: The ID of the active stack.
            config_name: The name of the configuration to validate (used in the
                displayed logs/messages).

        Returns:
            A tuple containing the active workspace and active stack.
        """
        active_workspace: WorkspaceResponse

        if active_workspace_name_or_id:
            try:
                active_workspace = self.get_workspace(
                    active_workspace_name_or_id
                )
            except KeyError:
                active_workspace = self._get_default_workspace()

                logger.warning(
                    f"The current {config_name} active workspace is no longer "
                    f"available. Resetting the active workspace to "
                    f"'{active_workspace.name}'."
                )
        else:
            active_workspace = self._get_default_workspace()

            logger.info(
                f"Setting the {config_name} active workspace "
                f"to '{active_workspace.name}'."
            )

        active_stack: StackResponse

        # Sanitize the active stack
        if active_stack_id:
            # Ensure that the active stack is still valid
            try:
                active_stack = self.get_stack(stack_id=active_stack_id)
            except KeyError:
                logger.warning(
                    "The current %s active stack is no longer available. "
                    "Resetting the active stack to default.",
                    config_name,
                )
                active_stack = self._get_default_stack(
                    workspace_id=active_workspace.id
                )
            else:
                if active_stack.workspace.id != active_workspace.id:
                    logger.warning(
                        "The current %s active stack is not part of the active "
                        "workspace. Resetting the active stack to default.",
                        config_name,
                    )
                    active_stack = self._get_default_stack(
                        workspace_id=active_workspace.id
                    )

        else:
            logger.warning(
                "Setting the %s active stack to default.",
                config_name,
            )
            active_stack = self._get_default_stack(
                workspace_id=active_workspace.id
            )

        return active_workspace, active_stack

    def get_store_info(self) -> ServerModel:
        """Get information about the store.

        Returns:
            Information about the store.
        """
        from zenml.zen_stores.sql_zen_store import SqlZenStore

        server_config = ServerConfiguration.get_server_config()
        deployment_type = server_config.deployment_type
        auth_scheme = server_config.auth_scheme
        base_url = server_config.base_url
        metadata = server_config.metadata
        secrets_store_type = SecretsStoreType.NONE
        if isinstance(self, SqlZenStore):
            secrets_store_type = self.secrets_store.type
        return ServerModel(
            id=GlobalConfiguration().user_id,
            version=zenml.__version__,
            deployment_type=deployment_type,
            database_type=ServerDatabaseType.OTHER,
            debug=IS_DEBUG_ENV,
            secrets_store_type=secrets_store_type,
            auth_scheme=auth_scheme,
            base_url=base_url,
            metadata=metadata,
        )

    def is_local_store(self) -> bool:
        """Check if the store is local or connected to a local ZenML server.

        Returns:
            True if the store is local, False otherwise.
        """
        return self.get_store_info().is_local()

    # -----------------------------
    # Default workspaces and stacks
    # -----------------------------

    @property
    def _default_workspace_name(self) -> str:
        """Get the default workspace name.

        Returns:
            The default workspace name.
        """
        return os.getenv(
            ENV_ZENML_DEFAULT_WORKSPACE_NAME, DEFAULT_WORKSPACE_NAME
        )

    def _get_default_workspace(self) -> WorkspaceResponse:
        """Get the default workspace.

        Raises:
            KeyError: If the default workspace doesn't exist.

        Returns:
            The default workspace.
        """
        try:
            return self.get_workspace(self._default_workspace_name)
        except KeyError:
            raise KeyError("Unable to find default workspace.")

    def _get_default_stack(
        self,
        workspace_id: UUID,
    ) -> StackResponse:
        """Get the default stack for a user in a workspace.

        Args:
            workspace_id: ID of the workspace.

        Returns:
            The default stack in the workspace.

        Raises:
            KeyError: if the workspace or default stack doesn't exist.
        """
        default_stacks = self.list_stacks(
            StackFilter(
                workspace_id=workspace_id,
                name=DEFAULT_STACK_AND_COMPONENT_NAME,
            )
        )
        if default_stacks.total == 0:
            raise KeyError(
                f"No default stack found in workspace {workspace_id}."
            )
        return default_stacks.items[0]

    def get_external_user(self, user_id: UUID) -> UserResponse:
        """Get a user by external ID.

        Args:
            user_id: The external ID of the user.

        Returns:
            The user with the supplied external ID.

        Raises:
            KeyError: If the user doesn't exist.
        """
        users = self.list_users(UserFilter(external_user_id=user_id))
        if users.total == 0:
            raise KeyError(f"User with external ID '{user_id}' not found.")
        return users.items[0]

    class Config:
        """Pydantic configuration class."""

        # Validate attributes when assigning them. We need to set this in order
        # to have a mix of mutable and immutable attributes
        validate_assignment = True
        # Ignore extra attributes from configs of previous ZenML versions
        extra = "ignore"
        # all attributes with leading underscore are private and therefore
        # are mutable and not included in serialization
        underscore_attrs_are_private = True
type: StoreType property readonly

The type of the store.

Returns:

Type Description
StoreType

The type of the store.

url: str property readonly

The URL of the store.

Returns:

Type Description
str

The URL of the store.

Config

Pydantic configuration class.

Source code in zenml/zen_stores/base_zen_store.py
class Config:
    """Pydantic configuration class."""

    # Validate attributes when assigning them. We need to set this in order
    # to have a mix of mutable and immutable attributes
    validate_assignment = True
    # Ignore extra attributes from configs of previous ZenML versions
    extra = "ignore"
    # all attributes with leading underscore are private and therefore
    # are mutable and not included in serialization
    underscore_attrs_are_private = True
__init__(self, skip_default_registrations=False, **kwargs) special

Create and initialize a store.

Parameters:

Name Type Description Default
skip_default_registrations bool

If True, the creation of the default stack and user in the store will be skipped.

False
**kwargs Any

Additional keyword arguments to pass to the Pydantic constructor.

{}

Exceptions:

Type Description
RuntimeError

If the store cannot be initialized.

AuthorizationException

If the store cannot be initialized due to authentication errors.

Source code in zenml/zen_stores/base_zen_store.py
def __init__(
    self,
    skip_default_registrations: bool = False,
    **kwargs: Any,
) -> None:
    """Create and initialize a store.

    Args:
        skip_default_registrations: If `True`, the creation of the default
            stack and user in the store will be skipped.
        **kwargs: Additional keyword arguments to pass to the Pydantic
            constructor.

    Raises:
        RuntimeError: If the store cannot be initialized.
        AuthorizationException: If the store cannot be initialized due to
            authentication errors.
    """
    super().__init__(**kwargs)

    try:
        self._initialize()

    # Handle cases where the ZenML server is not available
    except ConnectionError as e:
        error_message = (
            "Cannot connect to the ZenML database because the ZenML server "
            f"at {self.url} is not running."
        )
        if urlparse(self.url).hostname in ["localhost", "127.0.0.1"]:
            recommendation = (
                "Please run `zenml down` and `zenml up` to restart the "
                "server."
            )
        else:
            recommendation = (
                "Please run `zenml disconnect` and `zenml connect --url "
                f"{self.url}` to reconnect to the server."
            )
        raise RuntimeError(f"{error_message}\n{recommendation}") from e

    except AuthorizationException as e:
        raise AuthorizationException(
            f"Authorization failed for store at '{self.url}'. Please check "
            f"your credentials: {str(e)}"
        )

    except Exception as e:
        raise RuntimeError(
            f"Error initializing {self.type.value} store with URL "
            f"'{self.url}': {str(e)}"
        ) from e

    if not skip_default_registrations:
        logger.debug("Initializing database")
        self._initialize_database()
    else:
        logger.debug("Skipping database initialization")
create_store(config, skip_default_registrations=False, **kwargs) staticmethod

Create and initialize a store from a store configuration.

Parameters:

Name Type Description Default
config StoreConfiguration

The store configuration to use.

required
skip_default_registrations bool

If True, the creation of the default stack and user in the store will be skipped.

False
**kwargs Any

Additional keyword arguments to pass to the store class

{}

Returns:

Type Description
BaseZenStore

The initialized store.

Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def create_store(
    config: StoreConfiguration,
    skip_default_registrations: bool = False,
    **kwargs: Any,
) -> "BaseZenStore":
    """Create and initialize a store from a store configuration.

    Args:
        config: The store configuration to use.
        skip_default_registrations: If `True`, the creation of the default
            stack and user in the store will be skipped.
        **kwargs: Additional keyword arguments to pass to the store class

    Returns:
        The initialized store.
    """
    logger.debug(f"Creating store with config '{config}'...")
    store_class = BaseZenStore.get_store_class(config.type)
    store = store_class(
        config=config,
        skip_default_registrations=skip_default_registrations,
        **kwargs,
    )

    return store
get_default_store_config(path) staticmethod

Get the default store configuration.

The default store is a SQLite store that saves the DB contents on the local filesystem.

Parameters:

Name Type Description Default
path str

The local path where the store DB will be stored.

required

Returns:

Type Description
StoreConfiguration

The default store configuration.

Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_default_store_config(path: str) -> StoreConfiguration:
    """Get the default store configuration.

    The default store is a SQLite store that saves the DB contents on the
    local filesystem.

    Args:
        path: The local path where the store DB will be stored.

    Returns:
        The default store configuration.
    """
    from zenml.zen_stores.sql_zen_store import SqlZenStoreConfiguration

    config = SqlZenStoreConfiguration(
        type=StoreType.SQL,
        url=SqlZenStoreConfiguration.get_local_url(path),
        secrets_store=SqlSecretsStoreConfiguration(
            type=SecretsStoreType.SQL,
        ),
    )
    return config
get_external_user(self, user_id)

Get a user by external ID.

Parameters:

Name Type Description Default
user_id UUID

The external ID of the user.

required

Returns:

Type Description
UserResponse

The user with the supplied external ID.

Exceptions:

Type Description
KeyError

If the user doesn't exist.

Source code in zenml/zen_stores/base_zen_store.py
def get_external_user(self, user_id: UUID) -> UserResponse:
    """Get a user by external ID.

    Args:
        user_id: The external ID of the user.

    Returns:
        The user with the supplied external ID.

    Raises:
        KeyError: If the user doesn't exist.
    """
    users = self.list_users(UserFilter(external_user_id=user_id))
    if users.total == 0:
        raise KeyError(f"User with external ID '{user_id}' not found.")
    return users.items[0]
get_store_class(store_type) staticmethod

Returns the class of the given store type.

Parameters:

Name Type Description Default
store_type StoreType

The type of the store to get the class for.

required

Returns:

Type Description
Type[BaseZenStore]

The class of the given store type or None if the type is unknown.

Exceptions:

Type Description
TypeError

If the store type is unsupported.

Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_store_class(store_type: StoreType) -> Type["BaseZenStore"]:
    """Returns the class of the given store type.

    Args:
        store_type: The type of the store to get the class for.

    Returns:
        The class of the given store type or None if the type is unknown.

    Raises:
        TypeError: If the store type is unsupported.
    """
    if store_type == StoreType.SQL:
        from zenml.zen_stores.sql_zen_store import SqlZenStore

        return SqlZenStore
    elif store_type == StoreType.REST:
        from zenml.zen_stores.rest_zen_store import RestZenStore

        return RestZenStore
    else:
        raise TypeError(
            f"No store implementation found for store type "
            f"`{store_type.value}`."
        )
get_store_config_class(store_type) staticmethod

Returns the store config class of the given store type.

Parameters:

Name Type Description Default
store_type StoreType

The type of the store to get the class for.

required

Returns:

Type Description
Type[StoreConfiguration]

The config class of the given store type.

Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_store_config_class(
    store_type: StoreType,
) -> Type["StoreConfiguration"]:
    """Returns the store config class of the given store type.

    Args:
        store_type: The type of the store to get the class for.

    Returns:
        The config class of the given store type.
    """
    store_class = BaseZenStore.get_store_class(store_type)
    return store_class.CONFIG_TYPE
get_store_info(self)

Get information about the store.

Returns:

Type Description
ServerModel

Information about the store.

Source code in zenml/zen_stores/base_zen_store.py
def get_store_info(self) -> ServerModel:
    """Get information about the store.

    Returns:
        Information about the store.
    """
    from zenml.zen_stores.sql_zen_store import SqlZenStore

    server_config = ServerConfiguration.get_server_config()
    deployment_type = server_config.deployment_type
    auth_scheme = server_config.auth_scheme
    base_url = server_config.base_url
    metadata = server_config.metadata
    secrets_store_type = SecretsStoreType.NONE
    if isinstance(self, SqlZenStore):
        secrets_store_type = self.secrets_store.type
    return ServerModel(
        id=GlobalConfiguration().user_id,
        version=zenml.__version__,
        deployment_type=deployment_type,
        database_type=ServerDatabaseType.OTHER,
        debug=IS_DEBUG_ENV,
        secrets_store_type=secrets_store_type,
        auth_scheme=auth_scheme,
        base_url=base_url,
        metadata=metadata,
    )
get_store_type(url) staticmethod

Returns the store type associated with a URL schema.

Parameters:

Name Type Description Default
url str

The store URL.

required

Returns:

Type Description
StoreType

The store type associated with the supplied URL schema.

Exceptions:

Type Description
TypeError

If no store type was found to support the supplied URL.

Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_store_type(url: str) -> StoreType:
    """Returns the store type associated with a URL schema.

    Args:
        url: The store URL.

    Returns:
        The store type associated with the supplied URL schema.

    Raises:
        TypeError: If no store type was found to support the supplied URL.
    """
    from zenml.zen_stores.rest_zen_store import RestZenStoreConfiguration
    from zenml.zen_stores.sql_zen_store import SqlZenStoreConfiguration

    if SqlZenStoreConfiguration.supports_url_scheme(url):
        return StoreType.SQL
    elif RestZenStoreConfiguration.supports_url_scheme(url):
        return StoreType.REST
    else:
        raise TypeError(f"No store implementation found for URL: {url}.")
is_local_store(self)

Check if the store is local or connected to a local ZenML server.

Returns:

Type Description
bool

True if the store is local, False otherwise.

Source code in zenml/zen_stores/base_zen_store.py
def is_local_store(self) -> bool:
    """Check if the store is local or connected to a local ZenML server.

    Returns:
        True if the store is local, False otherwise.
    """
    return self.get_store_info().is_local()
validate_active_config(self, active_workspace_name_or_id=None, active_stack_id=None, config_name='')

Validate the active configuration.

Call this method to validate the supplied active workspace and active stack values.

This method is guaranteed to return valid workspace ID and stack ID values. If the supplied workspace and stack are not set or are not valid (e.g. they do not exist or are not accessible), the default workspace and default workspace stack will be returned in their stead.

Parameters:

Name Type Description Default
active_workspace_name_or_id Union[str, uuid.UUID]

The name or ID of the active workspace.

None
active_stack_id Optional[uuid.UUID]

The ID of the active stack.

None
config_name str

The name of the configuration to validate (used in the displayed logs/messages).

''

Returns:

Type Description
Tuple[zenml.models.v2.core.workspace.WorkspaceResponse, zenml.models.v2.core.stack.StackResponse]

A tuple containing the active workspace and active stack.

Source code in zenml/zen_stores/base_zen_store.py
def validate_active_config(
    self,
    active_workspace_name_or_id: Optional[Union[str, UUID]] = None,
    active_stack_id: Optional[UUID] = None,
    config_name: str = "",
) -> Tuple[WorkspaceResponse, StackResponse]:
    """Validate the active configuration.

    Call this method to validate the supplied active workspace and active
    stack values.

    This method is guaranteed to return valid workspace ID and stack ID
    values. If the supplied workspace and stack are not set or are not valid
    (e.g. they do not exist or are not accessible), the default workspace and
    default workspace stack will be returned in their stead.

    Args:
        active_workspace_name_or_id: The name or ID of the active workspace.
        active_stack_id: The ID of the active stack.
        config_name: The name of the configuration to validate (used in the
            displayed logs/messages).

    Returns:
        A tuple containing the active workspace and active stack.
    """
    active_workspace: WorkspaceResponse

    if active_workspace_name_or_id:
        try:
            active_workspace = self.get_workspace(
                active_workspace_name_or_id
            )
        except KeyError:
            active_workspace = self._get_default_workspace()

            logger.warning(
                f"The current {config_name} active workspace is no longer "
                f"available. Resetting the active workspace to "
                f"'{active_workspace.name}'."
            )
    else:
        active_workspace = self._get_default_workspace()

        logger.info(
            f"Setting the {config_name} active workspace "
            f"to '{active_workspace.name}'."
        )

    active_stack: StackResponse

    # Sanitize the active stack
    if active_stack_id:
        # Ensure that the active stack is still valid
        try:
            active_stack = self.get_stack(stack_id=active_stack_id)
        except KeyError:
            logger.warning(
                "The current %s active stack is no longer available. "
                "Resetting the active stack to default.",
                config_name,
            )
            active_stack = self._get_default_stack(
                workspace_id=active_workspace.id
            )
        else:
            if active_stack.workspace.id != active_workspace.id:
                logger.warning(
                    "The current %s active stack is not part of the active "
                    "workspace. Resetting the active stack to default.",
                    config_name,
                )
                active_stack = self._get_default_stack(
                    workspace_id=active_workspace.id
                )

    else:
        logger.warning(
            "Setting the %s active stack to default.",
            config_name,
        )
        active_stack = self._get_default_stack(
            workspace_id=active_workspace.id
        )

    return active_workspace, active_stack

migrations special

Alembic database migration utilities.

alembic

Alembic utilities wrapper.

The Alembic class defined here acts as a wrapper around the Alembic library that automatically configures Alembic to use the ZenML SQL store database connection.

Alembic

Alembic environment and migration API.

This class provides a wrapper around the Alembic library that automatically configures Alembic to use the ZenML SQL store database connection.

Source code in zenml/zen_stores/migrations/alembic.py
class Alembic:
    """Alembic environment and migration API.

    This class provides a wrapper around the Alembic library that automatically
    configures Alembic to use the ZenML SQL store database connection.
    """

    def __init__(
        self,
        engine: Engine,
        metadata: MetaData = SQLModel.metadata,
        context: Optional[EnvironmentContext] = None,
        **kwargs: Any,
    ) -> None:
        """Initialize the Alembic wrapper.

        Args:
            engine: The SQLAlchemy engine to use.
            metadata: The SQLAlchemy metadata to use.
            context: The Alembic environment context to use. If not set, a new
                context is created pointing to the ZenML migrations directory.
            **kwargs: Additional keyword arguments to pass to the Alembic
                environment context.
        """
        self.engine = engine
        self.metadata = metadata
        self.context_kwargs = kwargs

        self.config = Config()
        self.config.set_main_option(
            "script_location", str(Path(__file__).parent)
        )

        self.script_directory = ScriptDirectory.from_config(self.config)
        if context is None:
            self.environment_context = EnvironmentContext(
                self.config, self.script_directory
            )
        else:
            self.environment_context = context

    def db_is_empty(self) -> bool:
        """Check if the database is empty.

        Returns:
            True if the database is empty, False otherwise.
        """
        # Check the existence of any of the SQLModel tables
        return not self.engine.dialect.has_table(
            self.engine.connect(), schemas.StackSchema.__tablename__
        )

    def run_migrations(
        self,
        fn: Optional[Callable[[_RevIdType, MigrationContext], List[Any]]],
    ) -> None:
        """Run an online migration function in the current migration context.

        Args:
            fn: Migration function to run. If not set, the function configured
                externally by the Alembic CLI command is used.
        """
        fn_context_args: Dict[Any, Any] = {}
        if fn is not None:
            fn_context_args["fn"] = fn

        with self.engine.connect() as connection:
            self.environment_context.configure(
                connection=connection,
                target_metadata=self.metadata,
                include_object=include_object,
                compare_type=True,
                render_as_batch=True,
                **fn_context_args,
                **self.context_kwargs,
            )

            with self.environment_context.begin_transaction():
                self.environment_context.run_migrations()

    def head_revisions(self) -> List[str]:
        """Get the head database revisions.

        Returns:
            List of head revisions.
        """
        head_revisions: List[str] = []

        def do_get_head_rev(rev: _RevIdType, context: Any) -> List[Any]:
            nonlocal head_revisions

            for r in self.script_directory.get_heads():
                if r is None:
                    continue
                head_revisions.append(r)
            return []

        self.run_migrations(do_get_head_rev)

        return head_revisions

    def current_revisions(self) -> List[str]:
        """Get the current database revisions.

        Returns:
            List of head revisions.
        """
        current_revisions: List[str] = []

        def do_get_current_rev(rev: _RevIdType, context: Any) -> List[Any]:
            nonlocal current_revisions

            for r in self.script_directory.get_all_current(
                rev  # type:ignore [arg-type]
            ):
                if r is None:
                    continue
                current_revisions.append(r.revision)
            return []

        self.run_migrations(do_get_current_rev)

        return current_revisions

    def stamp(self, revision: str) -> None:
        """Stamp the revision table with the given revision without running any migrations.

        Args:
            revision: String revision target.
        """

        def do_stamp(rev: _RevIdType, context: Any) -> List[Any]:
            return self.script_directory._stamp_revs(revision, rev)

        self.run_migrations(do_stamp)

    def upgrade(self, revision: str = "heads") -> None:
        """Upgrade the database to a later version.

        Args:
            revision: String revision target.
        """

        def do_upgrade(rev: _RevIdType, context: Any) -> List[Any]:
            return self.script_directory._upgrade_revs(
                revision,
                rev,  # type:ignore [arg-type]
            )

        self.run_migrations(do_upgrade)

    def downgrade(self, revision: str) -> None:
        """Revert the database to a previous version.

        Args:
            revision: String revision target.
        """

        def do_downgrade(rev: _RevIdType, context: Any) -> List[Any]:
            return self.script_directory._downgrade_revs(
                revision,
                rev,  # type:ignore [arg-type]
            )

        self.run_migrations(do_downgrade)
__init__(self, engine, metadata=MetaData(), context=None, **kwargs) special

Initialize the Alembic wrapper.

Parameters:

Name Type Description Default
engine Engine

The SQLAlchemy engine to use.

required
metadata MetaData

The SQLAlchemy metadata to use.

MetaData()
context Optional[alembic.runtime.environment.EnvironmentContext]

The Alembic environment context to use. If not set, a new context is created pointing to the ZenML migrations directory.

None
**kwargs Any

Additional keyword arguments to pass to the Alembic environment context.

{}
Source code in zenml/zen_stores/migrations/alembic.py
def __init__(
    self,
    engine: Engine,
    metadata: MetaData = SQLModel.metadata,
    context: Optional[EnvironmentContext] = None,
    **kwargs: Any,
) -> None:
    """Initialize the Alembic wrapper.

    Args:
        engine: The SQLAlchemy engine to use.
        metadata: The SQLAlchemy metadata to use.
        context: The Alembic environment context to use. If not set, a new
            context is created pointing to the ZenML migrations directory.
        **kwargs: Additional keyword arguments to pass to the Alembic
            environment context.
    """
    self.engine = engine
    self.metadata = metadata
    self.context_kwargs = kwargs

    self.config = Config()
    self.config.set_main_option(
        "script_location", str(Path(__file__).parent)
    )

    self.script_directory = ScriptDirectory.from_config(self.config)
    if context is None:
        self.environment_context = EnvironmentContext(
            self.config, self.script_directory
        )
    else:
        self.environment_context = context
current_revisions(self)

Get the current database revisions.

Returns:

Type Description
List[str]

List of head revisions.

Source code in zenml/zen_stores/migrations/alembic.py
def current_revisions(self) -> List[str]:
    """Get the current database revisions.

    Returns:
        List of head revisions.
    """
    current_revisions: List[str] = []

    def do_get_current_rev(rev: _RevIdType, context: Any) -> List[Any]:
        nonlocal current_revisions

        for r in self.script_directory.get_all_current(
            rev  # type:ignore [arg-type]
        ):
            if r is None:
                continue
            current_revisions.append(r.revision)
        return []

    self.run_migrations(do_get_current_rev)

    return current_revisions
db_is_empty(self)

Check if the database is empty.

Returns:

Type Description
bool

True if the database is empty, False otherwise.

Source code in zenml/zen_stores/migrations/alembic.py
def db_is_empty(self) -> bool:
    """Check if the database is empty.

    Returns:
        True if the database is empty, False otherwise.
    """
    # Check the existence of any of the SQLModel tables
    return not self.engine.dialect.has_table(
        self.engine.connect(), schemas.StackSchema.__tablename__
    )
downgrade(self, revision)

Revert the database to a previous version.

Parameters:

Name Type Description Default
revision str

String revision target.

required
Source code in zenml/zen_stores/migrations/alembic.py
def downgrade(self, revision: str) -> None:
    """Revert the database to a previous version.

    Args:
        revision: String revision target.
    """

    def do_downgrade(rev: _RevIdType, context: Any) -> List[Any]:
        return self.script_directory._downgrade_revs(
            revision,
            rev,  # type:ignore [arg-type]
        )

    self.run_migrations(do_downgrade)
head_revisions(self)

Get the head database revisions.

Returns:

Type Description
List[str]

List of head revisions.

Source code in zenml/zen_stores/migrations/alembic.py
def head_revisions(self) -> List[str]:
    """Get the head database revisions.

    Returns:
        List of head revisions.
    """
    head_revisions: List[str] = []

    def do_get_head_rev(rev: _RevIdType, context: Any) -> List[Any]:
        nonlocal head_revisions

        for r in self.script_directory.get_heads():
            if r is None:
                continue
            head_revisions.append(r)
        return []

    self.run_migrations(do_get_head_rev)

    return head_revisions
run_migrations(self, fn)

Run an online migration function in the current migration context.

Parameters:

Name Type Description Default
fn Optional[Callable[[Union[str, Sequence[str]], alembic.runtime.migration.MigrationContext], List[Any]]]

Migration function to run. If not set, the function configured externally by the Alembic CLI command is used.

required
Source code in zenml/zen_stores/migrations/alembic.py
def run_migrations(
    self,
    fn: Optional[Callable[[_RevIdType, MigrationContext], List[Any]]],
) -> None:
    """Run an online migration function in the current migration context.

    Args:
        fn: Migration function to run. If not set, the function configured
            externally by the Alembic CLI command is used.
    """
    fn_context_args: Dict[Any, Any] = {}
    if fn is not None:
        fn_context_args["fn"] = fn

    with self.engine.connect() as connection:
        self.environment_context.configure(
            connection=connection,
            target_metadata=self.metadata,
            include_object=include_object,
            compare_type=True,
            render_as_batch=True,
            **fn_context_args,
            **self.context_kwargs,
        )

        with self.environment_context.begin_transaction():
            self.environment_context.run_migrations()
stamp(self, revision)

Stamp the revision table with the given revision without running any migrations.

Parameters:

Name Type Description Default
revision str

String revision target.

required
Source code in zenml/zen_stores/migrations/alembic.py
def stamp(self, revision: str) -> None:
    """Stamp the revision table with the given revision without running any migrations.

    Args:
        revision: String revision target.
    """

    def do_stamp(rev: _RevIdType, context: Any) -> List[Any]:
        return self.script_directory._stamp_revs(revision, rev)

    self.run_migrations(do_stamp)
upgrade(self, revision='heads')

Upgrade the database to a later version.

Parameters:

Name Type Description Default
revision str

String revision target.

'heads'
Source code in zenml/zen_stores/migrations/alembic.py
def upgrade(self, revision: str = "heads") -> None:
    """Upgrade the database to a later version.

    Args:
        revision: String revision target.
    """

    def do_upgrade(rev: _RevIdType, context: Any) -> List[Any]:
        return self.script_directory._upgrade_revs(
            revision,
            rev,  # type:ignore [arg-type]
        )

    self.run_migrations(do_upgrade)
AlembicVersion (Base)

Alembic version table.

Source code in zenml/zen_stores/migrations/alembic.py
class AlembicVersion(Base):  # type: ignore[valid-type,misc]
    """Alembic version table."""

    __tablename__ = "alembic_version"
    version_num = Column(String, nullable=False, primary_key=True)
include_object(object, name, type_, *args, **kwargs)

Function used to exclude tables from the migration scripts.

Parameters:

Name Type Description Default
object Any

The schema item object to check.

required
name str

The name of the object to check.

required
type_ str

The type of the object to check.

required
*args Any

Additional arguments.

()
**kwargs Any

Additional keyword arguments.

{}

Returns:

Type Description
bool

True if the object should be included, False otherwise.

Source code in zenml/zen_stores/migrations/alembic.py
def include_object(
    object: Any, name: str, type_: str, *args: Any, **kwargs: Any
) -> bool:
    """Function used to exclude tables from the migration scripts.

    Args:
        object: The schema item object to check.
        name: The name of the object to check.
        type_: The type of the object to check.
        *args: Additional arguments.
        **kwargs: Additional keyword arguments.

    Returns:
        True if the object should be included, False otherwise.
    """
    return not (type_ == "table" and name in exclude_tables)

utils

ZenML database migration, backup and recovery utilities.

MigrationUtils (BaseModel) pydantic-model

Utilities for database migration, backup and recovery.

Source code in zenml/zen_stores/migrations/utils.py
class MigrationUtils(BaseModel):
    """Utilities for database migration, backup and recovery."""

    url: URL
    connect_args: Dict[str, Any]
    engine_args: Dict[str, Any]

    _engine: Optional[Engine] = None
    _master_engine: Optional[Engine] = None

    def create_engine(self, database: Optional[str] = None) -> Engine:
        """Get the SQLAlchemy engine for a database.

        Args:
            database: The name of the database. If not set, a master engine
                will be returned.

        Returns:
            The SQLAlchemy engine.
        """
        url = self.url._replace(database=database)
        return create_engine(
            url=url,
            connect_args=self.connect_args,
            **self.engine_args,
        )

    @property
    def engine(self) -> Engine:
        """The SQLAlchemy engine.

        Returns:
            The SQLAlchemy engine.
        """
        if self._engine is None:
            self._engine = self.create_engine(database=self.url.database)
        return self._engine

    @property
    def master_engine(self) -> Engine:
        """The SQLAlchemy engine for the master database.

        Returns:
            The SQLAlchemy engine for the master database.
        """
        if self._master_engine is None:
            self._master_engine = self.create_engine()
        return self._master_engine

    @classmethod
    def is_mysql_missing_database_error(cls, error: OperationalError) -> bool:
        """Checks if the given error is due to a missing database.

        Args:
            error: The error to check.

        Returns:
            If the error because the MySQL database doesn't exist.
        """
        from pymysql.constants.ER import BAD_DB_ERROR

        if not isinstance(error.orig, pymysql.err.OperationalError):
            return False

        error_code = cast(int, error.orig.args[0])
        return error_code == BAD_DB_ERROR

    def database_exists(
        self,
        database: Optional[str] = None,
    ) -> bool:
        """Check if a database exists.

        Args:
            database: The name of the database to check. If not set, the
                database name from the configuration will be used.

        Returns:
            Whether the database exists.

        Raises:
            OperationalError: If connecting to the database failed.
        """
        database = database or self.url.database

        engine = self.create_engine(database=database)
        try:
            engine.connect()
        except OperationalError as e:
            if self.is_mysql_missing_database_error(e):
                return False
            else:
                logger.exception(
                    f"Failed to connect to mysql database `{database}`.",
                )
                raise
        else:
            return True

    def drop_database(
        self,
        database: Optional[str] = None,
    ) -> None:
        """Drops a mysql database.

        Args:
            database: The name of the database to drop. If not set, the
                database name from the configuration will be used.
        """
        database = database or self.url.database
        with self.master_engine.connect() as conn:
            # drop the database if it exists
            logger.info(f"Dropping database '{database}'")
            conn.execute(text(f"DROP DATABASE IF EXISTS `{database}`"))

    def create_database(
        self,
        database: Optional[str] = None,
        drop: bool = False,
    ) -> None:
        """Creates a mysql database.

        Args:
            database: The name of the database to create. If not set, the
                database name from the configuration will be used.
            drop: Whether to drop the database if it already exists.
        """
        database = database or self.url.database
        if drop:
            self.drop_database(database=database)

        with self.master_engine.connect() as conn:
            logger.info(f"Creating database '{database}'")
            conn.execute(text(f"CREATE DATABASE IF NOT EXISTS `{database}`"))

    def backup_database_to_storage(
        self, store_db_info: Callable[[Dict[str, Any]], None]
    ) -> None:
        """Backup the database to a storage location.

        Backup the database to an abstract storage location. The storage
        location is specified by a function that is called repeatedly to
        store the database information. The function is called with a single
        argument, which is a dictionary containing either the table schema or
        table data. The dictionary contains the following keys:

            * `table`: The name of the table.
            * `create_stmt`: The table creation statement.
            * `data`: A list of rows in the table.

        Args:
            store_db_info: The function to call to store the database
                information.
        """
        metadata = MetaData()
        metadata.reflect(bind=self.engine)
        with self.engine.connect() as conn:
            for table in metadata.sorted_tables:
                # 1. extract the table creation statements

                create_table_construct = CreateTable(table)
                create_table_stmt = str(create_table_construct).strip()
                for column in create_table_construct.columns:
                    # enclosing all column names in backticks. This is because
                    # some column names are reserved keywords in MySQL. For
                    # example, keys and values. So, instead of tracking all
                    # keywords, we just enclose all column names in backticks.
                    # enclose the first word in the column definition in
                    # backticks
                    words = str(column).split()
                    words[0] = f"`{words[0]}`"
                    create_table_stmt = create_table_stmt.replace(
                        f"\n\t{str(column)}", " ".join(words)
                    )
                # if any double quotes are used for column names, replace them
                # with backticks
                create_table_stmt = create_table_stmt.replace('"', "") + ";"

                # Store the table schema
                store_db_info(
                    dict(table=table.name, create_stmt=create_table_stmt)
                )

                # 2. extract the table data in batches

                # If the table has a `created` column, we use it to sort
                # the rows in the table starting with the oldest rows.
                # This is to ensure that the rows are inserted in the
                # correct order, since some tables have inner foreign key
                # constraints.
                if "created" in table.columns:
                    order_by = table.columns["created"]
                else:
                    order_by = None

                # Fetch the number of rows in the table
                row_count = conn.scalar(
                    select([func.count("*")]).select_from(table)
                )

                # Fetch the data from the table in batches
                batch_size = 50
                for i in range(0, row_count, batch_size):
                    rows = conn.execute(
                        table.select()
                        .order_by(order_by)
                        .limit(batch_size)
                        .offset(i)
                    ).fetchall()

                    store_db_info(
                        dict(
                            table=table.name,
                            data=[row._asdict() for row in rows],
                        ),
                    )

    def restore_database_from_storage(
        self, load_db_info: Callable[[], Generator[Dict[str, Any], None, None]]
    ) -> None:
        """Restore the database from a backup storage location.

        Restores the database from an abstract storage location. The storage
        location is specified by a function that is called repeatedly to
        load the database information from the external storage chunk by chunk.
        The function must yield a dictionary containing either the table schema
        or table data. The dictionary contains the following keys:

            * `table`: The name of the table.
            * `create_stmt`: The table creation statement.
            * `data`: A list of rows in the table.

        The function must return `None` when there is no more data to load.

        Args:
            load_db_info: The function to call to load the database
                information.
        """
        # Drop and re-create the primary database
        self.create_database(
            drop=True,
        )

        metadata = MetaData(bind=self.engine)

        with self.engine.begin() as connection:
            # read the DB information one JSON object at a time
            for table_dump in load_db_info():
                table_name = table_dump["table"]
                if "create_stmt" in table_dump:
                    # execute the table creation statement
                    connection.execute(text(table_dump["create_stmt"]))
                    # Reload the database metadata after creating the table
                    metadata.reflect()

                if "data" in table_dump:
                    # insert the data into the database
                    table = metadata.tables[table_name]
                    for row in table_dump["data"]:
                        # Convert column values to the correct type
                        for column in table.columns:
                            # Blob columns are stored as binary strings
                            if column.type.python_type == bytes and isinstance(
                                row[column.name], str
                            ):
                                # Convert the string to bytes
                                row[column.name] = bytes(
                                    row[column.name], "utf-8"
                                )

                    # Insert the rows into the table
                    connection.execute(
                        table.insert().values(table_dump["data"])
                    )

    def backup_database_to_file(self, dump_file: str) -> None:
        """Backup the database to a file.

        This method dumps the entire database into a JSON file. Instead of
        using a SQL dump, we use a proprietary JSON dump because:

            * it is (mostly) not dependent on the SQL dialect or database version
            * it is safer with respect to SQL injection attacks
            * it is easier to read and debug

        The JSON file contains a list of JSON objects instead of a single JSON
        object, because it allows for buffered reading and writing of the file
        and thus reduces the memory footprint. Each JSON object can contain
        either schema or data information about a single table. For tables with
        a large amount of data, the data is split into multiple JSON objects
        with the first object always containing the schema.

        The format of the dump is as depicted in the following example:

        ```json
        {
            "table": "table1",
            "create_stmt": "CREATE TABLE table1 (id INTEGER NOT NULL, "
                "name VARCHAR(255), PRIMARY KEY (id))"
        }
        {
            "table": "table1",
            "data": [
            {
                "id": 1,
                "name": "foo"
            },
            {
                "id": 1,
                "name": "bar"
            },
            ...
            ]
        }
        {
            "table": "table1",
            "data": [
            {
                "id": 101,
                "name": "fee"
            },
            {
                "id": 102,
                "name": "bee"
            },
            ...
            ]
        }
        ```

        Args:
            dump_file: The path to the dump file.
        """
        # create the directory if it does not exist
        dump_path = os.path.dirname(os.path.abspath(dump_file))
        if not os.path.exists(dump_path):
            os.makedirs(dump_path)

        if self.url.drivername == "sqlite":
            # For a sqlite database, we can just make a copy of the database
            # file
            assert self.url.database is not None
            shutil.copyfile(
                self.url.database,
                dump_file,
            )
            return

        with open(dump_file, "w") as f:

            def json_dump(obj: Dict[str, Any]) -> None:
                """Dump a JSON object to the dump file.

                Args:
                    obj: The JSON object to dump.
                """
                # Write the data to the JSON file. Use an encoder that
                # can handle datetime, Decimal and other types.
                json.dump(
                    obj,
                    f,
                    indent=4,
                    default=pydantic_encoder,
                )
                f.write("\n")

            # Call the generic backup method with a function that dumps the
            # JSON objects to the dump file
            self.backup_database_to_storage(json_dump)

        logger.debug(f"Database backed up to {dump_file}")

    def restore_database_from_file(self, dump_file: str) -> None:
        """Restore the database from a backup dump file.

        See the documentation of the `backup_database_to_file` method for
        details on the format of the dump file.

        Args:
            dump_file: The path to the dump file.

        Raises:
            RuntimeError: If the database cannot be restored successfully.
        """
        if not os.path.exists(dump_file):
            raise RuntimeError(
                f"Database backup file '{dump_file}' does not "
                f"exist or is not accessible."
            )

        if self.url.drivername == "sqlite":
            # For a sqlite database, we just overwrite the database file
            # with the backup file
            assert self.url.database is not None
            shutil.copyfile(
                dump_file,
                self.url.database,
            )
            return

        # read the DB dump file one JSON object at a time
        with open(dump_file, "r") as f:

            def json_load() -> Generator[Dict[str, Any], None, None]:
                """Generator that loads the JSON objects in the dump file.

                Yields:
                    The loaded JSON objects.
                """
                buffer = ""
                while True:
                    chunk = f.readline()
                    if not chunk:
                        break
                    buffer += chunk
                    if chunk.rstrip() == "}":
                        yield json.loads(buffer)
                        buffer = ""

            # Call the generic restore method with a function that loads the
            # JSON objects from the dump file
            self.restore_database_from_storage(json_load)

        logger.info(f"Database successfully restored from '{dump_file}'")

    def backup_database_to_memory(self) -> List[Dict[str, Any]]:
        """Backup the database in memory.

        Returns:
            The in-memory representation of the database backup.

        Raises:
            RuntimeError: If the database cannot be backed up successfully.
        """
        if self.url.drivername == "sqlite":
            # For a sqlite database, this is not supported.
            raise RuntimeError(
                "In-memory backup is not supported for sqlite databases."
            )

        db_dump: List[Dict[str, Any]] = []

        def store_in_mem(obj: Dict[str, Any]) -> None:
            """Store a JSON object in the in-memory database backup.

            Args:
                obj: The JSON object to store.
            """
            db_dump.append(obj)

        # Call the generic backup method with a function that stores the
        # JSON objects in the in-memory database backup
        self.backup_database_to_storage(store_in_mem)

        logger.debug("Database backed up in memory")

        return db_dump

    def restore_database_from_memory(
        self, db_dump: List[Dict[str, Any]]
    ) -> None:
        """Restore the database from an in-memory backup.

        Args:
            db_dump: The in-memory database backup to restore from generated
                by the `backup_database_to_memory` method.

        Raises:
            RuntimeError: If the database cannot be restored successfully.
        """
        if self.url.drivername == "sqlite":
            # For a sqlite database, this is not supported.
            raise RuntimeError(
                "In-memory backup is not supported for sqlite databases."
            )

        def load_from_mem() -> Generator[Dict[str, Any], None, None]:
            """Generator that loads the JSON objects from the in-memory backup.

            Yields:
                The loaded JSON objects.
            """
            for obj in db_dump:
                yield obj

        # Call the generic restore method with a function that loads the
        # JSON objects from the in-memory database backup
        self.restore_database_from_storage(load_from_mem)

        logger.info("Database successfully restored from memory")

    @classmethod
    def _copy_database(cls, src_engine: Engine, dst_engine: Engine) -> None:
        """Copy the database from one engine to another.

        This method assumes that the destination database exists and is empty.

        Args:
            src_engine: The source SQLAlchemy engine.
            dst_engine: The destination SQLAlchemy engine.
        """
        src_metadata = MetaData(bind=src_engine)
        src_metadata.reflect()

        dst_metadata = MetaData(bind=dst_engine)
        dst_metadata.reflect()

        # @event.listens_for(src_metadata, "column_reflect")
        # def generalize_datatypes(inspector, tablename, column_dict):
        #     column_dict["type"] = column_dict["type"].as_generic(allow_nulltype=True)

        # Create all tables in the target database
        for table in src_metadata.sorted_tables:
            table.create(bind=dst_engine)

        # Refresh target metadata after creating the tables
        dst_metadata.clear()
        dst_metadata.reflect()

        # Copy all data from the source database to the destination database
        with src_engine.begin() as src_conn:
            with dst_engine.begin() as dst_conn:
                for src_table in src_metadata.sorted_tables:
                    dst_table = dst_metadata.tables[src_table.name]
                    insert = dst_table.insert()
                    # If the table has a `created` column, we use it to sort
                    # the rows in the table starting with the oldest rows.
                    # This is to ensure that the rows are inserted in the
                    # correct order, since some tables have inner foreign key
                    # constraints.
                    if "created" in src_table.columns:
                        order_by = src_table.columns["created"]
                    else:
                        order_by = None

                    row_count = src_conn.scalar(
                        select([func.count("*")]).select_from(src_table)
                    )

                    # Copy rows in batches
                    batch_size = 50
                    for i in range(0, row_count, batch_size):
                        rows = src_conn.execute(
                            src_table.select()
                            .order_by(order_by)
                            .limit(batch_size)
                            .offset(i)
                        ).fetchall()

                        dst_conn.execute(
                            insert, [row._asdict() for row in rows]
                        )

    def backup_database_to_db(self, backup_db_name: str) -> None:
        """Backup the database to a backup database.

        Args:
            backup_db_name: Backup database name to backup to.
        """
        # Re-create the backup database
        self.create_database(
            database=backup_db_name,
            drop=True,
        )

        backup_engine = self.create_engine(database=backup_db_name)

        self._copy_database(self.engine, backup_engine)

        logger.debug(
            f"Database backed up to the `{backup_db_name}` backup database."
        )

    def restore_database_from_db(self, backup_db_name: str) -> None:
        """Restore the database from the backup database.

        Args:
            backup_db_name: Backup database name to restore from.

        Raises:
            RuntimeError: If the backup database does not exist.
        """
        if not self.database_exists(database=backup_db_name):
            raise RuntimeError(
                f"Backup database `{backup_db_name}` does not exist."
            )

        backup_engine = self.create_engine(database=backup_db_name)

        # Drop and re-create the primary database
        self.create_database(
            drop=True,
        )

        self._copy_database(backup_engine, self.engine)

        logger.debug(
            f"Database restored from the `{backup_db_name}` "
            "backup database."
        )

    class Config:
        """Pydantic configuration class."""

        # all attributes with leading underscore are private
        underscore_attrs_are_private = True
engine: Engine property readonly

The SQLAlchemy engine.

Returns:

Type Description
Engine

The SQLAlchemy engine.

master_engine: Engine property readonly

The SQLAlchemy engine for the master database.

Returns:

Type Description
Engine

The SQLAlchemy engine for the master database.

Config

Pydantic configuration class.

Source code in zenml/zen_stores/migrations/utils.py
class Config:
    """Pydantic configuration class."""

    # all attributes with leading underscore are private
    underscore_attrs_are_private = True
backup_database_to_db(self, backup_db_name)

Backup the database to a backup database.

Parameters:

Name Type Description Default
backup_db_name str

Backup database name to backup to.

required
Source code in zenml/zen_stores/migrations/utils.py
def backup_database_to_db(self, backup_db_name: str) -> None:
    """Backup the database to a backup database.

    Args:
        backup_db_name: Backup database name to backup to.
    """
    # Re-create the backup database
    self.create_database(
        database=backup_db_name,
        drop=True,
    )

    backup_engine = self.create_engine(database=backup_db_name)

    self._copy_database(self.engine, backup_engine)

    logger.debug(
        f"Database backed up to the `{backup_db_name}` backup database."
    )
backup_database_to_file(self, dump_file)

Backup the database to a file.

This method dumps the entire database into a JSON file. Instead of using a SQL dump, we use a proprietary JSON dump because:

* it is (mostly) not dependent on the SQL dialect or database version
* it is safer with respect to SQL injection attacks
* it is easier to read and debug

The JSON file contains a list of JSON objects instead of a single JSON object, because it allows for buffered reading and writing of the file and thus reduces the memory footprint. Each JSON object can contain either schema or data information about a single table. For tables with a large amount of data, the data is split into multiple JSON objects with the first object always containing the schema.

The format of the dump is as depicted in the following example:

{
    "table": "table1",
    "create_stmt": "CREATE TABLE table1 (id INTEGER NOT NULL, "
        "name VARCHAR(255), PRIMARY KEY (id))"
}
{
    "table": "table1",
    "data": [
    {
        "id": 1,
        "name": "foo"
    },
    {
        "id": 1,
        "name": "bar"
    },
    ...
    ]
}
{
    "table": "table1",
    "data": [
    {
        "id": 101,
        "name": "fee"
    },
    {
        "id": 102,
        "name": "bee"
    },
    ...
    ]
}

Parameters:

Name Type Description Default
dump_file str

The path to the dump file.

required
Source code in zenml/zen_stores/migrations/utils.py
def backup_database_to_file(self, dump_file: str) -> None:
    """Backup the database to a file.

    This method dumps the entire database into a JSON file. Instead of
    using a SQL dump, we use a proprietary JSON dump because:

        * it is (mostly) not dependent on the SQL dialect or database version
        * it is safer with respect to SQL injection attacks
        * it is easier to read and debug

    The JSON file contains a list of JSON objects instead of a single JSON
    object, because it allows for buffered reading and writing of the file
    and thus reduces the memory footprint. Each JSON object can contain
    either schema or data information about a single table. For tables with
    a large amount of data, the data is split into multiple JSON objects
    with the first object always containing the schema.

    The format of the dump is as depicted in the following example:

    ```json
    {
        "table": "table1",
        "create_stmt": "CREATE TABLE table1 (id INTEGER NOT NULL, "
            "name VARCHAR(255), PRIMARY KEY (id))"
    }
    {
        "table": "table1",
        "data": [
        {
            "id": 1,
            "name": "foo"
        },
        {
            "id": 1,
            "name": "bar"
        },
        ...
        ]
    }
    {
        "table": "table1",
        "data": [
        {
            "id": 101,
            "name": "fee"
        },
        {
            "id": 102,
            "name": "bee"
        },
        ...
        ]
    }
    ```

    Args:
        dump_file: The path to the dump file.
    """
    # create the directory if it does not exist
    dump_path = os.path.dirname(os.path.abspath(dump_file))
    if not os.path.exists(dump_path):
        os.makedirs(dump_path)

    if self.url.drivername == "sqlite":
        # For a sqlite database, we can just make a copy of the database
        # file
        assert self.url.database is not None
        shutil.copyfile(
            self.url.database,
            dump_file,
        )
        return

    with open(dump_file, "w") as f:

        def json_dump(obj: Dict[str, Any]) -> None:
            """Dump a JSON object to the dump file.

            Args:
                obj: The JSON object to dump.
            """
            # Write the data to the JSON file. Use an encoder that
            # can handle datetime, Decimal and other types.
            json.dump(
                obj,
                f,
                indent=4,
                default=pydantic_encoder,
            )
            f.write("\n")

        # Call the generic backup method with a function that dumps the
        # JSON objects to the dump file
        self.backup_database_to_storage(json_dump)

    logger.debug(f"Database backed up to {dump_file}")
backup_database_to_memory(self)

Backup the database in memory.

Returns:

Type Description
List[Dict[str, Any]]

The in-memory representation of the database backup.

Exceptions:

Type Description
RuntimeError

If the database cannot be backed up successfully.

Source code in zenml/zen_stores/migrations/utils.py
def backup_database_to_memory(self) -> List[Dict[str, Any]]:
    """Backup the database in memory.

    Returns:
        The in-memory representation of the database backup.

    Raises:
        RuntimeError: If the database cannot be backed up successfully.
    """
    if self.url.drivername == "sqlite":
        # For a sqlite database, this is not supported.
        raise RuntimeError(
            "In-memory backup is not supported for sqlite databases."
        )

    db_dump: List[Dict[str, Any]] = []

    def store_in_mem(obj: Dict[str, Any]) -> None:
        """Store a JSON object in the in-memory database backup.

        Args:
            obj: The JSON object to store.
        """
        db_dump.append(obj)

    # Call the generic backup method with a function that stores the
    # JSON objects in the in-memory database backup
    self.backup_database_to_storage(store_in_mem)

    logger.debug("Database backed up in memory")

    return db_dump
backup_database_to_storage(self, store_db_info)

Backup the database to a storage location.

Backup the database to an abstract storage location. The storage location is specified by a function that is called repeatedly to store the database information. The function is called with a single argument, which is a dictionary containing either the table schema or table data. The dictionary contains the following keys:

* `table`: The name of the table.
* `create_stmt`: The table creation statement.
* `data`: A list of rows in the table.

Parameters:

Name Type Description Default
store_db_info Callable[[Dict[str, Any]], NoneType]

The function to call to store the database information.

required
Source code in zenml/zen_stores/migrations/utils.py
def backup_database_to_storage(
    self, store_db_info: Callable[[Dict[str, Any]], None]
) -> None:
    """Backup the database to a storage location.

    Backup the database to an abstract storage location. The storage
    location is specified by a function that is called repeatedly to
    store the database information. The function is called with a single
    argument, which is a dictionary containing either the table schema or
    table data. The dictionary contains the following keys:

        * `table`: The name of the table.
        * `create_stmt`: The table creation statement.
        * `data`: A list of rows in the table.

    Args:
        store_db_info: The function to call to store the database
            information.
    """
    metadata = MetaData()
    metadata.reflect(bind=self.engine)
    with self.engine.connect() as conn:
        for table in metadata.sorted_tables:
            # 1. extract the table creation statements

            create_table_construct = CreateTable(table)
            create_table_stmt = str(create_table_construct).strip()
            for column in create_table_construct.columns:
                # enclosing all column names in backticks. This is because
                # some column names are reserved keywords in MySQL. For
                # example, keys and values. So, instead of tracking all
                # keywords, we just enclose all column names in backticks.
                # enclose the first word in the column definition in
                # backticks
                words = str(column).split()
                words[0] = f"`{words[0]}`"
                create_table_stmt = create_table_stmt.replace(
                    f"\n\t{str(column)}", " ".join(words)
                )
            # if any double quotes are used for column names, replace them
            # with backticks
            create_table_stmt = create_table_stmt.replace('"', "") + ";"

            # Store the table schema
            store_db_info(
                dict(table=table.name, create_stmt=create_table_stmt)
            )

            # 2. extract the table data in batches

            # If the table has a `created` column, we use it to sort
            # the rows in the table starting with the oldest rows.
            # This is to ensure that the rows are inserted in the
            # correct order, since some tables have inner foreign key
            # constraints.
            if "created" in table.columns:
                order_by = table.columns["created"]
            else:
                order_by = None

            # Fetch the number of rows in the table
            row_count = conn.scalar(
                select([func.count("*")]).select_from(table)
            )

            # Fetch the data from the table in batches
            batch_size = 50
            for i in range(0, row_count, batch_size):
                rows = conn.execute(
                    table.select()
                    .order_by(order_by)
                    .limit(batch_size)
                    .offset(i)
                ).fetchall()

                store_db_info(
                    dict(
                        table=table.name,
                        data=[row._asdict() for row in rows],
                    ),
                )
create_database(self, database=None, drop=False)

Creates a mysql database.

Parameters:

Name Type Description Default
database Optional[str]

The name of the database to create. If not set, the database name from the configuration will be used.

None
drop bool

Whether to drop the database if it already exists.

False
Source code in zenml/zen_stores/migrations/utils.py
def create_database(
    self,
    database: Optional[str] = None,
    drop: bool = False,
) -> None:
    """Creates a mysql database.

    Args:
        database: The name of the database to create. If not set, the
            database name from the configuration will be used.
        drop: Whether to drop the database if it already exists.
    """
    database = database or self.url.database
    if drop:
        self.drop_database(database=database)

    with self.master_engine.connect() as conn:
        logger.info(f"Creating database '{database}'")
        conn.execute(text(f"CREATE DATABASE IF NOT EXISTS `{database}`"))
create_engine(self, database=None)

Get the SQLAlchemy engine for a database.

Parameters:

Name Type Description Default
database Optional[str]

The name of the database. If not set, a master engine will be returned.

None

Returns:

Type Description
Engine

The SQLAlchemy engine.

Source code in zenml/zen_stores/migrations/utils.py
def create_engine(self, database: Optional[str] = None) -> Engine:
    """Get the SQLAlchemy engine for a database.

    Args:
        database: The name of the database. If not set, a master engine
            will be returned.

    Returns:
        The SQLAlchemy engine.
    """
    url = self.url._replace(database=database)
    return create_engine(
        url=url,
        connect_args=self.connect_args,
        **self.engine_args,
    )
database_exists(self, database=None)

Check if a database exists.

Parameters:

Name Type Description Default
database Optional[str]

The name of the database to check. If not set, the database name from the configuration will be used.

None

Returns:

Type Description
bool

Whether the database exists.

Exceptions:

Type Description
OperationalError

If connecting to the database failed.

Source code in zenml/zen_stores/migrations/utils.py
def database_exists(
    self,
    database: Optional[str] = None,
) -> bool:
    """Check if a database exists.

    Args:
        database: The name of the database to check. If not set, the
            database name from the configuration will be used.

    Returns:
        Whether the database exists.

    Raises:
        OperationalError: If connecting to the database failed.
    """
    database = database or self.url.database

    engine = self.create_engine(database=database)
    try:
        engine.connect()
    except OperationalError as e:
        if self.is_mysql_missing_database_error(e):
            return False
        else:
            logger.exception(
                f"Failed to connect to mysql database `{database}`.",
            )
            raise
    else:
        return True
drop_database(self, database=None)

Drops a mysql database.

Parameters:

Name Type Description Default
database Optional[str]

The name of the database to drop. If not set, the database name from the configuration will be used.

None
Source code in zenml/zen_stores/migrations/utils.py
def drop_database(
    self,
    database: Optional[str] = None,
) -> None:
    """Drops a mysql database.

    Args:
        database: The name of the database to drop. If not set, the
            database name from the configuration will be used.
    """
    database = database or self.url.database
    with self.master_engine.connect() as conn:
        # drop the database if it exists
        logger.info(f"Dropping database '{database}'")
        conn.execute(text(f"DROP DATABASE IF EXISTS `{database}`"))
is_mysql_missing_database_error(error) classmethod

Checks if the given error is due to a missing database.

Parameters:

Name Type Description Default
error OperationalError

The error to check.

required

Returns:

Type Description
bool

If the error because the MySQL database doesn't exist.

Source code in zenml/zen_stores/migrations/utils.py
@classmethod
def is_mysql_missing_database_error(cls, error: OperationalError) -> bool:
    """Checks if the given error is due to a missing database.

    Args:
        error: The error to check.

    Returns:
        If the error because the MySQL database doesn't exist.
    """
    from pymysql.constants.ER import BAD_DB_ERROR

    if not isinstance(error.orig, pymysql.err.OperationalError):
        return False

    error_code = cast(int, error.orig.args[0])
    return error_code == BAD_DB_ERROR
restore_database_from_db(self, backup_db_name)

Restore the database from the backup database.

Parameters:

Name Type Description Default
backup_db_name str

Backup database name to restore from.

required

Exceptions:

Type Description
RuntimeError

If the backup database does not exist.

Source code in zenml/zen_stores/migrations/utils.py
def restore_database_from_db(self, backup_db_name: str) -> None:
    """Restore the database from the backup database.

    Args:
        backup_db_name: Backup database name to restore from.

    Raises:
        RuntimeError: If the backup database does not exist.
    """
    if not self.database_exists(database=backup_db_name):
        raise RuntimeError(
            f"Backup database `{backup_db_name}` does not exist."
        )

    backup_engine = self.create_engine(database=backup_db_name)

    # Drop and re-create the primary database
    self.create_database(
        drop=True,
    )

    self._copy_database(backup_engine, self.engine)

    logger.debug(
        f"Database restored from the `{backup_db_name}` "
        "backup database."
    )
restore_database_from_file(self, dump_file)

Restore the database from a backup dump file.

See the documentation of the backup_database_to_file method for details on the format of the dump file.

Parameters:

Name Type Description Default
dump_file str

The path to the dump file.

required

Exceptions:

Type Description
RuntimeError

If the database cannot be restored successfully.

Source code in zenml/zen_stores/migrations/utils.py
def restore_database_from_file(self, dump_file: str) -> None:
    """Restore the database from a backup dump file.

    See the documentation of the `backup_database_to_file` method for
    details on the format of the dump file.

    Args:
        dump_file: The path to the dump file.

    Raises:
        RuntimeError: If the database cannot be restored successfully.
    """
    if not os.path.exists(dump_file):
        raise RuntimeError(
            f"Database backup file '{dump_file}' does not "
            f"exist or is not accessible."
        )

    if self.url.drivername == "sqlite":
        # For a sqlite database, we just overwrite the database file
        # with the backup file
        assert self.url.database is not None
        shutil.copyfile(
            dump_file,
            self.url.database,
        )
        return

    # read the DB dump file one JSON object at a time
    with open(dump_file, "r") as f:

        def json_load() -> Generator[Dict[str, Any], None, None]:
            """Generator that loads the JSON objects in the dump file.

            Yields:
                The loaded JSON objects.
            """
            buffer = ""
            while True:
                chunk = f.readline()
                if not chunk:
                    break
                buffer += chunk
                if chunk.rstrip() == "}":
                    yield json.loads(buffer)
                    buffer = ""

        # Call the generic restore method with a function that loads the
        # JSON objects from the dump file
        self.restore_database_from_storage(json_load)

    logger.info(f"Database successfully restored from '{dump_file}'")
restore_database_from_memory(self, db_dump)

Restore the database from an in-memory backup.

Parameters:

Name Type Description Default
db_dump List[Dict[str, Any]]

The in-memory database backup to restore from generated by the backup_database_to_memory method.

required

Exceptions:

Type Description
RuntimeError

If the database cannot be restored successfully.

Source code in zenml/zen_stores/migrations/utils.py
def restore_database_from_memory(
    self, db_dump: List[Dict[str, Any]]
) -> None:
    """Restore the database from an in-memory backup.

    Args:
        db_dump: The in-memory database backup to restore from generated
            by the `backup_database_to_memory` method.

    Raises:
        RuntimeError: If the database cannot be restored successfully.
    """
    if self.url.drivername == "sqlite":
        # For a sqlite database, this is not supported.
        raise RuntimeError(
            "In-memory backup is not supported for sqlite databases."
        )

    def load_from_mem() -> Generator[Dict[str, Any], None, None]:
        """Generator that loads the JSON objects from the in-memory backup.

        Yields:
            The loaded JSON objects.
        """
        for obj in db_dump:
            yield obj

    # Call the generic restore method with a function that loads the
    # JSON objects from the in-memory database backup
    self.restore_database_from_storage(load_from_mem)

    logger.info("Database successfully restored from memory")
restore_database_from_storage(self, load_db_info)

Restore the database from a backup storage location.

Restores the database from an abstract storage location. The storage location is specified by a function that is called repeatedly to load the database information from the external storage chunk by chunk. The function must yield a dictionary containing either the table schema or table data. The dictionary contains the following keys:

* `table`: The name of the table.
* `create_stmt`: The table creation statement.
* `data`: A list of rows in the table.

The function must return None when there is no more data to load.

Parameters:

Name Type Description Default
load_db_info Callable[[], Generator[Dict[str, Any], NoneType, NoneType]]

The function to call to load the database information.

required
Source code in zenml/zen_stores/migrations/utils.py
def restore_database_from_storage(
    self, load_db_info: Callable[[], Generator[Dict[str, Any], None, None]]
) -> None:
    """Restore the database from a backup storage location.

    Restores the database from an abstract storage location. The storage
    location is specified by a function that is called repeatedly to
    load the database information from the external storage chunk by chunk.
    The function must yield a dictionary containing either the table schema
    or table data. The dictionary contains the following keys:

        * `table`: The name of the table.
        * `create_stmt`: The table creation statement.
        * `data`: A list of rows in the table.

    The function must return `None` when there is no more data to load.

    Args:
        load_db_info: The function to call to load the database
            information.
    """
    # Drop and re-create the primary database
    self.create_database(
        drop=True,
    )

    metadata = MetaData(bind=self.engine)

    with self.engine.begin() as connection:
        # read the DB information one JSON object at a time
        for table_dump in load_db_info():
            table_name = table_dump["table"]
            if "create_stmt" in table_dump:
                # execute the table creation statement
                connection.execute(text(table_dump["create_stmt"]))
                # Reload the database metadata after creating the table
                metadata.reflect()

            if "data" in table_dump:
                # insert the data into the database
                table = metadata.tables[table_name]
                for row in table_dump["data"]:
                    # Convert column values to the correct type
                    for column in table.columns:
                        # Blob columns are stored as binary strings
                        if column.type.python_type == bytes and isinstance(
                            row[column.name], str
                        ):
                            # Convert the string to bytes
                            row[column.name] = bytes(
                                row[column.name], "utf-8"
                            )

                # Insert the rows into the table
                connection.execute(
                    table.insert().values(table_dump["data"])
                )

rest_zen_store

REST Zen Store implementation.

RestZenStore (BaseZenStore) pydantic-model

Store implementation for accessing data from a REST API.

Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStore(BaseZenStore):
    """Store implementation for accessing data from a REST API."""

    config: RestZenStoreConfiguration
    TYPE: ClassVar[StoreType] = StoreType.REST
    CONFIG_TYPE: ClassVar[Type[StoreConfiguration]] = RestZenStoreConfiguration
    _api_token: Optional[str] = None
    _session: Optional[requests.Session] = None

    # ====================================
    # ZenML Store interface implementation
    # ====================================

    # --------------------------------
    # Initialization and configuration
    # --------------------------------

    def _initialize(self) -> None:
        """Initialize the REST store."""
        client_version = zenml.__version__
        server_version = self.get_store_info().version

        if not DISABLE_CLIENT_SERVER_MISMATCH_WARNING and (
            server_version != client_version
        ):
            logger.warning(
                "Your ZenML client version (%s) does not match the server "
                "version (%s). This version mismatch might lead to errors or "
                "unexpected behavior. \nTo disable this warning message, set "
                "the environment variable `%s=True`",
                client_version,
                server_version,
                ENV_ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING,
            )

    def get_store_info(self) -> ServerModel:
        """Get information about the server.

        Returns:
            Information about the server.
        """
        body = self.get(INFO)
        return ServerModel.parse_obj(body)

    def get_deployment_id(self) -> UUID:
        """Get the ID of the deployment.

        Returns:
            The ID of the deployment.
        """
        return self.get_store_info().id

    # ----------------------------- API Keys -----------------------------

    def create_api_key(
        self, service_account_id: UUID, api_key: APIKeyRequest
    ) -> APIKeyResponse:
        """Create a new API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                create the API key.
            api_key: The API key to create.

        Returns:
            The created API key.
        """
        return self._create_resource(
            resource=api_key,
            route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
            response_model=APIKeyResponse,
        )

    def get_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        hydrate: bool = True,
    ) -> APIKeyResponse:
        """Get an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to fetch
                the API key.
            api_key_name_or_id: The name or ID of the API key to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The API key with the given ID.
        """
        return self._get_resource(
            resource_id=api_key_name_or_id,
            route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
            response_model=APIKeyResponse,
            params={"hydrate": hydrate},
        )

    def set_api_key(self, api_key: str) -> None:
        """Set the API key to use for authentication.

        Args:
            api_key: The API key to use for authentication.
        """
        self.config.api_key = api_key
        self.clear_session()
        # TODO: find a way to persist the API key in the configuration file
        #  without calling _write_config() here.
        # This is the only place where we need to explicitly call
        # _write_config() to persist the global configuration.
        GlobalConfiguration()._write_config()

    def list_api_keys(
        self,
        service_account_id: UUID,
        filter_model: APIKeyFilter,
        hydrate: bool = False,
    ) -> Page[APIKeyResponse]:
        """List all API keys for a service account matching the given filter criteria.

        Args:
            service_account_id: The ID of the service account for which to list
                the API keys.
            filter_model: All filter parameters including pagination
                params
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all API keys matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
            response_model=APIKeyResponse,
            filter_model=filter_model,
            params={"hydrate": hydrate},
        )

    def update_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        api_key_update: APIKeyUpdate,
    ) -> APIKeyResponse:
        """Update an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                update the API key.
            api_key_name_or_id: The name or ID of the API key to update.
            api_key_update: The update request on the API key.

        Returns:
            The updated API key.
        """
        return self._update_resource(
            resource_id=api_key_name_or_id,
            resource_update=api_key_update,
            route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
            response_model=APIKeyResponse,
        )

    def rotate_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        rotate_request: APIKeyRotateRequest,
    ) -> APIKeyResponse:
        """Rotate an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                rotate the API key.
            api_key_name_or_id: The name or ID of the API key to rotate.
            rotate_request: The rotate request on the API key.

        Returns:
            The updated API key.
        """
        response_body = self.put(
            f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}/{str(api_key_name_or_id)}{API_KEY_ROTATE}",
            body=rotate_request,
        )
        return APIKeyResponse.parse_obj(response_body)

    def delete_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
    ) -> None:
        """Delete an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                delete the API key.
            api_key_name_or_id: The name or ID of the API key to delete.
        """
        self._delete_resource(
            resource_id=api_key_name_or_id,
            route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
        )

    # ----------------------------- Artifacts -----------------------------

    def create_artifact(self, artifact: ArtifactRequest) -> ArtifactResponse:
        """Creates a new artifact.

        Args:
            artifact: The artifact to create.

        Returns:
            The newly created artifact.
        """
        return self._create_resource(
            resource=artifact,
            response_model=ArtifactResponse,
            route=ARTIFACTS,
        )

    def get_artifact(
        self, artifact_id: UUID, hydrate: bool = True
    ) -> ArtifactResponse:
        """Gets an artifact.

        Args:
            artifact_id: The ID of the artifact to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact.
        """
        return self._get_resource(
            resource_id=artifact_id,
            route=ARTIFACTS,
            response_model=ArtifactResponse,
            params={"hydrate": hydrate},
        )

    def list_artifacts(
        self, filter_model: ArtifactFilter, hydrate: bool = False
    ) -> Page[ArtifactResponse]:
        """List all artifacts matching the given filter criteria.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all artifacts matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=ARTIFACTS,
            response_model=ArtifactResponse,
            filter_model=filter_model,
            params={"hydrate": hydrate},
        )

    def update_artifact(
        self, artifact_id: UUID, artifact_update: ArtifactUpdate
    ) -> ArtifactResponse:
        """Updates an artifact.

        Args:
            artifact_id: The ID of the artifact to update.
            artifact_update: The update to be applied to the artifact.

        Returns:
            The updated artifact.
        """
        return self._update_resource(
            resource_id=artifact_id,
            resource_update=artifact_update,
            response_model=ArtifactResponse,
            route=ARTIFACTS,
        )

    def delete_artifact(self, artifact_id: UUID) -> None:
        """Deletes an artifact.

        Args:
            artifact_id: The ID of the artifact to delete.
        """
        self._delete_resource(resource_id=artifact_id, route=ARTIFACTS)

    # -------------------- Artifact Versions --------------------

    def create_artifact_version(
        self, artifact_version: ArtifactVersionRequest
    ) -> ArtifactVersionResponse:
        """Creates an artifact version.

        Args:
            artifact_version: The artifact version to create.

        Returns:
            The created artifact version.
        """
        return self._create_resource(
            resource=artifact_version,
            response_model=ArtifactVersionResponse,
            route=ARTIFACT_VERSIONS,
        )

    def get_artifact_version(
        self, artifact_version_id: UUID, hydrate: bool = True
    ) -> ArtifactVersionResponse:
        """Gets an artifact.

        Args:
            artifact_version_id: The ID of the artifact version to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact version.
        """
        return self._get_resource(
            resource_id=artifact_version_id,
            route=ARTIFACT_VERSIONS,
            response_model=ArtifactVersionResponse,
            params={"hydrate": hydrate},
        )

    def list_artifact_versions(
        self,
        artifact_version_filter_model: ArtifactVersionFilter,
        hydrate: bool = False,
    ) -> Page[ArtifactVersionResponse]:
        """List all artifact versions matching the given filter criteria.

        Args:
            artifact_version_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all artifact versions matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=ARTIFACT_VERSIONS,
            response_model=ArtifactVersionResponse,
            filter_model=artifact_version_filter_model,
            params={"hydrate": hydrate},
        )

    def update_artifact_version(
        self,
        artifact_version_id: UUID,
        artifact_version_update: ArtifactVersionUpdate,
    ) -> ArtifactVersionResponse:
        """Updates an artifact version.

        Args:
            artifact_version_id: The ID of the artifact version to update.
            artifact_version_update: The update to be applied to the artifact
                version.

        Returns:
            The updated artifact version.
        """
        return self._update_resource(
            resource_id=artifact_version_id,
            resource_update=artifact_version_update,
            response_model=ArtifactVersionResponse,
            route=ARTIFACT_VERSIONS,
        )

    def delete_artifact_version(self, artifact_version_id: UUID) -> None:
        """Deletes an artifact version.

        Args:
            artifact_version_id: The ID of the artifact version to delete.
        """
        self._delete_resource(
            resource_id=artifact_version_id, route=ARTIFACT_VERSIONS
        )

    def prune_artifact_versions(
        self,
        only_versions: bool = True,
    ) -> None:
        """Prunes unused artifact versions and their artifacts.

        Args:
            only_versions: Only delete artifact versions, keeping artifacts
        """
        self.delete(
            path=ARTIFACT_VERSIONS, params={"only_versions": only_versions}
        )

    # ------------------------ Artifact Visualizations ------------------------

    def get_artifact_visualization(
        self, artifact_visualization_id: UUID, hydrate: bool = True
    ) -> ArtifactVisualizationResponse:
        """Gets an artifact visualization.

        Args:
            artifact_visualization_id: The ID of the artifact visualization to
                get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact visualization.
        """
        return self._get_resource(
            resource_id=artifact_visualization_id,
            route=ARTIFACT_VISUALIZATIONS,
            response_model=ArtifactVisualizationResponse,
            params={"hydrate": hydrate},
        )

    # ------------------------ Code References ------------------------

    def get_code_reference(
        self, code_reference_id: UUID, hydrate: bool = True
    ) -> CodeReferenceResponse:
        """Gets a code reference.

        Args:
            code_reference_id: The ID of the code reference to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The code reference.
        """
        return self._get_resource(
            resource_id=code_reference_id,
            route=CODE_REFERENCES,
            response_model=CodeReferenceResponse,
            params={"hydrate": hydrate},
        )

    # --------------------------- Code Repositories ---------------------------

    def create_code_repository(
        self, code_repository: CodeRepositoryRequest
    ) -> CodeRepositoryResponse:
        """Creates a new code repository.

        Args:
            code_repository: Code repository to be created.

        Returns:
            The newly created code repository.
        """
        return self._create_workspace_scoped_resource(
            resource=code_repository,
            response_model=CodeRepositoryResponse,
            route=CODE_REPOSITORIES,
        )

    def get_code_repository(
        self, code_repository_id: UUID, hydrate: bool = True
    ) -> CodeRepositoryResponse:
        """Gets a specific code repository.

        Args:
            code_repository_id: The ID of the code repository to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested code repository, if it was found.
        """
        return self._get_resource(
            resource_id=code_repository_id,
            route=CODE_REPOSITORIES,
            response_model=CodeRepositoryResponse,
            params={"hydrate": hydrate},
        )

    def list_code_repositories(
        self,
        filter_model: CodeRepositoryFilter,
        hydrate: bool = False,
    ) -> Page[CodeRepositoryResponse]:
        """List all code repositories.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all code repositories.
        """
        return self._list_paginated_resources(
            route=CODE_REPOSITORIES,
            response_model=CodeRepositoryResponse,
            filter_model=filter_model,
            params={"hydrate": hydrate},
        )

    def update_code_repository(
        self, code_repository_id: UUID, update: CodeRepositoryUpdate
    ) -> CodeRepositoryResponse:
        """Updates an existing code repository.

        Args:
            code_repository_id: The ID of the code repository to update.
            update: The update to be applied to the code repository.

        Returns:
            The updated code repository.
        """
        return self._update_resource(
            resource_id=code_repository_id,
            resource_update=update,
            response_model=CodeRepositoryResponse,
            route=CODE_REPOSITORIES,
        )

    def delete_code_repository(self, code_repository_id: UUID) -> None:
        """Deletes a code repository.

        Args:
            code_repository_id: The ID of the code repository to delete.
        """
        self._delete_resource(
            resource_id=code_repository_id, route=CODE_REPOSITORIES
        )

    # ----------------------------- Components -----------------------------

    def create_stack_component(
        self,
        component: ComponentRequest,
    ) -> ComponentResponse:
        """Create a stack component.

        Args:
            component: The stack component to create.

        Returns:
            The created stack component.
        """
        return self._create_workspace_scoped_resource(
            resource=component,
            route=STACK_COMPONENTS,
            response_model=ComponentResponse,
        )

    def get_stack_component(
        self, component_id: UUID, hydrate: bool = True
    ) -> ComponentResponse:
        """Get a stack component by ID.

        Args:
            component_id: The ID of the stack component to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack component.
        """
        return self._get_resource(
            resource_id=component_id,
            route=STACK_COMPONENTS,
            response_model=ComponentResponse,
            params={"hydrate": hydrate},
        )

    def list_stack_components(
        self,
        component_filter_model: ComponentFilter,
        hydrate: bool = False,
    ) -> Page[ComponentResponse]:
        """List all stack components matching the given filter criteria.

        Args:
            component_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all stack components matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=STACK_COMPONENTS,
            response_model=ComponentResponse,
            filter_model=component_filter_model,
            params={"hydrate": hydrate},
        )

    def update_stack_component(
        self,
        component_id: UUID,
        component_update: ComponentUpdate,
    ) -> ComponentResponse:
        """Update an existing stack component.

        Args:
            component_id: The ID of the stack component to update.
            component_update: The update to be applied to the stack component.

        Returns:
            The updated stack component.
        """
        return self._update_resource(
            resource_id=component_id,
            resource_update=component_update,
            route=STACK_COMPONENTS,
            response_model=ComponentResponse,
        )

    def delete_stack_component(self, component_id: UUID) -> None:
        """Delete a stack component.

        Args:
            component_id: The ID of the stack component to delete.
        """
        self._delete_resource(
            resource_id=component_id,
            route=STACK_COMPONENTS,
        )

    #  ----------------------------- Flavors -----------------------------

    def create_flavor(self, flavor: FlavorRequest) -> FlavorResponse:
        """Creates a new stack component flavor.

        Args:
            flavor: The stack component flavor to create.

        Returns:
            The newly created flavor.
        """
        return self._create_resource(
            resource=flavor,
            route=FLAVORS,
            response_model=FlavorResponse,
        )

    def get_flavor(
        self, flavor_id: UUID, hydrate: bool = True
    ) -> FlavorResponse:
        """Get a stack component flavor by ID.

        Args:
            flavor_id: The ID of the stack component flavor to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack component flavor.
        """
        return self._get_resource(
            resource_id=flavor_id,
            route=FLAVORS,
            response_model=FlavorResponse,
            params={"hydrate": hydrate},
        )

    def list_flavors(
        self,
        flavor_filter_model: FlavorFilter,
        hydrate: bool = False,
    ) -> Page[FlavorResponse]:
        """List all stack component flavors matching the given filter criteria.

        Args:
            flavor_filter_model: All filter parameters including pagination
                params
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            List of all the stack component flavors matching the given criteria.
        """
        return self._list_paginated_resources(
            route=FLAVORS,
            response_model=FlavorResponse,
            filter_model=flavor_filter_model,
            params={"hydrate": hydrate},
        )

    def update_flavor(
        self, flavor_id: UUID, flavor_update: FlavorUpdate
    ) -> FlavorResponse:
        """Updates an existing user.

        Args:
            flavor_id: The id of the flavor to update.
            flavor_update: The update to be applied to the flavor.

        Returns:
            The updated flavor.
        """
        return self._update_resource(
            resource_id=flavor_id,
            resource_update=flavor_update,
            route=FLAVORS,
            response_model=FlavorResponse,
        )

    def delete_flavor(self, flavor_id: UUID) -> None:
        """Delete a stack component flavor.

        Args:
            flavor_id: The ID of the stack component flavor to delete.
        """
        self._delete_resource(
            resource_id=flavor_id,
            route=FLAVORS,
        )

    # ------------------------ Logs ------------------------

    def get_logs(self, logs_id: UUID, hydrate: bool = True) -> LogsResponse:
        """Gets logs with the given ID.

        Args:
            logs_id: The ID of the logs to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The logs.
        """
        return self._get_resource(
            resource_id=logs_id,
            route=LOGS,
            response_model=LogsResponse,
            params={"hydrate": hydrate},
        )

    # ----------------------------- Pipelines -----------------------------

    def create_pipeline(self, pipeline: PipelineRequest) -> PipelineResponse:
        """Creates a new pipeline in a workspace.

        Args:
            pipeline: The pipeline to create.

        Returns:
            The newly created pipeline.
        """
        return self._create_workspace_scoped_resource(
            resource=pipeline,
            route=PIPELINES,
            response_model=PipelineResponse,
        )

    def get_pipeline(
        self, pipeline_id: UUID, hydrate: bool = True
    ) -> PipelineResponse:
        """Get a pipeline with a given ID.

        Args:
            pipeline_id: ID of the pipeline.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The pipeline.
        """
        return self._get_resource(
            resource_id=pipeline_id,
            route=PIPELINES,
            response_model=PipelineResponse,
            params={"hydrate": hydrate},
        )

    def list_pipelines(
        self,
        pipeline_filter_model: PipelineFilter,
        hydrate: bool = False,
    ) -> Page[PipelineResponse]:
        """List all pipelines matching the given filter criteria.

        Args:
            pipeline_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all pipelines matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=PIPELINES,
            response_model=PipelineResponse,
            filter_model=pipeline_filter_model,
            params={"hydrate": hydrate},
        )

    def update_pipeline(
        self, pipeline_id: UUID, pipeline_update: PipelineUpdate
    ) -> PipelineResponse:
        """Updates a pipeline.

        Args:
            pipeline_id: The ID of the pipeline to be updated.
            pipeline_update: The update to be applied.

        Returns:
            The updated pipeline.
        """
        return self._update_resource(
            resource_id=pipeline_id,
            resource_update=pipeline_update,
            route=PIPELINES,
            response_model=PipelineResponse,
        )

    def delete_pipeline(self, pipeline_id: UUID) -> None:
        """Deletes a pipeline.

        Args:
            pipeline_id: The ID of the pipeline to delete.
        """
        self._delete_resource(
            resource_id=pipeline_id,
            route=PIPELINES,
        )

    # --------------------------- Pipeline Builds ---------------------------

    def create_build(
        self,
        build: PipelineBuildRequest,
    ) -> PipelineBuildResponse:
        """Creates a new build in a workspace.

        Args:
            build: The build to create.

        Returns:
            The newly created build.
        """
        return self._create_workspace_scoped_resource(
            resource=build,
            route=PIPELINE_BUILDS,
            response_model=PipelineBuildResponse,
        )

    def get_build(
        self, build_id: UUID, hydrate: bool = True
    ) -> PipelineBuildResponse:
        """Get a build with a given ID.

        Args:
            build_id: ID of the build.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The build.
        """
        return self._get_resource(
            resource_id=build_id,
            route=PIPELINE_BUILDS,
            response_model=PipelineBuildResponse,
            params={"hydrate": hydrate},
        )

    def list_builds(
        self,
        build_filter_model: PipelineBuildFilter,
        hydrate: bool = False,
    ) -> Page[PipelineBuildResponse]:
        """List all builds matching the given filter criteria.

        Args:
            build_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all builds matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=PIPELINE_BUILDS,
            response_model=PipelineBuildResponse,
            filter_model=build_filter_model,
            params={"hydrate": hydrate},
        )

    def delete_build(self, build_id: UUID) -> None:
        """Deletes a build.

        Args:
            build_id: The ID of the build to delete.
        """
        self._delete_resource(
            resource_id=build_id,
            route=PIPELINE_BUILDS,
        )

        # ----------------------
        # Pipeline Deployments
        # ----------------------

    # -------------------------- Pipeline Deployments --------------------------

    def create_deployment(
        self,
        deployment: PipelineDeploymentRequest,
    ) -> PipelineDeploymentResponse:
        """Creates a new deployment in a workspace.

        Args:
            deployment: The deployment to create.

        Returns:
            The newly created deployment.
        """
        return self._create_workspace_scoped_resource(
            resource=deployment,
            route=PIPELINE_DEPLOYMENTS,
            response_model=PipelineDeploymentResponse,
        )

    def get_deployment(
        self, deployment_id: UUID, hydrate: bool = True
    ) -> PipelineDeploymentResponse:
        """Get a deployment with a given ID.

        Args:
            deployment_id: ID of the deployment.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The deployment.
        """
        return self._get_resource(
            resource_id=deployment_id,
            route=PIPELINE_DEPLOYMENTS,
            response_model=PipelineDeploymentResponse,
            params={"hydrate": hydrate},
        )

    def list_deployments(
        self,
        deployment_filter_model: PipelineDeploymentFilter,
        hydrate: bool = False,
    ) -> Page[PipelineDeploymentResponse]:
        """List all deployments matching the given filter criteria.

        Args:
            deployment_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all deployments matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=PIPELINE_DEPLOYMENTS,
            response_model=PipelineDeploymentResponse,
            filter_model=deployment_filter_model,
            params={"hydrate": hydrate},
        )

    def delete_deployment(self, deployment_id: UUID) -> None:
        """Deletes a deployment.

        Args:
            deployment_id: The ID of the deployment to delete.
        """
        self._delete_resource(
            resource_id=deployment_id,
            route=PIPELINE_DEPLOYMENTS,
        )

    # -------------------- Event Sources  --------------------

    def create_event_source(
        self, event_source: EventSourceRequest
    ) -> EventSourceResponse:
        """Create an event_source.

        Args:
            event_source: The event_source to create.

        Returns:
            The created event_source.
        """
        return self._create_resource(
            resource=event_source,
            route=EVENT_SOURCES,
            response_model=EventSourceResponse,
        )

    def get_event_source(
        self,
        event_source_id: UUID,
        hydrate: bool = True,
    ) -> EventSourceResponse:
        """Get an event_source by ID.

        Args:
            event_source_id: The ID of the event_source to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The event_source.
        """
        return self._get_resource(
            resource_id=event_source_id,
            route=EVENT_SOURCES,
            response_model=EventSourceResponse,
            params={"hydrate": hydrate},
        )

    def list_event_sources(
        self,
        event_source_filter_model: EventSourceFilter,
        hydrate: bool = False,
    ) -> Page[EventSourceResponse]:
        """List all event_sources matching the given filter criteria.

        Args:
            event_source_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all event_sources matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=EVENT_SOURCES,
            response_model=EventSourceResponse,
            filter_model=event_source_filter_model,
            params={"hydrate": hydrate},
        )

    def update_event_source(
        self,
        event_source_id: UUID,
        event_source_update: EventSourceUpdate,
    ) -> EventSourceResponse:
        """Update an existing event_source.

        Args:
            event_source_id: The ID of the event_source to update.
            event_source_update: The update to be applied to the event_source.

        Returns:
            The updated event_source.
        """
        return self._update_resource(
            resource_id=event_source_id,
            resource_update=event_source_update,
            route=EVENT_SOURCES,
            response_model=EventSourceResponse,
        )

    def delete_event_source(self, event_source_id: UUID) -> None:
        """Delete an event_source.

        Args:
            event_source_id: The ID of the event_source to delete.
        """
        self._delete_resource(
            resource_id=event_source_id,
            route=EVENT_SOURCES,
        )

    # ----------------------------- Pipeline runs -----------------------------

    def create_run(
        self, pipeline_run: PipelineRunRequest
    ) -> PipelineRunResponse:
        """Creates a pipeline run.

        Args:
            pipeline_run: The pipeline run to create.

        Returns:
            The created pipeline run.
        """
        return self._create_workspace_scoped_resource(
            resource=pipeline_run,
            response_model=PipelineRunResponse,
            route=RUNS,
        )

    def get_run(
        self, run_name_or_id: Union[UUID, str], hydrate: bool = True
    ) -> PipelineRunResponse:
        """Gets a pipeline run.

        Args:
            run_name_or_id: The name or ID of the pipeline run to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The pipeline run.
        """
        return self._get_resource(
            resource_id=run_name_or_id,
            route=RUNS,
            response_model=PipelineRunResponse,
            params={"hydrate": hydrate},
        )

    def list_runs(
        self,
        runs_filter_model: PipelineRunFilter,
        hydrate: bool = False,
    ) -> Page[PipelineRunResponse]:
        """List all pipeline runs matching the given filter criteria.

        Args:
            runs_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all pipeline runs matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=RUNS,
            response_model=PipelineRunResponse,
            filter_model=runs_filter_model,
            params={"hydrate": hydrate},
        )

    def update_run(
        self, run_id: UUID, run_update: PipelineRunUpdate
    ) -> PipelineRunResponse:
        """Updates a pipeline run.

        Args:
            run_id: The ID of the pipeline run to update.
            run_update: The update to be applied to the pipeline run.


        Returns:
            The updated pipeline run.
        """
        return self._update_resource(
            resource_id=run_id,
            resource_update=run_update,
            response_model=PipelineRunResponse,
            route=RUNS,
        )

    def delete_run(self, run_id: UUID) -> None:
        """Deletes a pipeline run.

        Args:
            run_id: The ID of the pipeline run to delete.
        """
        self._delete_resource(
            resource_id=run_id,
            route=RUNS,
        )

    def get_or_create_run(
        self, pipeline_run: PipelineRunRequest
    ) -> Tuple[PipelineRunResponse, bool]:
        """Gets or creates a pipeline run.

        If a run with the same ID or name already exists, it is returned.
        Otherwise, a new run is created.

        Args:
            pipeline_run: The pipeline run to get or create.

        Returns:
            The pipeline run, and a boolean indicating whether the run was
            created or not.
        """
        return self._get_or_create_workspace_scoped_resource(
            resource=pipeline_run,
            route=RUNS,
            response_model=PipelineRunResponse,
        )

    # ----------------------------- Run Metadata -----------------------------

    def create_run_metadata(
        self, run_metadata: RunMetadataRequest
    ) -> List[RunMetadataResponse]:
        """Creates run metadata.

        Args:
            run_metadata: The run metadata to create.

        Returns:
            The created run metadata.
        """
        route = f"{WORKSPACES}/{str(run_metadata.workspace)}{RUN_METADATA}"
        response_body = self.post(f"{route}", body=run_metadata)
        result: List[RunMetadataResponse] = []
        if isinstance(response_body, list):
            for metadata in response_body or []:
                result.append(RunMetadataResponse.parse_obj(metadata))
        return result

    def get_run_metadata(
        self, run_metadata_id: UUID, hydrate: bool = True
    ) -> RunMetadataResponse:
        """Gets run metadata with the given ID.

        Args:
            run_metadata_id: The ID of the run metadata to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The run metadata.
        """
        return self._get_resource(
            resource_id=run_metadata_id,
            route=RUN_METADATA,
            response_model=RunMetadataResponse,
            params={"hydrate": hydrate},
        )

    def list_run_metadata(
        self,
        run_metadata_filter_model: RunMetadataFilter,
        hydrate: bool = False,
    ) -> Page[RunMetadataResponse]:
        """List run metadata.

        Args:
            run_metadata_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The run metadata.
        """
        return self._list_paginated_resources(
            route=RUN_METADATA,
            response_model=RunMetadataResponse,
            filter_model=run_metadata_filter_model,
            params={"hydrate": hydrate},
        )

    # ----------------------------- Schedules -----------------------------

    def create_schedule(self, schedule: ScheduleRequest) -> ScheduleResponse:
        """Creates a new schedule.

        Args:
            schedule: The schedule to create.

        Returns:
            The newly created schedule.
        """
        return self._create_workspace_scoped_resource(
            resource=schedule,
            route=SCHEDULES,
            response_model=ScheduleResponse,
        )

    def get_schedule(
        self, schedule_id: UUID, hydrate: bool = True
    ) -> ScheduleResponse:
        """Get a schedule with a given ID.

        Args:
            schedule_id: ID of the schedule.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The schedule.
        """
        return self._get_resource(
            resource_id=schedule_id,
            route=SCHEDULES,
            response_model=ScheduleResponse,
            params={"hydrate": hydrate},
        )

    def list_schedules(
        self,
        schedule_filter_model: ScheduleFilter,
        hydrate: bool = False,
    ) -> Page[ScheduleResponse]:
        """List all schedules in the workspace.

        Args:
            schedule_filter_model: All filter parameters including pagination
                params
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of schedules.
        """
        return self._list_paginated_resources(
            route=SCHEDULES,
            response_model=ScheduleResponse,
            filter_model=schedule_filter_model,
            params={"hydrate": hydrate},
        )

    def update_schedule(
        self,
        schedule_id: UUID,
        schedule_update: ScheduleUpdate,
    ) -> ScheduleResponse:
        """Updates a schedule.

        Args:
            schedule_id: The ID of the schedule to be updated.
            schedule_update: The update to be applied.

        Returns:
            The updated schedule.
        """
        return self._update_resource(
            resource_id=schedule_id,
            resource_update=schedule_update,
            route=SCHEDULES,
            response_model=ScheduleResponse,
        )

    def delete_schedule(self, schedule_id: UUID) -> None:
        """Deletes a schedule.

        Args:
            schedule_id: The ID of the schedule to delete.
        """
        self._delete_resource(
            resource_id=schedule_id,
            route=SCHEDULES,
        )

    # --------------------------- Secrets ---------------------------

    def create_secret(self, secret: SecretRequest) -> SecretResponse:
        """Creates a new secret.

        The new secret is also validated against the scoping rules enforced in
        the secrets store:

          - only one workspace-scoped secret with the given name can exist
            in the target workspace.
          - only one user-scoped secret with the given name can exist in the
            target workspace for the target user.

        Args:
            secret: The secret to create.

        Returns:
            The newly created secret.
        """
        return self._create_workspace_scoped_resource(
            resource=secret,
            route=SECRETS,
            response_model=SecretResponse,
        )

    def get_secret(
        self, secret_id: UUID, hydrate: bool = True
    ) -> SecretResponse:
        """Get a secret by ID.

        Args:
            secret_id: The ID of the secret to fetch.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The secret.
        """
        return self._get_resource(
            resource_id=secret_id,
            route=SECRETS,
            response_model=SecretResponse,
            params={"hydrate": hydrate},
        )

    def list_secrets(
        self, secret_filter_model: SecretFilter, hydrate: bool = False
    ) -> Page[SecretResponse]:
        """List all secrets matching the given filter criteria.

        Note that returned secrets do not include any secret values. To fetch
        the secret values, use `get_secret`.

        Args:
            secret_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all secrets matching the filter criteria, with pagination
            information and sorted according to the filter criteria. The
            returned secrets do not include any secret values, only metadata. To
            fetch the secret values, use `get_secret` individually with each
            secret.
        """
        return self._list_paginated_resources(
            route=SECRETS,
            response_model=SecretResponse,
            filter_model=secret_filter_model,
            params={"hydrate": hydrate},
        )

    def update_secret(
        self, secret_id: UUID, secret_update: SecretUpdate
    ) -> SecretResponse:
        """Updates a secret.

        Secret values that are specified as `None` in the update that are
        present in the existing secret are removed from the existing secret.
        Values that are present in both secrets are overwritten. All other
        values in both the existing secret and the update are kept (merged).

        If the update includes a change of name or scope, the scoping rules
        enforced in the secrets store are used to validate the update:

          - only one workspace-scoped secret with the given name can exist
            in the target workspace.
          - only one user-scoped secret with the given name can exist in the
            target workspace for the target user.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_update: The update to be applied.

        Returns:
            The updated secret.
        """
        return self._update_resource(
            resource_id=secret_id,
            resource_update=secret_update,
            route=SECRETS,
            response_model=SecretResponse,
            # The default endpoint behavior is to replace all secret values
            # with the values in the update. We want to merge the values
            # instead.
            params=dict(patch_values=True),
        )

    def delete_secret(self, secret_id: UUID) -> None:
        """Delete a secret.

        Args:
            secret_id: The id of the secret to delete.
        """
        self._delete_resource(
            resource_id=secret_id,
            route=SECRETS,
        )

    def backup_secrets(
        self, ignore_errors: bool = True, delete_secrets: bool = False
    ) -> None:
        """Backs up all secrets to the configured backup secrets store.

        Args:
            ignore_errors: Whether to ignore individual errors during the backup
                process and attempt to backup all secrets.
            delete_secrets: Whether to delete the secrets that have been
                successfully backed up from the primary secrets store. Setting
                this flag effectively moves all secrets from the primary secrets
                store to the backup secrets store.
        """
        params: Dict[str, Any] = {
            "ignore_errors": ignore_errors,
            "delete_secrets": delete_secrets,
        }
        self.put(
            f"{SECRETS_OPERATIONS}{SECRETS_BACKUP}",
            params=params,
        )

    def restore_secrets(
        self, ignore_errors: bool = False, delete_secrets: bool = False
    ) -> None:
        """Restore all secrets from the configured backup secrets store.

        Args:
            ignore_errors: Whether to ignore individual errors during the
                restore process and attempt to restore all secrets.
            delete_secrets: Whether to delete the secrets that have been
                successfully restored from the backup secrets store. Setting
                this flag effectively moves all secrets from the backup secrets
                store to the primary secrets store.
        """
        params: Dict[str, Any] = {
            "ignore_errors": ignore_errors,
            "delete_secrets": delete_secrets,
        }
        self.put(
            f"{SECRETS_OPERATIONS}{SECRETS_RESTORE}",
            params=params,
        )

    # --------------------------- Service Accounts ---------------------------

    def create_service_account(
        self, service_account: ServiceAccountRequest
    ) -> ServiceAccountResponse:
        """Creates a new service account.

        Args:
            service_account: Service account to be created.

        Returns:
            The newly created service account.
        """
        return self._create_resource(
            resource=service_account,
            route=SERVICE_ACCOUNTS,
            response_model=ServiceAccountResponse,
        )

    def get_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
        hydrate: bool = True,
    ) -> ServiceAccountResponse:
        """Gets a specific service account.

        Args:
            service_account_name_or_id: The name or ID of the service account to
                get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested service account, if it was found.
        """
        return self._get_resource(
            resource_id=service_account_name_or_id,
            route=SERVICE_ACCOUNTS,
            response_model=ServiceAccountResponse,
            params={"hydrate": hydrate},
        )

    def list_service_accounts(
        self, filter_model: ServiceAccountFilter, hydrate: bool = False
    ) -> Page[ServiceAccountResponse]:
        """List all service accounts.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of filtered service accounts.
        """
        return self._list_paginated_resources(
            route=SERVICE_ACCOUNTS,
            response_model=ServiceAccountResponse,
            filter_model=filter_model,
            params={"hydrate": hydrate},
        )

    def update_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
        service_account_update: ServiceAccountUpdate,
    ) -> ServiceAccountResponse:
        """Updates an existing service account.

        Args:
            service_account_name_or_id: The name or the ID of the service
                account to update.
            service_account_update: The update to be applied to the service
                account.

        Returns:
            The updated service account.
        """
        return self._update_resource(
            resource_id=service_account_name_or_id,
            resource_update=service_account_update,
            route=SERVICE_ACCOUNTS,
            response_model=ServiceAccountResponse,
        )

    def delete_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
    ) -> None:
        """Delete a service account.

        Args:
            service_account_name_or_id: The name or the ID of the service
                account to delete.
        """
        self._delete_resource(
            resource_id=service_account_name_or_id,
            route=SERVICE_ACCOUNTS,
        )

    # --------------------------- Service Connectors ---------------------------

    def create_service_connector(
        self, service_connector: ServiceConnectorRequest
    ) -> ServiceConnectorResponse:
        """Creates a new service connector.

        Args:
            service_connector: Service connector to be created.

        Returns:
            The newly created service connector.
        """
        connector_model = self._create_workspace_scoped_resource(
            resource=service_connector,
            route=SERVICE_CONNECTORS,
            response_model=ServiceConnectorResponse,
        )
        self._populate_connector_type(connector_model)
        return connector_model

    def get_service_connector(
        self, service_connector_id: UUID, hydrate: bool = True
    ) -> ServiceConnectorResponse:
        """Gets a specific service connector.

        Args:
            service_connector_id: The ID of the service connector to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested service connector, if it was found.
        """
        connector_model = self._get_resource(
            resource_id=service_connector_id,
            route=SERVICE_CONNECTORS,
            response_model=ServiceConnectorResponse,
            params={"expand_secrets": False, "hydrate": hydrate},
        )
        self._populate_connector_type(connector_model)
        return connector_model

    def list_service_connectors(
        self,
        filter_model: ServiceConnectorFilter,
        hydrate: bool = False,
    ) -> Page[ServiceConnectorResponse]:
        """List all service connectors.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all service connectors.
        """
        connector_models = self._list_paginated_resources(
            route=SERVICE_CONNECTORS,
            response_model=ServiceConnectorResponse,
            filter_model=filter_model,
            params={"expand_secrets": False, "hydrate": hydrate},
        )
        self._populate_connector_type(*connector_models.items)
        return connector_models

    def update_service_connector(
        self, service_connector_id: UUID, update: ServiceConnectorUpdate
    ) -> ServiceConnectorResponse:
        """Updates an existing service connector.

        The update model contains the fields to be updated. If a field value is
        set to None in the model, the field is not updated, but there are
        special rules concerning some fields:

        * the `configuration` and `secrets` fields together represent a full
        valid configuration update, not just a partial update. If either is
        set (i.e. not None) in the update, their values are merged together and
        will replace the existing configuration and secrets values.
        * the `resource_id` field value is also a full replacement value: if set
        to `None`, the resource ID is removed from the service connector.
        * the `expiration_seconds` field value is also a full replacement value:
        if set to `None`, the expiration is removed from the service connector.
        * the `secret_id` field value in the update is ignored, given that
        secrets are managed internally by the ZenML store.
        * the `labels` field is also a full labels update: if set (i.e. not
        `None`), all existing labels are removed and replaced by the new labels
        in the update.

        Args:
            service_connector_id: The ID of the service connector to update.
            update: The update to be applied to the service connector.

        Returns:
            The updated service connector.
        """
        connector_model = self._update_resource(
            resource_id=service_connector_id,
            resource_update=update,
            response_model=ServiceConnectorResponse,
            route=SERVICE_CONNECTORS,
        )
        self._populate_connector_type(connector_model)
        return connector_model

    def delete_service_connector(self, service_connector_id: UUID) -> None:
        """Deletes a service connector.

        Args:
            service_connector_id: The ID of the service connector to delete.
        """
        self._delete_resource(
            resource_id=service_connector_id, route=SERVICE_CONNECTORS
        )

    def _populate_connector_type(
        self,
        *connector_models: Union[
            ServiceConnectorResponse, ServiceConnectorResourcesModel
        ],
    ) -> None:
        """Populates or updates the connector type of the given connector or resource models.

        If the connector type is not locally available, the connector type
        field is left as is. The local and remote flags of the connector type
        are updated accordingly.

        Args:
            connector_models: The service connector or resource models to
                populate.
        """
        for service_connector in connector_models:
            # Mark the remote connector type as being only remotely available
            if not isinstance(service_connector.connector_type, str):
                service_connector.connector_type.local = False
                service_connector.connector_type.remote = True

            if not service_connector_registry.is_registered(
                service_connector.type
            ):
                continue

            connector_type = (
                service_connector_registry.get_service_connector_type(
                    service_connector.type
                )
            )
            connector_type.local = True
            if not isinstance(service_connector.connector_type, str):
                connector_type.remote = True

            # TODO: Normally, this could have been handled with setter
            #   functions over the connector type property in the response
            #   model. However, pydantic breaks property setter functions.
            #   We can find a more elegant solution here.
            if isinstance(service_connector, ServiceConnectorResponse):
                service_connector.set_connector_type(connector_type)
            elif isinstance(service_connector, ServiceConnectorResourcesModel):
                service_connector.connector_type = connector_type
            else:
                TypeError(
                    "The service connector must be an instance of either"
                    "`ServiceConnectorResponse` or "
                    "`ServiceConnectorResourcesModel`."
                )

    def verify_service_connector_config(
        self,
        service_connector: ServiceConnectorRequest,
        list_resources: bool = True,
    ) -> ServiceConnectorResourcesModel:
        """Verifies if a service connector configuration has access to resources.

        Args:
            service_connector: The service connector configuration to verify.
            list_resources: If True, the list of all resources accessible
                through the service connector and matching the supplied resource
                type and ID are returned.

        Returns:
            The list of resources that the service connector configuration has
            access to.
        """
        response_body = self.post(
            f"{SERVICE_CONNECTORS}{SERVICE_CONNECTOR_VERIFY}",
            body=service_connector,
            params={"list_resources": list_resources},
        )

        resources = ServiceConnectorResourcesModel.parse_obj(response_body)
        self._populate_connector_type(resources)
        return resources

    def verify_service_connector(
        self,
        service_connector_id: UUID,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
        list_resources: bool = True,
    ) -> ServiceConnectorResourcesModel:
        """Verifies if a service connector instance has access to one or more resources.

        Args:
            service_connector_id: The ID of the service connector to verify.
            resource_type: The type of resource to verify access to.
            resource_id: The ID of the resource to verify access to.
            list_resources: If True, the list of all resources accessible
                through the service connector and matching the supplied resource
                type and ID are returned.

        Returns:
            The list of resources that the service connector has access to,
            scoped to the supplied resource type and ID, if provided.
        """
        params: Dict[str, Any] = {"list_resources": list_resources}
        if resource_type:
            params["resource_type"] = resource_type
        if resource_id:
            params["resource_id"] = resource_id
        response_body = self.put(
            f"{SERVICE_CONNECTORS}/{str(service_connector_id)}{SERVICE_CONNECTOR_VERIFY}",
            params=params,
        )

        resources = ServiceConnectorResourcesModel.parse_obj(response_body)
        self._populate_connector_type(resources)
        return resources

    def get_service_connector_client(
        self,
        service_connector_id: UUID,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
    ) -> ServiceConnectorResponse:
        """Get a service connector client for a service connector and given resource.

        Args:
            service_connector_id: The ID of the base service connector to use.
            resource_type: The type of resource to get a client for.
            resource_id: The ID of the resource to get a client for.

        Returns:
            A service connector client that can be used to access the given
            resource.
        """
        params = {}
        if resource_type:
            params["resource_type"] = resource_type
        if resource_id:
            params["resource_id"] = resource_id
        response_body = self.get(
            f"{SERVICE_CONNECTORS}/{str(service_connector_id)}{SERVICE_CONNECTOR_CLIENT}",
            params=params,
        )

        connector = ServiceConnectorResponse.parse_obj(response_body)
        self._populate_connector_type(connector)
        return connector

    def list_service_connector_resources(
        self,
        workspace_name_or_id: Union[str, UUID],
        connector_type: Optional[str] = None,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
    ) -> List[ServiceConnectorResourcesModel]:
        """List resources that can be accessed by service connectors.

        Args:
            workspace_name_or_id: The name or ID of the workspace to scope to.
            connector_type: The type of service connector to scope to.
            resource_type: The type of resource to scope to.
            resource_id: The ID of the resource to scope to.

        Returns:
            The matching list of resources that available service
            connectors have access to.
        """
        params = {}
        if connector_type:
            params["connector_type"] = connector_type
        if resource_type:
            params["resource_type"] = resource_type
        if resource_id:
            params["resource_id"] = resource_id
        response_body = self.get(
            f"{WORKSPACES}/{workspace_name_or_id}{SERVICE_CONNECTORS}{SERVICE_CONNECTOR_RESOURCES}",
            params=params,
        )

        assert isinstance(response_body, list)
        resource_list = [
            ServiceConnectorResourcesModel.parse_obj(item)
            for item in response_body
        ]

        self._populate_connector_type(*resource_list)

        # For service connectors with types that are only locally available,
        # we need to retrieve the resource list locally
        for idx, resources in enumerate(resource_list):
            if isinstance(resources.connector_type, str):
                # Skip connector types that are neither locally nor remotely
                # available
                continue
            if resources.connector_type.remote:
                # Skip connector types that are remotely available
                continue

            # Retrieve the resource list locally
            assert resources.id is not None
            connector = self.get_service_connector(resources.id)
            connector_instance = (
                service_connector_registry.instantiate_connector(
                    model=connector
                )
            )

            try:
                local_resources = connector_instance.verify(
                    resource_type=resource_type,
                    resource_id=resource_id,
                )
            except (ValueError, AuthorizationException) as e:
                logger.error(
                    f'Failed to fetch {resource_type or "available"} '
                    f"resources from service connector {connector.name}/"
                    f"{connector.id}: {e}"
                )
                continue

            resource_list[idx] = local_resources

        return resource_list

    def list_service_connector_types(
        self,
        connector_type: Optional[str] = None,
        resource_type: Optional[str] = None,
        auth_method: Optional[str] = None,
    ) -> List[ServiceConnectorTypeModel]:
        """Get a list of service connector types.

        Args:
            connector_type: Filter by connector type.
            resource_type: Filter by resource type.
            auth_method: Filter by authentication method.

        Returns:
            List of service connector types.
        """
        params = {}
        if connector_type:
            params["connector_type"] = connector_type
        if resource_type:
            params["resource_type"] = resource_type
        if auth_method:
            params["auth_method"] = auth_method
        response_body = self.get(
            SERVICE_CONNECTOR_TYPES,
            params=params,
        )

        assert isinstance(response_body, list)
        remote_connector_types = [
            ServiceConnectorTypeModel.parse_obj(item) for item in response_body
        ]

        # Mark the remote connector types as being only remotely available
        for c in remote_connector_types:
            c.local = False
            c.remote = True

        local_connector_types = (
            service_connector_registry.list_service_connector_types(
                connector_type=connector_type,
                resource_type=resource_type,
                auth_method=auth_method,
            )
        )

        # Add the connector types in the local registry to the list of
        # connector types available remotely. Overwrite those that have
        # the same connector type but mark them as being remotely available.
        connector_types_map = {
            connector_type.connector_type: connector_type
            for connector_type in remote_connector_types
        }

        for connector in local_connector_types:
            if connector.connector_type in connector_types_map:
                connector.remote = True
            connector_types_map[connector.connector_type] = connector

        return list(connector_types_map.values())

    def get_service_connector_type(
        self,
        connector_type: str,
    ) -> ServiceConnectorTypeModel:
        """Returns the requested service connector type.

        Args:
            connector_type: the service connector type identifier.

        Returns:
            The requested service connector type.
        """
        # Use the local registry to get the service connector type, if it
        # exists.
        local_connector_type: Optional[ServiceConnectorTypeModel] = None
        if service_connector_registry.is_registered(connector_type):
            local_connector_type = (
                service_connector_registry.get_service_connector_type(
                    connector_type
                )
            )
        try:
            response_body = self.get(
                f"{SERVICE_CONNECTOR_TYPES}/{connector_type}",
            )
            remote_connector_type = ServiceConnectorTypeModel.parse_obj(
                response_body
            )
            if local_connector_type:
                # If locally available, return the local connector type but
                # mark it as being remotely available.
                local_connector_type.remote = True
                return local_connector_type

            # Mark the remote connector type as being only remotely available
            remote_connector_type.local = False
            remote_connector_type.remote = True

            return remote_connector_type
        except KeyError:
            # If the service connector type is not found, check the local
            # registry.
            return service_connector_registry.get_service_connector_type(
                connector_type
            )

    # ----------------------------- Stacks -----------------------------

    def create_stack(self, stack: StackRequest) -> StackResponse:
        """Register a new stack.

        Args:
            stack: The stack to register.

        Returns:
            The registered stack.
        """
        return self._create_workspace_scoped_resource(
            resource=stack,
            route=STACKS,
            response_model=StackResponse,
        )

    def get_stack(self, stack_id: UUID, hydrate: bool = True) -> StackResponse:
        """Get a stack by its unique ID.

        Args:
            stack_id: The ID of the stack to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack with the given ID.
        """
        return self._get_resource(
            resource_id=stack_id,
            route=STACKS,
            response_model=StackResponse,
            params={"hydrate": hydrate},
        )

    def list_stacks(
        self, stack_filter_model: StackFilter, hydrate: bool = False
    ) -> Page[StackResponse]:
        """List all stacks matching the given filter criteria.

        Args:
            stack_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all stacks matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=STACKS,
            response_model=StackResponse,
            filter_model=stack_filter_model,
            params={"hydrate": hydrate},
        )

    def update_stack(
        self, stack_id: UUID, stack_update: StackUpdate
    ) -> StackResponse:
        """Update a stack.

        Args:
            stack_id: The ID of the stack update.
            stack_update: The update request on the stack.

        Returns:
            The updated stack.
        """
        return self._update_resource(
            resource_id=stack_id,
            resource_update=stack_update,
            route=STACKS,
            response_model=StackResponse,
        )

    def delete_stack(self, stack_id: UUID) -> None:
        """Delete a stack.

        Args:
            stack_id: The ID of the stack to delete.
        """
        self._delete_resource(
            resource_id=stack_id,
            route=STACKS,
        )

    # ----------------------------- Step runs -----------------------------

    def create_run_step(self, step_run: StepRunRequest) -> StepRunResponse:
        """Creates a step run.

        Args:
            step_run: The step run to create.

        Returns:
            The created step run.
        """
        return self._create_resource(
            resource=step_run,
            response_model=StepRunResponse,
            route=STEPS,
        )

    def get_run_step(
        self, step_run_id: UUID, hydrate: bool = True
    ) -> StepRunResponse:
        """Get a step run by ID.

        Args:
            step_run_id: The ID of the step run to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The step run.
        """
        return self._get_resource(
            resource_id=step_run_id,
            route=STEPS,
            response_model=StepRunResponse,
            params={"hydrate": hydrate},
        )

    def list_run_steps(
        self,
        step_run_filter_model: StepRunFilter,
        hydrate: bool = False,
    ) -> Page[StepRunResponse]:
        """List all step runs matching the given filter criteria.

        Args:
            step_run_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all step runs matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=STEPS,
            response_model=StepRunResponse,
            filter_model=step_run_filter_model,
            params={"hydrate": hydrate},
        )

    def update_run_step(
        self,
        step_run_id: UUID,
        step_run_update: StepRunUpdate,
    ) -> StepRunResponse:
        """Updates a step run.

        Args:
            step_run_id: The ID of the step to update.
            step_run_update: The update to be applied to the step.

        Returns:
            The updated step run.
        """
        return self._update_resource(
            resource_id=step_run_id,
            resource_update=step_run_update,
            response_model=StepRunResponse,
            route=STEPS,
        )

    # -------------------- Triggers  --------------------

    def create_trigger(self, trigger: TriggerRequest) -> TriggerResponse:
        """Create an trigger.

        Args:
            trigger: The trigger to create.

        Returns:
            The created trigger.
        """
        return self._create_resource(
            resource=trigger,
            route=TRIGGERS,
            response_model=TriggerResponse,
        )

    def get_trigger(
        self,
        trigger_id: UUID,
        hydrate: bool = True,
    ) -> TriggerResponse:
        """Get a trigger by ID.

        Args:
            trigger_id: The ID of the trigger to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The trigger.
        """
        return self._get_resource(
            resource_id=trigger_id,
            route=TRIGGERS,
            response_model=TriggerResponse,
            params={"hydrate": hydrate},
        )

    def list_triggers(
        self,
        trigger_filter_model: TriggerFilter,
        hydrate: bool = False,
    ) -> Page[TriggerResponse]:
        """List all triggers matching the given filter criteria.

        Args:
            trigger_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all triggers matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=TRIGGERS,
            response_model=TriggerResponse,
            filter_model=trigger_filter_model,
            params={"hydrate": hydrate},
        )

    def update_trigger(
        self,
        trigger_id: UUID,
        trigger_update: TriggerUpdate,
    ) -> TriggerResponse:
        """Update an existing trigger.

        Args:
            trigger_id: The ID of the trigger to update.
            trigger_update: The update to be applied to the trigger.

        Returns:
            The updated trigger.
        """
        return self._update_resource(
            resource_id=trigger_id,
            resource_update=trigger_update,
            route=TRIGGERS,
            response_model=TriggerResponse,
        )

    def delete_trigger(self, trigger_id: UUID) -> None:
        """Delete an trigger.

        Args:
            trigger_id: The ID of the trigger to delete.
        """
        self._delete_resource(
            resource_id=trigger_id,
            route=TRIGGERS,
        )

    # -------------------- Trigger Executions --------------------

    def get_trigger_execution(
        self,
        trigger_execution_id: UUID,
        hydrate: bool = True,
    ) -> TriggerExecutionResponse:
        """Get an trigger execution by ID.

        Args:
            trigger_execution_id: The ID of the trigger execution to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The trigger execution.
        """
        return self._get_resource(
            resource_id=trigger_execution_id,
            route=TRIGGER_EXECUTIONS,
            response_model=TriggerExecutionResponse,
            params={"hydrate": hydrate},
        )

    def list_trigger_executions(
        self,
        trigger_execution_filter_model: TriggerExecutionFilter,
        hydrate: bool = False,
    ) -> Page[TriggerExecutionResponse]:
        """List all trigger executions matching the given filter criteria.

        Args:
            trigger_execution_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all trigger executions matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=TRIGGER_EXECUTIONS,
            response_model=TriggerExecutionResponse,
            filter_model=trigger_execution_filter_model,
            params={"hydrate": hydrate},
        )

    def delete_trigger_execution(self, trigger_execution_id: UUID) -> None:
        """Delete a trigger execution.

        Args:
            trigger_execution_id: The ID of the trigger execution to delete.
        """
        self._delete_resource(
            resource_id=trigger_execution_id,
            route=TRIGGER_EXECUTIONS,
        )

    # ----------------------------- Users -----------------------------

    def create_user(self, user: UserRequest) -> UserResponse:
        """Creates a new user.

        Args:
            user: User to be created.

        Returns:
            The newly created user.
        """
        return self._create_resource(
            resource=user,
            route=USERS,
            response_model=UserResponse,
        )

    def get_user(
        self,
        user_name_or_id: Optional[Union[str, UUID]] = None,
        include_private: bool = False,
        hydrate: bool = True,
    ) -> UserResponse:
        """Gets a specific user, when no id is specified get the active user.

        The `include_private` parameter is ignored here as it is handled
        implicitly by the /current-user endpoint that is queried when no
        user_name_or_id is set. Raises a KeyError in case a user with that id
        does not exist.

        Args:
            user_name_or_id: The name or ID of the user to get.
            include_private: Whether to include private user information.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested user, if it was found.
        """
        if user_name_or_id:
            return self._get_resource(
                resource_id=user_name_or_id,
                route=USERS,
                response_model=UserResponse,
                params={"hydrate": hydrate},
            )
        else:
            body = self.get(CURRENT_USER, params={"hydrate": hydrate})
            return UserResponse.parse_obj(body)

    def list_users(
        self,
        user_filter_model: UserFilter,
        hydrate: bool = False,
    ) -> Page[UserResponse]:
        """List all users.

        Args:
            user_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all users.
        """
        return self._list_paginated_resources(
            route=USERS,
            response_model=UserResponse,
            filter_model=user_filter_model,
            params={"hydrate": hydrate},
        )

    def update_user(
        self, user_id: UUID, user_update: UserUpdate
    ) -> UserResponse:
        """Updates an existing user.

        Args:
            user_id: The id of the user to update.
            user_update: The update to be applied to the user.

        Returns:
            The updated user.
        """
        return self._update_resource(
            resource_id=user_id,
            resource_update=user_update,
            route=USERS,
            response_model=UserResponse,
        )

    def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
        """Deletes a user.

        Args:
            user_name_or_id: The name or ID of the user to delete.
        """
        self._delete_resource(
            resource_id=user_name_or_id,
            route=USERS,
        )

    # ----------------------------- Workspaces -----------------------------

    def create_workspace(
        self, workspace: WorkspaceRequest
    ) -> WorkspaceResponse:
        """Creates a new workspace.

        Args:
            workspace: The workspace to create.

        Returns:
            The newly created workspace.
        """
        return self._create_resource(
            resource=workspace,
            route=WORKSPACES,
            response_model=WorkspaceResponse,
        )

    def get_workspace(
        self, workspace_name_or_id: Union[UUID, str], hydrate: bool = True
    ) -> WorkspaceResponse:
        """Get an existing workspace by name or ID.

        Args:
            workspace_name_or_id: Name or ID of the workspace to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested workspace.
        """
        return self._get_resource(
            resource_id=workspace_name_or_id,
            route=WORKSPACES,
            response_model=WorkspaceResponse,
            params={"hydrate": hydrate},
        )

    def list_workspaces(
        self,
        workspace_filter_model: WorkspaceFilter,
        hydrate: bool = False,
    ) -> Page[WorkspaceResponse]:
        """List all workspace matching the given filter criteria.

        Args:
            workspace_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all workspace matching the filter criteria.
        """
        return self._list_paginated_resources(
            route=WORKSPACES,
            response_model=WorkspaceResponse,
            filter_model=workspace_filter_model,
            params={"hydrate": hydrate},
        )

    def update_workspace(
        self, workspace_id: UUID, workspace_update: WorkspaceUpdate
    ) -> WorkspaceResponse:
        """Update an existing workspace.

        Args:
            workspace_id: The ID of the workspace to be updated.
            workspace_update: The update to be applied to the workspace.

        Returns:
            The updated workspace.
        """
        return self._update_resource(
            resource_id=workspace_id,
            resource_update=workspace_update,
            route=WORKSPACES,
            response_model=WorkspaceResponse,
        )

    def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
        """Deletes a workspace.

        Args:
            workspace_name_or_id: Name or ID of the workspace to delete.
        """
        self._delete_resource(
            resource_id=workspace_name_or_id,
            route=WORKSPACES,
        )

    # --------------------------- Model ---------------------------

    def create_model(self, model: ModelRequest) -> ModelResponse:
        """Creates a new model.

        Args:
            model: the Model to be created.

        Returns:
            The newly created model.
        """
        return self._create_workspace_scoped_resource(
            resource=model,
            response_model=ModelResponse,
            route=MODELS,
        )

    def delete_model(self, model_name_or_id: Union[str, UUID]) -> None:
        """Deletes a model.

        Args:
            model_name_or_id: name or id of the model to be deleted.
        """
        self._delete_resource(resource_id=model_name_or_id, route=MODELS)

    def update_model(
        self,
        model_id: UUID,
        model_update: ModelUpdate,
    ) -> ModelResponse:
        """Updates an existing model.

        Args:
            model_id: UUID of the model to be updated.
            model_update: the Model to be updated.

        Returns:
            The updated model.
        """
        return self._update_resource(
            resource_id=model_id,
            resource_update=model_update,
            route=MODELS,
            response_model=ModelResponse,
        )

    def get_model(
        self, model_name_or_id: Union[str, UUID], hydrate: bool = True
    ) -> ModelResponse:
        """Get an existing model.

        Args:
            model_name_or_id: name or id of the model to be retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The model of interest.
        """
        return self._get_resource(
            resource_id=model_name_or_id,
            route=MODELS,
            response_model=ModelResponse,
            params={"hydrate": hydrate},
        )

    def list_models(
        self,
        model_filter_model: ModelFilter,
        hydrate: bool = False,
    ) -> Page[ModelResponse]:
        """Get all models by filter.

        Args:
            model_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all models.
        """
        return self._list_paginated_resources(
            route=MODELS,
            response_model=ModelResponse,
            filter_model=model_filter_model,
            params={"hydrate": hydrate},
        )

    # ----------------------------- Model Versions -----------------------------

    def create_model_version(
        self, model_version: ModelVersionRequest
    ) -> ModelVersionResponse:
        """Creates a new model version.

        Args:
            model_version: the Model Version to be created.

        Returns:
            The newly created model version.
        """
        return self._create_workspace_scoped_resource(
            resource=model_version,
            response_model=ModelVersionResponse,
            route=f"{MODELS}/{model_version.model}{MODEL_VERSIONS}",
        )

    def delete_model_version(
        self,
        model_version_id: UUID,
    ) -> None:
        """Deletes a model version.

        Args:
            model_version_id: name or id of the model version to be deleted.
        """
        self._delete_resource(
            resource_id=model_version_id,
            route=f"{MODEL_VERSIONS}",
        )

    def get_model_version(
        self, model_version_id: UUID, hydrate: bool = True
    ) -> ModelVersionResponse:
        """Get an existing model version.

        Args:
            model_version_id: name, id, stage or number of the model version to
                be retrieved. If skipped - latest is retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The model version of interest.
        """
        return self._get_resource(
            resource_id=model_version_id,
            route=MODEL_VERSIONS,
            response_model=ModelVersionResponse,
            params={"hydrate": hydrate},
        )

    def list_model_versions(
        self,
        model_version_filter_model: ModelVersionFilter,
        model_name_or_id: Optional[Union[str, UUID]] = None,
        hydrate: bool = False,
    ) -> Page[ModelVersionResponse]:
        """Get all model versions by filter.

        Args:
            model_name_or_id: name or id of the model containing the model
                versions.
            model_version_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model versions.
        """
        if model_name_or_id:
            return self._list_paginated_resources(
                route=f"{MODELS}/{model_name_or_id}{MODEL_VERSIONS}",
                response_model=ModelVersionResponse,
                filter_model=model_version_filter_model,
                params={"hydrate": hydrate},
            )
        else:
            return self._list_paginated_resources(
                route=MODEL_VERSIONS,
                response_model=ModelVersionResponse,
                filter_model=model_version_filter_model,
                params={"hydrate": hydrate},
            )

    def update_model_version(
        self,
        model_version_id: UUID,
        model_version_update_model: ModelVersionUpdate,
    ) -> ModelVersionResponse:
        """Get all model versions by filter.

        Args:
            model_version_id: The ID of model version to be updated.
            model_version_update_model: The model version to be updated.

        Returns:
            An updated model version.

        """
        return self._update_resource(
            resource_id=model_version_id,
            resource_update=model_version_update_model,
            route=MODEL_VERSIONS,
            response_model=ModelVersionResponse,
        )

    # ------------------------ Model Versions Artifacts ------------------------

    def create_model_version_artifact_link(
        self, model_version_artifact_link: ModelVersionArtifactRequest
    ) -> ModelVersionArtifactResponse:
        """Creates a new model version link.

        Args:
            model_version_artifact_link: the Model Version to Artifact Link
                to be created.

        Returns:
            The newly created model version to artifact link.
        """
        return self._create_workspace_scoped_resource(
            resource=model_version_artifact_link,
            response_model=ModelVersionArtifactResponse,
            route=f"{MODEL_VERSIONS}/{model_version_artifact_link.model_version}{ARTIFACTS}",
        )

    def list_model_version_artifact_links(
        self,
        model_version_artifact_link_filter_model: ModelVersionArtifactFilter,
        hydrate: bool = False,
    ) -> Page[ModelVersionArtifactResponse]:
        """Get all model version to artifact links by filter.

        Args:
            model_version_artifact_link_filter_model: All filter parameters
                including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model version to artifact links.
        """
        return self._list_paginated_resources(
            route=MODEL_VERSION_ARTIFACTS,
            response_model=ModelVersionArtifactResponse,
            filter_model=model_version_artifact_link_filter_model,
            params={"hydrate": hydrate},
        )

    def delete_model_version_artifact_link(
        self,
        model_version_id: UUID,
        model_version_artifact_link_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a model version to artifact link.

        Args:
            model_version_id: ID of the model version containing the link.
            model_version_artifact_link_name_or_id: name or ID of the model
                version to artifact link to be deleted.
        """
        self._delete_resource(
            resource_id=model_version_artifact_link_name_or_id,
            route=f"{MODEL_VERSIONS}/{model_version_id}{ARTIFACTS}",
        )

    def delete_all_model_version_artifact_links(
        self,
        model_version_id: UUID,
        only_links: bool = True,
    ) -> None:
        """Deletes all links between model version and an artifact.

        Args:
            model_version_id: ID of the model version containing the link.
            only_links: Flag deciding whether to delete only links or all.
        """
        self.delete(
            f"{MODEL_VERSIONS}/{model_version_id}{ARTIFACTS}",
            params={"only_links": only_links},
        )

    # ---------------------- Model Versions Pipeline Runs ----------------------

    def create_model_version_pipeline_run_link(
        self,
        model_version_pipeline_run_link: ModelVersionPipelineRunRequest,
    ) -> ModelVersionPipelineRunResponse:
        """Creates a new model version to pipeline run link.

        Args:
            model_version_pipeline_run_link: the Model Version to Pipeline Run
                Link to be created.

        Returns:
            - If Model Version to Pipeline Run Link already exists - returns
                the existing link.
            - Otherwise, returns the newly created model version to pipeline
                run link.
        """
        return self._create_workspace_scoped_resource(
            resource=model_version_pipeline_run_link,
            response_model=ModelVersionPipelineRunResponse,
            route=f"{MODEL_VERSIONS}/{model_version_pipeline_run_link.model_version}{RUNS}",
        )

    def list_model_version_pipeline_run_links(
        self,
        model_version_pipeline_run_link_filter_model: ModelVersionPipelineRunFilter,
        hydrate: bool = False,
    ) -> Page[ModelVersionPipelineRunResponse]:
        """Get all model version to pipeline run links by filter.

        Args:
            model_version_pipeline_run_link_filter_model: All filter parameters
                including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model version to pipeline run links.
        """
        return self._list_paginated_resources(
            route=MODEL_VERSION_PIPELINE_RUNS,
            response_model=ModelVersionPipelineRunResponse,
            filter_model=model_version_pipeline_run_link_filter_model,
            params={"hydrate": hydrate},
        )

    def delete_model_version_pipeline_run_link(
        self,
        model_version_id: UUID,
        model_version_pipeline_run_link_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a model version to pipeline run link.

        Args:
            model_version_id: ID of the model version containing the link.
            model_version_pipeline_run_link_name_or_id: name or ID of the model version to pipeline run link to be deleted.
        """
        self._delete_resource(
            resource_id=model_version_pipeline_run_link_name_or_id,
            route=f"{MODEL_VERSIONS}/{model_version_id}{RUNS}",
        )

    # ---------------------------- Devices ----------------------------

    def get_authorized_device(
        self, device_id: UUID, hydrate: bool = True
    ) -> OAuthDeviceResponse:
        """Gets a specific OAuth 2.0 authorized device.

        Args:
            device_id: The ID of the device to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested device, if it was found.
        """
        return self._get_resource(
            resource_id=device_id,
            route=DEVICES,
            response_model=OAuthDeviceResponse,
            params={"hydrate": hydrate},
        )

    def list_authorized_devices(
        self, filter_model: OAuthDeviceFilter, hydrate: bool = False
    ) -> Page[OAuthDeviceResponse]:
        """List all OAuth 2.0 authorized devices for a user.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all matching OAuth 2.0 authorized devices.
        """
        return self._list_paginated_resources(
            route=DEVICES,
            response_model=OAuthDeviceResponse,
            filter_model=filter_model,
            params={"hydrate": hydrate},
        )

    def update_authorized_device(
        self, device_id: UUID, update: OAuthDeviceUpdate
    ) -> OAuthDeviceResponse:
        """Updates an existing OAuth 2.0 authorized device for internal use.

        Args:
            device_id: The ID of the device to update.
            update: The update to be applied to the device.

        Returns:
            The updated OAuth 2.0 authorized device.
        """
        return self._update_resource(
            resource_id=device_id,
            resource_update=update,
            response_model=OAuthDeviceResponse,
            route=DEVICES,
        )

    def delete_authorized_device(self, device_id: UUID) -> None:
        """Deletes an OAuth 2.0 authorized device.

        Args:
            device_id: The ID of the device to delete.
        """
        self._delete_resource(resource_id=device_id, route=DEVICES)

    # -------------------
    # Pipeline API Tokens
    # -------------------

    def get_api_token(
        self,
        pipeline_id: Optional[UUID] = None,
        schedule_id: Optional[UUID] = None,
        expires_minutes: Optional[int] = None,
    ) -> str:
        """Get an API token for a workload.

        Args:
            pipeline_id: The ID of the pipeline to get a token for.
            schedule_id: The ID of the schedule to get a token for.
            expires_minutes: The number of minutes for which the token should
                be valid. If not provided, the token will be valid indefinitely.

        Returns:
            The API token.

        Raises:
            ValueError: if the server response is not valid.
        """
        params: Dict[str, Any] = {}
        if pipeline_id:
            params["pipeline_id"] = pipeline_id
        if schedule_id:
            params["schedule_id"] = schedule_id
        if expires_minutes:
            params["expires_minutes"] = expires_minutes
        response_body = self.get(API_TOKEN, params=params)
        if not isinstance(response_body, str):
            raise ValueError(
                f"Bad API Response. Expected API token, got "
                f"{type(response_body)}"
            )
        return response_body

    #################
    # Tags
    #################

    def create_tag(self, tag: TagRequest) -> TagResponse:
        """Creates a new tag.

        Args:
            tag: the tag to be created.

        Returns:
            The newly created tag.
        """
        return self._create_resource(
            resource=tag,
            response_model=TagResponse,
            route=TAGS,
        )

    def delete_tag(
        self,
        tag_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a tag.

        Args:
            tag_name_or_id: name or id of the tag to delete.
        """
        self._delete_resource(resource_id=tag_name_or_id, route=TAGS)

    def get_tag(
        self, tag_name_or_id: Union[str, UUID], hydrate: bool = True
    ) -> TagResponse:
        """Get an existing tag.

        Args:
            tag_name_or_id: name or id of the tag to be retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The tag of interest.
        """
        return self._get_resource(
            resource_id=tag_name_or_id,
            route=TAGS,
            response_model=TagResponse,
            params={"hydrate": hydrate},
        )

    def list_tags(
        self,
        tag_filter_model: TagFilter,
        hydrate: bool = False,
    ) -> Page[TagResponse]:
        """Get all tags by filter.

        Args:
            tag_filter_model: All filter parameters including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all tags.
        """
        return self._list_paginated_resources(
            route=TAGS,
            response_model=TagResponse,
            filter_model=tag_filter_model,
            params={"hydrate": hydrate},
        )

    def update_tag(
        self,
        tag_name_or_id: Union[str, UUID],
        tag_update_model: TagUpdate,
    ) -> TagResponse:
        """Update tag.

        Args:
            tag_name_or_id: name or id of the tag to be updated.
            tag_update_model: Tag to use for the update.

        Returns:
            An updated tag.
        """
        tag = self.get_tag(tag_name_or_id)
        return self._update_resource(
            resource_id=tag.id,
            resource_update=tag_update_model,
            route=TAGS,
            response_model=TagResponse,
        )

    # =======================
    # Internal helper methods
    # =======================

    def _get_auth_token(self) -> str:
        """Get the authentication token for the REST store.

        Returns:
            The authentication token.

        Raises:
            ValueError: if the response from the server isn't in the right
                format.
        """
        if self._api_token is None:
            # Check if the API token is already stored in the config
            if self.config.api_token:
                self._api_token = self.config.api_token
            # Check if the username and password are provided in the config
            elif (
                self.config.username is not None
                and self.config.password is not None
                or self.config.api_key is not None
            ):
                data: Optional[Dict[str, str]] = None
                if self.config.api_key is not None:
                    data = {
                        "grant_type": OAuthGrantTypes.ZENML_API_KEY.value,
                        "password": self.config.api_key,
                    }
                elif (
                    self.config.username is not None
                    and self.config.password is not None
                ):
                    data = {
                        "grant_type": OAuthGrantTypes.OAUTH_PASSWORD.value,
                        "username": self.config.username,
                        "password": self.config.password,
                    }

                response = self._handle_response(
                    requests.post(
                        self.url + API + VERSION_1 + LOGIN,
                        data=data,
                        verify=self.config.verify_ssl,
                        timeout=self.config.http_timeout,
                    )
                )
                if (
                    not isinstance(response, dict)
                    or "access_token" not in response
                ):
                    raise ValueError(
                        f"Bad API Response. Expected access token dict, got "
                        f"{type(response)}"
                    )
                self._api_token = response["access_token"]
                self.config.api_token = self._api_token
            else:
                raise ValueError(
                    "No API token, API key or username/password provided. "
                    "Please provide either an API token, an API key or a "
                    "username and password in the ZenML config."
                )
        return self._api_token

    @property
    def session(self) -> requests.Session:
        """Authenticate to the ZenML server.

        Returns:
            A requests session with the authentication token.
        """
        if self._session is None:
            if self.config.verify_ssl is False:
                urllib3.disable_warnings(
                    urllib3.exceptions.InsecureRequestWarning
                )

            self._session = requests.Session()
            retries = Retry(backoff_factor=0.1, connect=5)
            self._session.mount("https://", HTTPAdapter(max_retries=retries))
            self._session.mount("http://", HTTPAdapter(max_retries=retries))
            self._session.verify = self.config.verify_ssl
            token = self._get_auth_token()
            self._session.headers.update({"Authorization": "Bearer " + token})
            logger.debug("Authenticated to ZenML server.")
        return self._session

    def clear_session(self) -> None:
        """Clear the authentication session and any cached API tokens."""
        self._session = None
        self._api_token = None
        # Clear the configured API token only if it's possible to fetch a new
        # one from the server using other credentials (username/password or
        # service account API key).
        if (
            self.config.username is not None
            and self.config.password is not None
            or self.config.api_key is not None
        ):
            self.config.api_token = None

    @staticmethod
    def _handle_response(response: requests.Response) -> Json:
        """Handle API response, translating http status codes to Exception.

        Args:
            response: The response to handle.

        Returns:
            The parsed response.

        Raises:
            ValueError: if the response is not in the right format.
            RuntimeError: if an error response is received from the server
                and a more specific exception cannot be determined.
            exc: the exception converted from an error response, if one
                is returned from the server.
        """
        if 200 <= response.status_code < 300:
            try:
                payload: Json = response.json()
                return payload
            except requests.exceptions.JSONDecodeError:
                raise ValueError(
                    "Bad response from API. Expected json, got\n"
                    f"{response.text}"
                )
        elif response.status_code >= 400:
            exc = exception_from_response(response)
            if exc is not None:
                raise exc
            else:
                raise RuntimeError(
                    f"{response.status_code} HTTP Error received from server: "
                    f"{response.text}"
                )
        else:
            raise RuntimeError(
                "Error retrieving from API. Got response "
                f"{response.status_code} with body:\n{response.text}"
            )

    def _request(
        self,
        method: str,
        url: str,
        params: Optional[Dict[str, Any]] = None,
        **kwargs: Any,
    ) -> Json:
        """Make a request to the REST API.

        Args:
            method: The HTTP method to use.
            url: The URL to request.
            params: The query parameters to pass to the endpoint.
            kwargs: Additional keyword arguments to pass to the request.

        Returns:
            The parsed response.

        Raises:
            AuthorizationException: if the request fails due to an expired
                authentication token.
        """
        params = {k: str(v) for k, v in params.items()} if params else {}

        self.session.headers.update(
            {source_context.name: source_context.get().value}
        )

        try:
            return self._handle_response(
                self.session.request(
                    method,
                    url,
                    params=params,
                    verify=self.config.verify_ssl,
                    timeout=self.config.http_timeout,
                    **kwargs,
                )
            )
        except AuthorizationException:
            # The authentication token could have expired; refresh it and try
            # again. This will clear any cached token and trigger a new
            # authentication flow.
            self.clear_session()
            logger.info("Authentication token expired; refreshing...")

        try:
            return self._handle_response(
                self.session.request(
                    method,
                    url,
                    params=params,
                    verify=self.config.verify_ssl,
                    timeout=self.config.http_timeout,
                    **kwargs,
                )
            )
        except AuthorizationException:
            logger.info(
                "Your authentication token has expired. Please re-authenticate."
            )
            raise

    def get(
        self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
    ) -> Json:
        """Make a GET request to the given endpoint path.

        Args:
            path: The path to the endpoint.
            params: The query parameters to pass to the endpoint.
            kwargs: Additional keyword arguments to pass to the request.

        Returns:
            The response body.
        """
        logger.debug(f"Sending GET request to {path}...")
        return self._request(
            "GET", self.url + API + VERSION_1 + path, params=params, **kwargs
        )

    def delete(
        self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
    ) -> Json:
        """Make a DELETE request to the given endpoint path.

        Args:
            path: The path to the endpoint.
            params: The query parameters to pass to the endpoint.
            kwargs: Additional keyword arguments to pass to the request.

        Returns:
            The response body.
        """
        logger.debug(f"Sending DELETE request to {path}...")
        return self._request(
            "DELETE",
            self.url + API + VERSION_1 + path,
            params=params,
            **kwargs,
        )

    def post(
        self,
        path: str,
        body: BaseModel,
        params: Optional[Dict[str, Any]] = None,
        **kwargs: Any,
    ) -> Json:
        """Make a POST request to the given endpoint path.

        Args:
            path: The path to the endpoint.
            body: The body to send.
            params: The query parameters to pass to the endpoint.
            kwargs: Additional keyword arguments to pass to the request.

        Returns:
            The response body.
        """
        logger.debug(f"Sending POST request to {path}...")
        return self._request(
            "POST",
            self.url + API + VERSION_1 + path,
            data=body.json(),
            params=params,
            **kwargs,
        )

    def put(
        self,
        path: str,
        body: Optional[BaseModel] = None,
        params: Optional[Dict[str, Any]] = None,
        **kwargs: Any,
    ) -> Json:
        """Make a PUT request to the given endpoint path.

        Args:
            path: The path to the endpoint.
            body: The body to send.
            params: The query parameters to pass to the endpoint.
            kwargs: Additional keyword arguments to pass to the request.

        Returns:
            The response body.
        """
        logger.debug(f"Sending PUT request to {path}...")
        data = body.json(exclude_unset=True) if body else None
        return self._request(
            "PUT",
            self.url + API + VERSION_1 + path,
            data=data,
            params=params,
            **kwargs,
        )

    def _create_resource(
        self,
        resource: AnyRequest,
        response_model: Type[AnyResponse],
        route: str,
        params: Optional[Dict[str, Any]] = None,
    ) -> AnyResponse:
        """Create a new resource.

        Args:
            resource: The resource to create.
            route: The resource REST API route to use.
            response_model: Optional model to use to deserialize the response
                body. If not provided, the resource class itself will be used.
            params: Optional query parameters to pass to the endpoint.

        Returns:
            The created resource.
        """
        response_body = self.post(f"{route}", body=resource, params=params)
        return response_model.parse_obj(response_body)

    def _create_workspace_scoped_resource(
        self,
        resource: AnyWorkspaceScopedRequest,
        response_model: Type[AnyResponse],
        route: str,
        params: Optional[Dict[str, Any]] = None,
    ) -> AnyResponse:
        """Create a new workspace scoped resource.

        Args:
            resource: The resource to create.
            route: The resource REST API route to use.
            response_model: Optional model to use to deserialize the response
                body. If not provided, the resource class itself will be used.
            params: Optional query parameters to pass to the endpoint.

        Returns:
            The created resource.
        """
        return self._create_resource(
            resource=resource,
            response_model=response_model,
            route=f"{WORKSPACES}/{str(resource.workspace)}{route}",
            params=params,
        )

    def _get_or_create_resource(
        self,
        resource: AnyRequest,
        response_model: Type[AnyResponse],
        route: str,
        params: Optional[Dict[str, Any]] = None,
    ) -> Tuple[AnyResponse, bool]:
        """Get or create a resource.

        Args:
            resource: The resource to get or create.
            route: The resource REST API route to use.
            response_model: Optional model to use to deserialize the response
                body. If not provided, the resource class itself will be used.
            params: Optional query parameters to pass to the endpoint.

        Returns:
            The created resource, and a boolean indicating whether the resource
            was created or not.

        Raises:
            ValueError: If the response body is not a list with 2 elements
                where the first element is the resource and the second element
                a boolean indicating whether the resource was created or not.
        """
        response_body = self.post(
            f"{route}{GET_OR_CREATE}",
            body=resource,
            params=params,
        )
        if not isinstance(response_body, list):
            raise ValueError(
                f"Expected a list response from the {route}{GET_OR_CREATE} "
                f"endpoint but got {type(response_body)} instead."
            )
        if len(response_body) != 2:
            raise ValueError(
                f"Expected a list response with 2 elements from the "
                f"{route}{GET_OR_CREATE} endpoint but got {len(response_body)} "
                f"elements instead."
            )
        model_json, was_created = response_body
        if not isinstance(was_created, bool):
            raise ValueError(
                f"Expected a boolean as the second element of the list "
                f"response from the {route}{GET_OR_CREATE} endpoint but got "
                f"{type(was_created)} instead."
            )
        return response_model.parse_obj(model_json), was_created

    def _get_or_create_workspace_scoped_resource(
        self,
        resource: AnyWorkspaceScopedRequest,
        response_model: Type[AnyResponse],
        route: str,
        params: Optional[Dict[str, Any]] = None,
    ) -> Tuple[AnyResponse, bool]:
        """Get or create a workspace scoped resource.

        Args:
            resource: The resource to get or create.
            route: The resource REST API route to use.
            response_model: Optional model to use to deserialize the response
                body. If not provided, the resource class itself will be used.
            params: Optional query parameters to pass to the endpoint.

        Returns:
            The created resource, and a boolean indicating whether the resource
            was created or not.
        """
        return self._get_or_create_resource(
            resource=resource,
            response_model=response_model,
            route=f"{WORKSPACES}/{str(resource.workspace)}{route}",
            params=params,
        )

    def _get_resource(
        self,
        resource_id: Union[str, int, UUID],
        route: str,
        response_model: Type[AnyResponse],
        params: Optional[Dict[str, Any]] = None,
    ) -> AnyResponse:
        """Retrieve a single resource.

        Args:
            resource_id: The ID of the resource to retrieve.
            route: The resource REST API route to use.
            response_model: Model to use to serialize the response body.
            params: Optional query parameters to pass to the endpoint.

        Returns:
            The retrieved resource.
        """
        body = self.get(f"{route}/{str(resource_id)}", params=params)
        return response_model.parse_obj(body)

    def _list_paginated_resources(
        self,
        route: str,
        response_model: Type[AnyResponse],
        filter_model: BaseFilter,
        params: Optional[Dict[str, Any]] = None,
    ) -> Page[AnyResponse]:
        """Retrieve a list of resources filtered by some criteria.

        Args:
            route: The resource REST API route to use.
            response_model: Model to use to serialize the response body.
            filter_model: The filter model to use for the list query.
            params: Optional query parameters to pass to the endpoint.

        Returns:
            List of retrieved resources matching the filter criteria.

        Raises:
            ValueError: If the value returned by the server is not a list.
        """
        # leave out filter params that are not supplied
        params = params or {}
        params.update(filter_model.dict(exclude_none=True))
        body = self.get(f"{route}", params=params)
        if not isinstance(body, dict):
            raise ValueError(
                f"Bad API Response. Expected list, got {type(body)}"
            )
        # The initial page of items will be of type BaseResponseModel
        page_of_items: Page[AnyResponse] = Page.parse_obj(body)
        # So these items will be parsed into their correct types like here
        page_of_items.items = [
            response_model.parse_obj(generic_item)
            for generic_item in body["items"]
        ]
        return page_of_items

    def _list_resources(
        self,
        route: str,
        response_model: Type[AnyResponse],
        **filters: Any,
    ) -> List[AnyResponse]:
        """Retrieve a list of resources filtered by some criteria.

        Args:
            route: The resource REST API route to use.
            response_model: Model to use to serialize the response body.
            filters: Filter parameters to use in the query.

        Returns:
            List of retrieved resources matching the filter criteria.

        Raises:
            ValueError: If the value returned by the server is not a list.
        """
        # leave out filter params that are not supplied
        params = dict(filter(lambda x: x[1] is not None, filters.items()))
        body = self.get(f"{route}", params=params)
        if not isinstance(body, list):
            raise ValueError(
                f"Bad API Response. Expected list, got {type(body)}"
            )
        return [response_model.parse_obj(entry) for entry in body]

    def _update_resource(
        self,
        resource_id: Union[str, int, UUID],
        resource_update: BaseModel,
        response_model: Type[AnyResponse],
        route: str,
        params: Optional[Dict[str, Any]] = None,
    ) -> AnyResponse:
        """Update an existing resource.

        Args:
            resource_id: The id of the resource to update.
            resource_update: The resource update.
            response_model: Optional model to use to deserialize the response
                body. If not provided, the resource class itself will be used.
            route: The resource REST API route to use.
            params: Optional query parameters to pass to the endpoint.

        Returns:
            The updated resource.
        """
        response_body = self.put(
            f"{route}/{str(resource_id)}", body=resource_update, params=params
        )

        return response_model.parse_obj(response_body)

    def _delete_resource(
        self, resource_id: Union[str, UUID], route: str
    ) -> None:
        """Delete a resource.

        Args:
            resource_id: The ID of the resource to delete.
            route: The resource REST API route to use.
        """
        self.delete(f"{route}/{str(resource_id)}")
session: Session property readonly

Authenticate to the ZenML server.

Returns:

Type Description
Session

A requests session with the authentication token.

CONFIG_TYPE (StoreConfiguration) pydantic-model

REST ZenML store configuration.

Attributes:

Name Type Description
type StoreType

The type of the store.

username Optional[str]

The username to use to connect to the Zen server.

password Optional[str]

The password to use to connect to the Zen server.

api_key Optional[str]

The service account API key to use to connect to the Zen server.

api_token Optional[str]

The API token to use to connect to the Zen server. Generated by the client and stored in the configuration file on the first login and every time the API key is refreshed.

verify_ssl Union[bool, str]

Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use or the CA bundle value itself.

http_timeout int

The timeout to use for all requests.

Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStoreConfiguration(StoreConfiguration):
    """REST ZenML store configuration.

    Attributes:
        type: The type of the store.
        username: The username to use to connect to the Zen server.
        password: The password to use to connect to the Zen server.
        api_key: The service account API key to use to connect to the Zen
            server.
        api_token: The API token to use to connect to the Zen server. Generated
            by the client and stored in the configuration file on the first
            login and every time the API key is refreshed.
        verify_ssl: Either a boolean, in which case it controls whether we
            verify the server's TLS certificate, or a string, in which case it
            must be a path to a CA bundle to use or the CA bundle value itself.
        http_timeout: The timeout to use for all requests.

    """

    type: StoreType = StoreType.REST

    username: Optional[str] = None
    password: Optional[str] = None
    api_key: Optional[str] = None
    api_token: Optional[str] = None
    verify_ssl: Union[bool, str] = True
    http_timeout: int = DEFAULT_HTTP_TIMEOUT

    @root_validator
    def validate_credentials(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validates the credentials provided in the values dictionary.

        Args:
            values: A dictionary containing the values to be validated.

        Raises:
            ValueError: If neither api_token nor username nor api_key is set.

        Returns:
            The values dictionary.
        """
        # Check if the values dictionary contains either an API token, an API
        # key or a username as non-empty strings.
        if (
            values.get("api_token")
            or values.get("username")
            or values.get("api_key")
        ):
            return values
        raise ValueError(
            "Neither api_token nor username nor api_key is set in the "
            "store config."
        )

    @validator("url")
    def validate_url(cls, url: str) -> str:
        """Validates that the URL is a well-formed REST store URL.

        Args:
            url: The URL to be validated.

        Returns:
            The validated URL without trailing slashes.

        Raises:
            ValueError: If the URL is not a well-formed REST store URL.
        """
        url = url.rstrip("/")
        scheme = re.search("^([a-z0-9]+://)", url)
        if scheme is None or scheme.group() not in ("https://", "http://"):
            raise ValueError(
                "Invalid URL for REST store: {url}. Should be in the form "
                "https://hostname[:port] or http://hostname[:port]."
            )

        # When running inside a container, if the URL uses localhost, the
        # target service will not be available. We try to replace localhost
        # with one of the special Docker or K3D internal hostnames.
        url = replace_localhost_with_internal_hostname(url)

        return url

    @validator("verify_ssl")
    def validate_verify_ssl(
        cls, verify_ssl: Union[bool, str]
    ) -> Union[bool, str]:
        """Validates that the verify_ssl either points to a file or is a bool.

        Args:
            verify_ssl: The verify_ssl value to be validated.

        Returns:
            The validated verify_ssl value.
        """
        secret_folder = Path(
            GlobalConfiguration().local_stores_path,
            "certificates",
        )
        if isinstance(verify_ssl, bool) or verify_ssl.startswith(
            str(secret_folder)
        ):
            return verify_ssl

        if os.path.isfile(verify_ssl):
            with open(verify_ssl, "r") as f:
                verify_ssl = f.read()

        fileio.makedirs(str(secret_folder))
        file_path = Path(secret_folder, "ca_bundle.pem")
        with open(file_path, "w") as f:
            f.write(verify_ssl)
        file_path.chmod(0o600)
        verify_ssl = str(file_path)

        return verify_ssl

    @classmethod
    def supports_url_scheme(cls, url: str) -> bool:
        """Check if a URL scheme is supported by this store.

        Args:
            url: The URL to check.

        Returns:
            True if the URL scheme is supported, False otherwise.
        """
        return urlparse(url).scheme in ("http", "https")

    def expand_certificates(self) -> None:
        """Expands the certificates in the verify_ssl field."""
        # Load the certificate values back into the configuration
        if isinstance(self.verify_ssl, str) and os.path.isfile(
            self.verify_ssl
        ):
            with open(self.verify_ssl, "r") as f:
                self.verify_ssl = f.read()

    class Config:
        """Pydantic configuration class."""

        # Don't validate attributes when assigning them. This is necessary
        # because the `verify_ssl` attribute can be expanded to the contents
        # of the certificate file.
        validate_assignment = False
        # Forbid extra attributes set in the class.
        extra = "forbid"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/rest_zen_store.py
class Config:
    """Pydantic configuration class."""

    # Don't validate attributes when assigning them. This is necessary
    # because the `verify_ssl` attribute can be expanded to the contents
    # of the certificate file.
    validate_assignment = False
    # Forbid extra attributes set in the class.
    extra = "forbid"
expand_certificates(self)

Expands the certificates in the verify_ssl field.

Source code in zenml/zen_stores/rest_zen_store.py
def expand_certificates(self) -> None:
    """Expands the certificates in the verify_ssl field."""
    # Load the certificate values back into the configuration
    if isinstance(self.verify_ssl, str) and os.path.isfile(
        self.verify_ssl
    ):
        with open(self.verify_ssl, "r") as f:
            self.verify_ssl = f.read()
supports_url_scheme(url) classmethod

Check if a URL scheme is supported by this store.

Parameters:

Name Type Description Default
url str

The URL to check.

required

Returns:

Type Description
bool

True if the URL scheme is supported, False otherwise.

Source code in zenml/zen_stores/rest_zen_store.py
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
    """Check if a URL scheme is supported by this store.

    Args:
        url: The URL to check.

    Returns:
        True if the URL scheme is supported, False otherwise.
    """
    return urlparse(url).scheme in ("http", "https")
validate_credentials(values) classmethod

Validates the credentials provided in the values dictionary.

Parameters:

Name Type Description Default
values Dict[str, Any]

A dictionary containing the values to be validated.

required

Exceptions:

Type Description
ValueError

If neither api_token nor username nor api_key is set.

Returns:

Type Description
Dict[str, Any]

The values dictionary.

Source code in zenml/zen_stores/rest_zen_store.py
@root_validator
def validate_credentials(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Validates the credentials provided in the values dictionary.

    Args:
        values: A dictionary containing the values to be validated.

    Raises:
        ValueError: If neither api_token nor username nor api_key is set.

    Returns:
        The values dictionary.
    """
    # Check if the values dictionary contains either an API token, an API
    # key or a username as non-empty strings.
    if (
        values.get("api_token")
        or values.get("username")
        or values.get("api_key")
    ):
        return values
    raise ValueError(
        "Neither api_token nor username nor api_key is set in the "
        "store config."
    )
validate_url(url) classmethod

Validates that the URL is a well-formed REST store URL.

Parameters:

Name Type Description Default
url str

The URL to be validated.

required

Returns:

Type Description
str

The validated URL without trailing slashes.

Exceptions:

Type Description
ValueError

If the URL is not a well-formed REST store URL.

Source code in zenml/zen_stores/rest_zen_store.py
@validator("url")
def validate_url(cls, url: str) -> str:
    """Validates that the URL is a well-formed REST store URL.

    Args:
        url: The URL to be validated.

    Returns:
        The validated URL without trailing slashes.

    Raises:
        ValueError: If the URL is not a well-formed REST store URL.
    """
    url = url.rstrip("/")
    scheme = re.search("^([a-z0-9]+://)", url)
    if scheme is None or scheme.group() not in ("https://", "http://"):
        raise ValueError(
            "Invalid URL for REST store: {url}. Should be in the form "
            "https://hostname[:port] or http://hostname[:port]."
        )

    # When running inside a container, if the URL uses localhost, the
    # target service will not be available. We try to replace localhost
    # with one of the special Docker or K3D internal hostnames.
    url = replace_localhost_with_internal_hostname(url)

    return url
validate_verify_ssl(verify_ssl) classmethod

Validates that the verify_ssl either points to a file or is a bool.

Parameters:

Name Type Description Default
verify_ssl Union[bool, str]

The verify_ssl value to be validated.

required

Returns:

Type Description
Union[bool, str]

The validated verify_ssl value.

Source code in zenml/zen_stores/rest_zen_store.py
@validator("verify_ssl")
def validate_verify_ssl(
    cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
    """Validates that the verify_ssl either points to a file or is a bool.

    Args:
        verify_ssl: The verify_ssl value to be validated.

    Returns:
        The validated verify_ssl value.
    """
    secret_folder = Path(
        GlobalConfiguration().local_stores_path,
        "certificates",
    )
    if isinstance(verify_ssl, bool) or verify_ssl.startswith(
        str(secret_folder)
    ):
        return verify_ssl

    if os.path.isfile(verify_ssl):
        with open(verify_ssl, "r") as f:
            verify_ssl = f.read()

    fileio.makedirs(str(secret_folder))
    file_path = Path(secret_folder, "ca_bundle.pem")
    with open(file_path, "w") as f:
        f.write(verify_ssl)
    file_path.chmod(0o600)
    verify_ssl = str(file_path)

    return verify_ssl
backup_secrets(self, ignore_errors=True, delete_secrets=False)

Backs up all secrets to the configured backup secrets store.

Parameters:

Name Type Description Default
ignore_errors bool

Whether to ignore individual errors during the backup process and attempt to backup all secrets.

True
delete_secrets bool

Whether to delete the secrets that have been successfully backed up from the primary secrets store. Setting this flag effectively moves all secrets from the primary secrets store to the backup secrets store.

False
Source code in zenml/zen_stores/rest_zen_store.py
def backup_secrets(
    self, ignore_errors: bool = True, delete_secrets: bool = False
) -> None:
    """Backs up all secrets to the configured backup secrets store.

    Args:
        ignore_errors: Whether to ignore individual errors during the backup
            process and attempt to backup all secrets.
        delete_secrets: Whether to delete the secrets that have been
            successfully backed up from the primary secrets store. Setting
            this flag effectively moves all secrets from the primary secrets
            store to the backup secrets store.
    """
    params: Dict[str, Any] = {
        "ignore_errors": ignore_errors,
        "delete_secrets": delete_secrets,
    }
    self.put(
        f"{SECRETS_OPERATIONS}{SECRETS_BACKUP}",
        params=params,
    )
clear_session(self)

Clear the authentication session and any cached API tokens.

Source code in zenml/zen_stores/rest_zen_store.py
def clear_session(self) -> None:
    """Clear the authentication session and any cached API tokens."""
    self._session = None
    self._api_token = None
    # Clear the configured API token only if it's possible to fetch a new
    # one from the server using other credentials (username/password or
    # service account API key).
    if (
        self.config.username is not None
        and self.config.password is not None
        or self.config.api_key is not None
    ):
        self.config.api_token = None
create_api_key(self, service_account_id, api_key)

Create a new API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to create the API key.

required
api_key APIKeyRequest

The API key to create.

required

Returns:

Type Description
APIKeyResponse

The created API key.

Source code in zenml/zen_stores/rest_zen_store.py
def create_api_key(
    self, service_account_id: UUID, api_key: APIKeyRequest
) -> APIKeyResponse:
    """Create a new API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            create the API key.
        api_key: The API key to create.

    Returns:
        The created API key.
    """
    return self._create_resource(
        resource=api_key,
        route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
        response_model=APIKeyResponse,
    )
create_artifact(self, artifact)

Creates a new artifact.

Parameters:

Name Type Description Default
artifact ArtifactRequest

The artifact to create.

required

Returns:

Type Description
ArtifactResponse

The newly created artifact.

Source code in zenml/zen_stores/rest_zen_store.py
def create_artifact(self, artifact: ArtifactRequest) -> ArtifactResponse:
    """Creates a new artifact.

    Args:
        artifact: The artifact to create.

    Returns:
        The newly created artifact.
    """
    return self._create_resource(
        resource=artifact,
        response_model=ArtifactResponse,
        route=ARTIFACTS,
    )
create_artifact_version(self, artifact_version)

Creates an artifact version.

Parameters:

Name Type Description Default
artifact_version ArtifactVersionRequest

The artifact version to create.

required

Returns:

Type Description
ArtifactVersionResponse

The created artifact version.

Source code in zenml/zen_stores/rest_zen_store.py
def create_artifact_version(
    self, artifact_version: ArtifactVersionRequest
) -> ArtifactVersionResponse:
    """Creates an artifact version.

    Args:
        artifact_version: The artifact version to create.

    Returns:
        The created artifact version.
    """
    return self._create_resource(
        resource=artifact_version,
        response_model=ArtifactVersionResponse,
        route=ARTIFACT_VERSIONS,
    )
create_build(self, build)

Creates a new build in a workspace.

Parameters:

Name Type Description Default
build PipelineBuildRequest

The build to create.

required

Returns:

Type Description
PipelineBuildResponse

The newly created build.

Source code in zenml/zen_stores/rest_zen_store.py
def create_build(
    self,
    build: PipelineBuildRequest,
) -> PipelineBuildResponse:
    """Creates a new build in a workspace.

    Args:
        build: The build to create.

    Returns:
        The newly created build.
    """
    return self._create_workspace_scoped_resource(
        resource=build,
        route=PIPELINE_BUILDS,
        response_model=PipelineBuildResponse,
    )
create_code_repository(self, code_repository)

Creates a new code repository.

Parameters:

Name Type Description Default
code_repository CodeRepositoryRequest

Code repository to be created.

required

Returns:

Type Description
CodeRepositoryResponse

The newly created code repository.

Source code in zenml/zen_stores/rest_zen_store.py
def create_code_repository(
    self, code_repository: CodeRepositoryRequest
) -> CodeRepositoryResponse:
    """Creates a new code repository.

    Args:
        code_repository: Code repository to be created.

    Returns:
        The newly created code repository.
    """
    return self._create_workspace_scoped_resource(
        resource=code_repository,
        response_model=CodeRepositoryResponse,
        route=CODE_REPOSITORIES,
    )
create_deployment(self, deployment)

Creates a new deployment in a workspace.

Parameters:

Name Type Description Default
deployment PipelineDeploymentRequest

The deployment to create.

required

Returns:

Type Description
PipelineDeploymentResponse

The newly created deployment.

Source code in zenml/zen_stores/rest_zen_store.py
def create_deployment(
    self,
    deployment: PipelineDeploymentRequest,
) -> PipelineDeploymentResponse:
    """Creates a new deployment in a workspace.

    Args:
        deployment: The deployment to create.

    Returns:
        The newly created deployment.
    """
    return self._create_workspace_scoped_resource(
        resource=deployment,
        route=PIPELINE_DEPLOYMENTS,
        response_model=PipelineDeploymentResponse,
    )
create_event_source(self, event_source)

Create an event_source.

Parameters:

Name Type Description Default
event_source EventSourceRequest

The event_source to create.

required

Returns:

Type Description
EventSourceResponse

The created event_source.

Source code in zenml/zen_stores/rest_zen_store.py
def create_event_source(
    self, event_source: EventSourceRequest
) -> EventSourceResponse:
    """Create an event_source.

    Args:
        event_source: The event_source to create.

    Returns:
        The created event_source.
    """
    return self._create_resource(
        resource=event_source,
        route=EVENT_SOURCES,
        response_model=EventSourceResponse,
    )
create_flavor(self, flavor)

Creates a new stack component flavor.

Parameters:

Name Type Description Default
flavor FlavorRequest

The stack component flavor to create.

required

Returns:

Type Description
FlavorResponse

The newly created flavor.

Source code in zenml/zen_stores/rest_zen_store.py
def create_flavor(self, flavor: FlavorRequest) -> FlavorResponse:
    """Creates a new stack component flavor.

    Args:
        flavor: The stack component flavor to create.

    Returns:
        The newly created flavor.
    """
    return self._create_resource(
        resource=flavor,
        route=FLAVORS,
        response_model=FlavorResponse,
    )
create_model(self, model)

Creates a new model.

Parameters:

Name Type Description Default
model ModelRequest

the Model to be created.

required

Returns:

Type Description
ModelResponse

The newly created model.

Source code in zenml/zen_stores/rest_zen_store.py
def create_model(self, model: ModelRequest) -> ModelResponse:
    """Creates a new model.

    Args:
        model: the Model to be created.

    Returns:
        The newly created model.
    """
    return self._create_workspace_scoped_resource(
        resource=model,
        response_model=ModelResponse,
        route=MODELS,
    )
create_model_version(self, model_version)

Creates a new model version.

Parameters:

Name Type Description Default
model_version ModelVersionRequest

the Model Version to be created.

required

Returns:

Type Description
ModelVersionResponse

The newly created model version.

Source code in zenml/zen_stores/rest_zen_store.py
def create_model_version(
    self, model_version: ModelVersionRequest
) -> ModelVersionResponse:
    """Creates a new model version.

    Args:
        model_version: the Model Version to be created.

    Returns:
        The newly created model version.
    """
    return self._create_workspace_scoped_resource(
        resource=model_version,
        response_model=ModelVersionResponse,
        route=f"{MODELS}/{model_version.model}{MODEL_VERSIONS}",
    )

Creates a new model version link.

Parameters:

Name Type Description Default
model_version_artifact_link ModelVersionArtifactRequest

the Model Version to Artifact Link to be created.

required

Returns:

Type Description
ModelVersionArtifactResponse

The newly created model version to artifact link.

Source code in zenml/zen_stores/rest_zen_store.py
def create_model_version_artifact_link(
    self, model_version_artifact_link: ModelVersionArtifactRequest
) -> ModelVersionArtifactResponse:
    """Creates a new model version link.

    Args:
        model_version_artifact_link: the Model Version to Artifact Link
            to be created.

    Returns:
        The newly created model version to artifact link.
    """
    return self._create_workspace_scoped_resource(
        resource=model_version_artifact_link,
        response_model=ModelVersionArtifactResponse,
        route=f"{MODEL_VERSIONS}/{model_version_artifact_link.model_version}{ARTIFACTS}",
    )

Creates a new model version to pipeline run link.

Parameters:

Name Type Description Default
model_version_pipeline_run_link ModelVersionPipelineRunRequest

the Model Version to Pipeline Run Link to be created.

required

Returns:

Type Description
ModelVersionPipelineRunResponse
  • If Model Version to Pipeline Run Link already exists - returns the existing link.
  • Otherwise, returns the newly created model version to pipeline run link.
Source code in zenml/zen_stores/rest_zen_store.py
def create_model_version_pipeline_run_link(
    self,
    model_version_pipeline_run_link: ModelVersionPipelineRunRequest,
) -> ModelVersionPipelineRunResponse:
    """Creates a new model version to pipeline run link.

    Args:
        model_version_pipeline_run_link: the Model Version to Pipeline Run
            Link to be created.

    Returns:
        - If Model Version to Pipeline Run Link already exists - returns
            the existing link.
        - Otherwise, returns the newly created model version to pipeline
            run link.
    """
    return self._create_workspace_scoped_resource(
        resource=model_version_pipeline_run_link,
        response_model=ModelVersionPipelineRunResponse,
        route=f"{MODEL_VERSIONS}/{model_version_pipeline_run_link.model_version}{RUNS}",
    )
create_pipeline(self, pipeline)

Creates a new pipeline in a workspace.

Parameters:

Name Type Description Default
pipeline PipelineRequest

The pipeline to create.

required

Returns:

Type Description
PipelineResponse

The newly created pipeline.

Source code in zenml/zen_stores/rest_zen_store.py
def create_pipeline(self, pipeline: PipelineRequest) -> PipelineResponse:
    """Creates a new pipeline in a workspace.

    Args:
        pipeline: The pipeline to create.

    Returns:
        The newly created pipeline.
    """
    return self._create_workspace_scoped_resource(
        resource=pipeline,
        route=PIPELINES,
        response_model=PipelineResponse,
    )
create_run(self, pipeline_run)

Creates a pipeline run.

Parameters:

Name Type Description Default
pipeline_run PipelineRunRequest

The pipeline run to create.

required

Returns:

Type Description
PipelineRunResponse

The created pipeline run.

Source code in zenml/zen_stores/rest_zen_store.py
def create_run(
    self, pipeline_run: PipelineRunRequest
) -> PipelineRunResponse:
    """Creates a pipeline run.

    Args:
        pipeline_run: The pipeline run to create.

    Returns:
        The created pipeline run.
    """
    return self._create_workspace_scoped_resource(
        resource=pipeline_run,
        response_model=PipelineRunResponse,
        route=RUNS,
    )
create_run_metadata(self, run_metadata)

Creates run metadata.

Parameters:

Name Type Description Default
run_metadata RunMetadataRequest

The run metadata to create.

required

Returns:

Type Description
List[zenml.models.v2.core.run_metadata.RunMetadataResponse]

The created run metadata.

Source code in zenml/zen_stores/rest_zen_store.py
def create_run_metadata(
    self, run_metadata: RunMetadataRequest
) -> List[RunMetadataResponse]:
    """Creates run metadata.

    Args:
        run_metadata: The run metadata to create.

    Returns:
        The created run metadata.
    """
    route = f"{WORKSPACES}/{str(run_metadata.workspace)}{RUN_METADATA}"
    response_body = self.post(f"{route}", body=run_metadata)
    result: List[RunMetadataResponse] = []
    if isinstance(response_body, list):
        for metadata in response_body or []:
            result.append(RunMetadataResponse.parse_obj(metadata))
    return result
create_run_step(self, step_run)

Creates a step run.

Parameters:

Name Type Description Default
step_run StepRunRequest

The step run to create.

required

Returns:

Type Description
StepRunResponse

The created step run.

Source code in zenml/zen_stores/rest_zen_store.py
def create_run_step(self, step_run: StepRunRequest) -> StepRunResponse:
    """Creates a step run.

    Args:
        step_run: The step run to create.

    Returns:
        The created step run.
    """
    return self._create_resource(
        resource=step_run,
        response_model=StepRunResponse,
        route=STEPS,
    )
create_schedule(self, schedule)

Creates a new schedule.

Parameters:

Name Type Description Default
schedule ScheduleRequest

The schedule to create.

required

Returns:

Type Description
ScheduleResponse

The newly created schedule.

Source code in zenml/zen_stores/rest_zen_store.py
def create_schedule(self, schedule: ScheduleRequest) -> ScheduleResponse:
    """Creates a new schedule.

    Args:
        schedule: The schedule to create.

    Returns:
        The newly created schedule.
    """
    return self._create_workspace_scoped_resource(
        resource=schedule,
        route=SCHEDULES,
        response_model=ScheduleResponse,
    )
create_secret(self, secret)

Creates a new secret.

The new secret is also validated against the scoping rules enforced in the secrets store:

  • only one workspace-scoped secret with the given name can exist in the target workspace.
  • only one user-scoped secret with the given name can exist in the target workspace for the target user.

Parameters:

Name Type Description Default
secret SecretRequest

The secret to create.

required

Returns:

Type Description
SecretResponse

The newly created secret.

Source code in zenml/zen_stores/rest_zen_store.py
def create_secret(self, secret: SecretRequest) -> SecretResponse:
    """Creates a new secret.

    The new secret is also validated against the scoping rules enforced in
    the secrets store:

      - only one workspace-scoped secret with the given name can exist
        in the target workspace.
      - only one user-scoped secret with the given name can exist in the
        target workspace for the target user.

    Args:
        secret: The secret to create.

    Returns:
        The newly created secret.
    """
    return self._create_workspace_scoped_resource(
        resource=secret,
        route=SECRETS,
        response_model=SecretResponse,
    )
create_service_account(self, service_account)

Creates a new service account.

Parameters:

Name Type Description Default
service_account ServiceAccountRequest

Service account to be created.

required

Returns:

Type Description
ServiceAccountResponse

The newly created service account.

Source code in zenml/zen_stores/rest_zen_store.py
def create_service_account(
    self, service_account: ServiceAccountRequest
) -> ServiceAccountResponse:
    """Creates a new service account.

    Args:
        service_account: Service account to be created.

    Returns:
        The newly created service account.
    """
    return self._create_resource(
        resource=service_account,
        route=SERVICE_ACCOUNTS,
        response_model=ServiceAccountResponse,
    )
create_service_connector(self, service_connector)

Creates a new service connector.

Parameters:

Name Type Description Default
service_connector ServiceConnectorRequest

Service connector to be created.

required

Returns:

Type Description
ServiceConnectorResponse

The newly created service connector.

Source code in zenml/zen_stores/rest_zen_store.py
def create_service_connector(
    self, service_connector: ServiceConnectorRequest
) -> ServiceConnectorResponse:
    """Creates a new service connector.

    Args:
        service_connector: Service connector to be created.

    Returns:
        The newly created service connector.
    """
    connector_model = self._create_workspace_scoped_resource(
        resource=service_connector,
        route=SERVICE_CONNECTORS,
        response_model=ServiceConnectorResponse,
    )
    self._populate_connector_type(connector_model)
    return connector_model
create_stack(self, stack)

Register a new stack.

Parameters:

Name Type Description Default
stack StackRequest

The stack to register.

required

Returns:

Type Description
StackResponse

The registered stack.

Source code in zenml/zen_stores/rest_zen_store.py
def create_stack(self, stack: StackRequest) -> StackResponse:
    """Register a new stack.

    Args:
        stack: The stack to register.

    Returns:
        The registered stack.
    """
    return self._create_workspace_scoped_resource(
        resource=stack,
        route=STACKS,
        response_model=StackResponse,
    )
create_stack_component(self, component)

Create a stack component.

Parameters:

Name Type Description Default
component ComponentRequest

The stack component to create.

required

Returns:

Type Description
ComponentResponse

The created stack component.

Source code in zenml/zen_stores/rest_zen_store.py
def create_stack_component(
    self,
    component: ComponentRequest,
) -> ComponentResponse:
    """Create a stack component.

    Args:
        component: The stack component to create.

    Returns:
        The created stack component.
    """
    return self._create_workspace_scoped_resource(
        resource=component,
        route=STACK_COMPONENTS,
        response_model=ComponentResponse,
    )
create_tag(self, tag)

Creates a new tag.

Parameters:

Name Type Description Default
tag TagRequest

the tag to be created.

required

Returns:

Type Description
TagResponse

The newly created tag.

Source code in zenml/zen_stores/rest_zen_store.py
def create_tag(self, tag: TagRequest) -> TagResponse:
    """Creates a new tag.

    Args:
        tag: the tag to be created.

    Returns:
        The newly created tag.
    """
    return self._create_resource(
        resource=tag,
        response_model=TagResponse,
        route=TAGS,
    )
create_trigger(self, trigger)

Create an trigger.

Parameters:

Name Type Description Default
trigger TriggerRequest

The trigger to create.

required

Returns:

Type Description
TriggerResponse

The created trigger.

Source code in zenml/zen_stores/rest_zen_store.py
def create_trigger(self, trigger: TriggerRequest) -> TriggerResponse:
    """Create an trigger.

    Args:
        trigger: The trigger to create.

    Returns:
        The created trigger.
    """
    return self._create_resource(
        resource=trigger,
        route=TRIGGERS,
        response_model=TriggerResponse,
    )
create_user(self, user)

Creates a new user.

Parameters:

Name Type Description Default
user UserRequest

User to be created.

required

Returns:

Type Description
UserResponse

The newly created user.

Source code in zenml/zen_stores/rest_zen_store.py
def create_user(self, user: UserRequest) -> UserResponse:
    """Creates a new user.

    Args:
        user: User to be created.

    Returns:
        The newly created user.
    """
    return self._create_resource(
        resource=user,
        route=USERS,
        response_model=UserResponse,
    )
create_workspace(self, workspace)

Creates a new workspace.

Parameters:

Name Type Description Default
workspace WorkspaceRequest

The workspace to create.

required

Returns:

Type Description
WorkspaceResponse

The newly created workspace.

Source code in zenml/zen_stores/rest_zen_store.py
def create_workspace(
    self, workspace: WorkspaceRequest
) -> WorkspaceResponse:
    """Creates a new workspace.

    Args:
        workspace: The workspace to create.

    Returns:
        The newly created workspace.
    """
    return self._create_resource(
        resource=workspace,
        route=WORKSPACES,
        response_model=WorkspaceResponse,
    )
delete(self, path, params=None, **kwargs)

Make a DELETE request to the given endpoint path.

Parameters:

Name Type Description Default
path str

The path to the endpoint.

required
params Optional[Dict[str, Any]]

The query parameters to pass to the endpoint.

None
kwargs Any

Additional keyword arguments to pass to the request.

{}

Returns:

Type Description
Union[Dict[str, Any], List[Any], str, int, float, bool]

The response body.

Source code in zenml/zen_stores/rest_zen_store.py
def delete(
    self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
    """Make a DELETE request to the given endpoint path.

    Args:
        path: The path to the endpoint.
        params: The query parameters to pass to the endpoint.
        kwargs: Additional keyword arguments to pass to the request.

    Returns:
        The response body.
    """
    logger.debug(f"Sending DELETE request to {path}...")
    return self._request(
        "DELETE",
        self.url + API + VERSION_1 + path,
        params=params,
        **kwargs,
    )

Deletes all links between model version and an artifact.

Parameters:

Name Type Description Default
model_version_id UUID

ID of the model version containing the link.

required
only_links bool

Flag deciding whether to delete only links or all.

True
Source code in zenml/zen_stores/rest_zen_store.py
def delete_all_model_version_artifact_links(
    self,
    model_version_id: UUID,
    only_links: bool = True,
) -> None:
    """Deletes all links between model version and an artifact.

    Args:
        model_version_id: ID of the model version containing the link.
        only_links: Flag deciding whether to delete only links or all.
    """
    self.delete(
        f"{MODEL_VERSIONS}/{model_version_id}{ARTIFACTS}",
        params={"only_links": only_links},
    )
delete_api_key(self, service_account_id, api_key_name_or_id)

Delete an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to delete the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
) -> None:
    """Delete an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            delete the API key.
        api_key_name_or_id: The name or ID of the API key to delete.
    """
    self._delete_resource(
        resource_id=api_key_name_or_id,
        route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
    )
delete_artifact(self, artifact_id)

Deletes an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_artifact(self, artifact_id: UUID) -> None:
    """Deletes an artifact.

    Args:
        artifact_id: The ID of the artifact to delete.
    """
    self._delete_resource(resource_id=artifact_id, route=ARTIFACTS)
delete_artifact_version(self, artifact_version_id)

Deletes an artifact version.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_artifact_version(self, artifact_version_id: UUID) -> None:
    """Deletes an artifact version.

    Args:
        artifact_version_id: The ID of the artifact version to delete.
    """
    self._delete_resource(
        resource_id=artifact_version_id, route=ARTIFACT_VERSIONS
    )
delete_authorized_device(self, device_id)

Deletes an OAuth 2.0 authorized device.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_authorized_device(self, device_id: UUID) -> None:
    """Deletes an OAuth 2.0 authorized device.

    Args:
        device_id: The ID of the device to delete.
    """
    self._delete_resource(resource_id=device_id, route=DEVICES)
delete_build(self, build_id)

Deletes a build.

Parameters:

Name Type Description Default
build_id UUID

The ID of the build to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_build(self, build_id: UUID) -> None:
    """Deletes a build.

    Args:
        build_id: The ID of the build to delete.
    """
    self._delete_resource(
        resource_id=build_id,
        route=PIPELINE_BUILDS,
    )

    # ----------------------
    # Pipeline Deployments
    # ----------------------
delete_code_repository(self, code_repository_id)

Deletes a code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_code_repository(self, code_repository_id: UUID) -> None:
    """Deletes a code repository.

    Args:
        code_repository_id: The ID of the code repository to delete.
    """
    self._delete_resource(
        resource_id=code_repository_id, route=CODE_REPOSITORIES
    )
delete_deployment(self, deployment_id)

Deletes a deployment.

Parameters:

Name Type Description Default
deployment_id UUID

The ID of the deployment to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_deployment(self, deployment_id: UUID) -> None:
    """Deletes a deployment.

    Args:
        deployment_id: The ID of the deployment to delete.
    """
    self._delete_resource(
        resource_id=deployment_id,
        route=PIPELINE_DEPLOYMENTS,
    )
delete_event_source(self, event_source_id)

Delete an event_source.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_event_source(self, event_source_id: UUID) -> None:
    """Delete an event_source.

    Args:
        event_source_id: The ID of the event_source to delete.
    """
    self._delete_resource(
        resource_id=event_source_id,
        route=EVENT_SOURCES,
    )
delete_flavor(self, flavor_id)

Delete a stack component flavor.

Parameters:

Name Type Description Default
flavor_id UUID

The ID of the stack component flavor to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_flavor(self, flavor_id: UUID) -> None:
    """Delete a stack component flavor.

    Args:
        flavor_id: The ID of the stack component flavor to delete.
    """
    self._delete_resource(
        resource_id=flavor_id,
        route=FLAVORS,
    )
delete_model(self, model_name_or_id)

Deletes a model.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model to be deleted.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_model(self, model_name_or_id: Union[str, UUID]) -> None:
    """Deletes a model.

    Args:
        model_name_or_id: name or id of the model to be deleted.
    """
    self._delete_resource(resource_id=model_name_or_id, route=MODELS)
delete_model_version(self, model_version_id)

Deletes a model version.

Parameters:

Name Type Description Default
model_version_id UUID

name or id of the model version to be deleted.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_model_version(
    self,
    model_version_id: UUID,
) -> None:
    """Deletes a model version.

    Args:
        model_version_id: name or id of the model version to be deleted.
    """
    self._delete_resource(
        resource_id=model_version_id,
        route=f"{MODEL_VERSIONS}",
    )

Deletes a model version to artifact link.

Parameters:

Name Type Description Default
model_version_id UUID

ID of the model version containing the link.

required
model_version_artifact_link_name_or_id Union[str, uuid.UUID]

name or ID of the model version to artifact link to be deleted.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_model_version_artifact_link(
    self,
    model_version_id: UUID,
    model_version_artifact_link_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a model version to artifact link.

    Args:
        model_version_id: ID of the model version containing the link.
        model_version_artifact_link_name_or_id: name or ID of the model
            version to artifact link to be deleted.
    """
    self._delete_resource(
        resource_id=model_version_artifact_link_name_or_id,
        route=f"{MODEL_VERSIONS}/{model_version_id}{ARTIFACTS}",
    )

Deletes a model version to pipeline run link.

Parameters:

Name Type Description Default
model_version_id UUID

ID of the model version containing the link.

required
model_version_pipeline_run_link_name_or_id Union[str, uuid.UUID]

name or ID of the model version to pipeline run link to be deleted.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_model_version_pipeline_run_link(
    self,
    model_version_id: UUID,
    model_version_pipeline_run_link_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a model version to pipeline run link.

    Args:
        model_version_id: ID of the model version containing the link.
        model_version_pipeline_run_link_name_or_id: name or ID of the model version to pipeline run link to be deleted.
    """
    self._delete_resource(
        resource_id=model_version_pipeline_run_link_name_or_id,
        route=f"{MODEL_VERSIONS}/{model_version_id}{RUNS}",
    )
delete_pipeline(self, pipeline_id)

Deletes a pipeline.

Parameters:

Name Type Description Default
pipeline_id UUID

The ID of the pipeline to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_pipeline(self, pipeline_id: UUID) -> None:
    """Deletes a pipeline.

    Args:
        pipeline_id: The ID of the pipeline to delete.
    """
    self._delete_resource(
        resource_id=pipeline_id,
        route=PIPELINES,
    )
delete_run(self, run_id)

Deletes a pipeline run.

Parameters:

Name Type Description Default
run_id UUID

The ID of the pipeline run to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_run(self, run_id: UUID) -> None:
    """Deletes a pipeline run.

    Args:
        run_id: The ID of the pipeline run to delete.
    """
    self._delete_resource(
        resource_id=run_id,
        route=RUNS,
    )
delete_schedule(self, schedule_id)

Deletes a schedule.

Parameters:

Name Type Description Default
schedule_id UUID

The ID of the schedule to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_schedule(self, schedule_id: UUID) -> None:
    """Deletes a schedule.

    Args:
        schedule_id: The ID of the schedule to delete.
    """
    self._delete_resource(
        resource_id=schedule_id,
        route=SCHEDULES,
    )
delete_secret(self, secret_id)

Delete a secret.

Parameters:

Name Type Description Default
secret_id UUID

The id of the secret to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_secret(self, secret_id: UUID) -> None:
    """Delete a secret.

    Args:
        secret_id: The id of the secret to delete.
    """
    self._delete_resource(
        resource_id=secret_id,
        route=SECRETS,
    )
delete_service_account(self, service_account_name_or_id)

Delete a service account.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or the ID of the service account to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
) -> None:
    """Delete a service account.

    Args:
        service_account_name_or_id: The name or the ID of the service
            account to delete.
    """
    self._delete_resource(
        resource_id=service_account_name_or_id,
        route=SERVICE_ACCOUNTS,
    )
delete_service_connector(self, service_connector_id)

Deletes a service connector.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_service_connector(self, service_connector_id: UUID) -> None:
    """Deletes a service connector.

    Args:
        service_connector_id: The ID of the service connector to delete.
    """
    self._delete_resource(
        resource_id=service_connector_id, route=SERVICE_CONNECTORS
    )
delete_stack(self, stack_id)

Delete a stack.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_stack(self, stack_id: UUID) -> None:
    """Delete a stack.

    Args:
        stack_id: The ID of the stack to delete.
    """
    self._delete_resource(
        resource_id=stack_id,
        route=STACKS,
    )
delete_stack_component(self, component_id)

Delete a stack component.

Parameters:

Name Type Description Default
component_id UUID

The ID of the stack component to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_stack_component(self, component_id: UUID) -> None:
    """Delete a stack component.

    Args:
        component_id: The ID of the stack component to delete.
    """
    self._delete_resource(
        resource_id=component_id,
        route=STACK_COMPONENTS,
    )
delete_tag(self, tag_name_or_id)

Deletes a tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_tag(
    self,
    tag_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a tag.

    Args:
        tag_name_or_id: name or id of the tag to delete.
    """
    self._delete_resource(resource_id=tag_name_or_id, route=TAGS)
delete_trigger(self, trigger_id)

Delete an trigger.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_trigger(self, trigger_id: UUID) -> None:
    """Delete an trigger.

    Args:
        trigger_id: The ID of the trigger to delete.
    """
    self._delete_resource(
        resource_id=trigger_id,
        route=TRIGGERS,
    )
delete_trigger_execution(self, trigger_execution_id)

Delete a trigger execution.

Parameters:

Name Type Description Default
trigger_execution_id UUID

The ID of the trigger execution to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_trigger_execution(self, trigger_execution_id: UUID) -> None:
    """Delete a trigger execution.

    Args:
        trigger_execution_id: The ID of the trigger execution to delete.
    """
    self._delete_resource(
        resource_id=trigger_execution_id,
        route=TRIGGER_EXECUTIONS,
    )
delete_user(self, user_name_or_id)

Deletes a user.

Parameters:

Name Type Description Default
user_name_or_id Union[str, uuid.UUID]

The name or ID of the user to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
    """Deletes a user.

    Args:
        user_name_or_id: The name or ID of the user to delete.
    """
    self._delete_resource(
        resource_id=user_name_or_id,
        route=USERS,
    )
delete_workspace(self, workspace_name_or_id)

Deletes a workspace.

Parameters:

Name Type Description Default
workspace_name_or_id Union[str, uuid.UUID]

Name or ID of the workspace to delete.

required
Source code in zenml/zen_stores/rest_zen_store.py
def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
    """Deletes a workspace.

    Args:
        workspace_name_or_id: Name or ID of the workspace to delete.
    """
    self._delete_resource(
        resource_id=workspace_name_or_id,
        route=WORKSPACES,
    )
get(self, path, params=None, **kwargs)

Make a GET request to the given endpoint path.

Parameters:

Name Type Description Default
path str

The path to the endpoint.

required
params Optional[Dict[str, Any]]

The query parameters to pass to the endpoint.

None
kwargs Any

Additional keyword arguments to pass to the request.

{}

Returns:

Type Description
Union[Dict[str, Any], List[Any], str, int, float, bool]

The response body.

Source code in zenml/zen_stores/rest_zen_store.py
def get(
    self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
    """Make a GET request to the given endpoint path.

    Args:
        path: The path to the endpoint.
        params: The query parameters to pass to the endpoint.
        kwargs: Additional keyword arguments to pass to the request.

    Returns:
        The response body.
    """
    logger.debug(f"Sending GET request to {path}...")
    return self._request(
        "GET", self.url + API + VERSION_1 + path, params=params, **kwargs
    )
get_api_key(self, service_account_id, api_key_name_or_id, hydrate=True)

Get an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to fetch the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
APIKeyResponse

The API key with the given ID.

Source code in zenml/zen_stores/rest_zen_store.py
def get_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    hydrate: bool = True,
) -> APIKeyResponse:
    """Get an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to fetch
            the API key.
        api_key_name_or_id: The name or ID of the API key to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The API key with the given ID.
    """
    return self._get_resource(
        resource_id=api_key_name_or_id,
        route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
        response_model=APIKeyResponse,
        params={"hydrate": hydrate},
    )
get_api_token(self, pipeline_id=None, schedule_id=None, expires_minutes=None)

Get an API token for a workload.

Parameters:

Name Type Description Default
pipeline_id Optional[uuid.UUID]

The ID of the pipeline to get a token for.

None
schedule_id Optional[uuid.UUID]

The ID of the schedule to get a token for.

None
expires_minutes Optional[int]

The number of minutes for which the token should be valid. If not provided, the token will be valid indefinitely.

None

Returns:

Type Description
str

The API token.

Exceptions:

Type Description
ValueError

if the server response is not valid.

Source code in zenml/zen_stores/rest_zen_store.py
def get_api_token(
    self,
    pipeline_id: Optional[UUID] = None,
    schedule_id: Optional[UUID] = None,
    expires_minutes: Optional[int] = None,
) -> str:
    """Get an API token for a workload.

    Args:
        pipeline_id: The ID of the pipeline to get a token for.
        schedule_id: The ID of the schedule to get a token for.
        expires_minutes: The number of minutes for which the token should
            be valid. If not provided, the token will be valid indefinitely.

    Returns:
        The API token.

    Raises:
        ValueError: if the server response is not valid.
    """
    params: Dict[str, Any] = {}
    if pipeline_id:
        params["pipeline_id"] = pipeline_id
    if schedule_id:
        params["schedule_id"] = schedule_id
    if expires_minutes:
        params["expires_minutes"] = expires_minutes
    response_body = self.get(API_TOKEN, params=params)
    if not isinstance(response_body, str):
        raise ValueError(
            f"Bad API Response. Expected API token, got "
            f"{type(response_body)}"
        )
    return response_body
get_artifact(self, artifact_id, hydrate=True)

Gets an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactResponse

The artifact.

Source code in zenml/zen_stores/rest_zen_store.py
def get_artifact(
    self, artifact_id: UUID, hydrate: bool = True
) -> ArtifactResponse:
    """Gets an artifact.

    Args:
        artifact_id: The ID of the artifact to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact.
    """
    return self._get_resource(
        resource_id=artifact_id,
        route=ARTIFACTS,
        response_model=ArtifactResponse,
        params={"hydrate": hydrate},
    )
get_artifact_version(self, artifact_version_id, hydrate=True)

Gets an artifact.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactVersionResponse

The artifact version.

Source code in zenml/zen_stores/rest_zen_store.py
def get_artifact_version(
    self, artifact_version_id: UUID, hydrate: bool = True
) -> ArtifactVersionResponse:
    """Gets an artifact.

    Args:
        artifact_version_id: The ID of the artifact version to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact version.
    """
    return self._get_resource(
        resource_id=artifact_version_id,
        route=ARTIFACT_VERSIONS,
        response_model=ArtifactVersionResponse,
        params={"hydrate": hydrate},
    )
get_artifact_visualization(self, artifact_visualization_id, hydrate=True)

Gets an artifact visualization.

Parameters:

Name Type Description Default
artifact_visualization_id UUID

The ID of the artifact visualization to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactVisualizationResponse

The artifact visualization.

Source code in zenml/zen_stores/rest_zen_store.py
def get_artifact_visualization(
    self, artifact_visualization_id: UUID, hydrate: bool = True
) -> ArtifactVisualizationResponse:
    """Gets an artifact visualization.

    Args:
        artifact_visualization_id: The ID of the artifact visualization to
            get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact visualization.
    """
    return self._get_resource(
        resource_id=artifact_visualization_id,
        route=ARTIFACT_VISUALIZATIONS,
        response_model=ArtifactVisualizationResponse,
        params={"hydrate": hydrate},
    )
get_authorized_device(self, device_id, hydrate=True)

Gets a specific OAuth 2.0 authorized device.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
OAuthDeviceResponse

The requested device, if it was found.

Source code in zenml/zen_stores/rest_zen_store.py
def get_authorized_device(
    self, device_id: UUID, hydrate: bool = True
) -> OAuthDeviceResponse:
    """Gets a specific OAuth 2.0 authorized device.

    Args:
        device_id: The ID of the device to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested device, if it was found.
    """
    return self._get_resource(
        resource_id=device_id,
        route=DEVICES,
        response_model=OAuthDeviceResponse,
        params={"hydrate": hydrate},
    )
get_build(self, build_id, hydrate=True)

Get a build with a given ID.

Parameters:

Name Type Description Default
build_id UUID

ID of the build.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineBuildResponse

The build.

Source code in zenml/zen_stores/rest_zen_store.py
def get_build(
    self, build_id: UUID, hydrate: bool = True
) -> PipelineBuildResponse:
    """Get a build with a given ID.

    Args:
        build_id: ID of the build.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The build.
    """
    return self._get_resource(
        resource_id=build_id,
        route=PIPELINE_BUILDS,
        response_model=PipelineBuildResponse,
        params={"hydrate": hydrate},
    )
get_code_reference(self, code_reference_id, hydrate=True)

Gets a code reference.

Parameters:

Name Type Description Default
code_reference_id UUID

The ID of the code reference to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
CodeReferenceResponse

The code reference.

Source code in zenml/zen_stores/rest_zen_store.py
def get_code_reference(
    self, code_reference_id: UUID, hydrate: bool = True
) -> CodeReferenceResponse:
    """Gets a code reference.

    Args:
        code_reference_id: The ID of the code reference to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The code reference.
    """
    return self._get_resource(
        resource_id=code_reference_id,
        route=CODE_REFERENCES,
        response_model=CodeReferenceResponse,
        params={"hydrate": hydrate},
    )
get_code_repository(self, code_repository_id, hydrate=True)

Gets a specific code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
CodeRepositoryResponse

The requested code repository, if it was found.

Source code in zenml/zen_stores/rest_zen_store.py
def get_code_repository(
    self, code_repository_id: UUID, hydrate: bool = True
) -> CodeRepositoryResponse:
    """Gets a specific code repository.

    Args:
        code_repository_id: The ID of the code repository to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested code repository, if it was found.
    """
    return self._get_resource(
        resource_id=code_repository_id,
        route=CODE_REPOSITORIES,
        response_model=CodeRepositoryResponse,
        params={"hydrate": hydrate},
    )
get_deployment(self, deployment_id, hydrate=True)

Get a deployment with a given ID.

Parameters:

Name Type Description Default
deployment_id UUID

ID of the deployment.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineDeploymentResponse

The deployment.

Source code in zenml/zen_stores/rest_zen_store.py
def get_deployment(
    self, deployment_id: UUID, hydrate: bool = True
) -> PipelineDeploymentResponse:
    """Get a deployment with a given ID.

    Args:
        deployment_id: ID of the deployment.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The deployment.
    """
    return self._get_resource(
        resource_id=deployment_id,
        route=PIPELINE_DEPLOYMENTS,
        response_model=PipelineDeploymentResponse,
        params={"hydrate": hydrate},
    )
get_deployment_id(self)

Get the ID of the deployment.

Returns:

Type Description
UUID

The ID of the deployment.

Source code in zenml/zen_stores/rest_zen_store.py
def get_deployment_id(self) -> UUID:
    """Get the ID of the deployment.

    Returns:
        The ID of the deployment.
    """
    return self.get_store_info().id
get_event_source(self, event_source_id, hydrate=True)

Get an event_source by ID.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
EventSourceResponse

The event_source.

Source code in zenml/zen_stores/rest_zen_store.py
def get_event_source(
    self,
    event_source_id: UUID,
    hydrate: bool = True,
) -> EventSourceResponse:
    """Get an event_source by ID.

    Args:
        event_source_id: The ID of the event_source to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The event_source.
    """
    return self._get_resource(
        resource_id=event_source_id,
        route=EVENT_SOURCES,
        response_model=EventSourceResponse,
        params={"hydrate": hydrate},
    )
get_flavor(self, flavor_id, hydrate=True)

Get a stack component flavor by ID.

Parameters:

Name Type Description Default
flavor_id UUID

The ID of the stack component flavor to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
FlavorResponse

The stack component flavor.

Source code in zenml/zen_stores/rest_zen_store.py
def get_flavor(
    self, flavor_id: UUID, hydrate: bool = True
) -> FlavorResponse:
    """Get a stack component flavor by ID.

    Args:
        flavor_id: The ID of the stack component flavor to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack component flavor.
    """
    return self._get_resource(
        resource_id=flavor_id,
        route=FLAVORS,
        response_model=FlavorResponse,
        params={"hydrate": hydrate},
    )
get_logs(self, logs_id, hydrate=True)

Gets logs with the given ID.

Parameters:

Name Type Description Default
logs_id UUID

The ID of the logs to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
LogsResponse

The logs.

Source code in zenml/zen_stores/rest_zen_store.py
def get_logs(self, logs_id: UUID, hydrate: bool = True) -> LogsResponse:
    """Gets logs with the given ID.

    Args:
        logs_id: The ID of the logs to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The logs.
    """
    return self._get_resource(
        resource_id=logs_id,
        route=LOGS,
        response_model=LogsResponse,
        params={"hydrate": hydrate},
    )
get_model(self, model_name_or_id, hydrate=True)

Get an existing model.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model to be retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ModelResponse

The model of interest.

Source code in zenml/zen_stores/rest_zen_store.py
def get_model(
    self, model_name_or_id: Union[str, UUID], hydrate: bool = True
) -> ModelResponse:
    """Get an existing model.

    Args:
        model_name_or_id: name or id of the model to be retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The model of interest.
    """
    return self._get_resource(
        resource_id=model_name_or_id,
        route=MODELS,
        response_model=ModelResponse,
        params={"hydrate": hydrate},
    )
get_model_version(self, model_version_id, hydrate=True)

Get an existing model version.

Parameters:

Name Type Description Default
model_version_id UUID

name, id, stage or number of the model version to be retrieved. If skipped - latest is retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ModelVersionResponse

The model version of interest.

Source code in zenml/zen_stores/rest_zen_store.py
def get_model_version(
    self, model_version_id: UUID, hydrate: bool = True
) -> ModelVersionResponse:
    """Get an existing model version.

    Args:
        model_version_id: name, id, stage or number of the model version to
            be retrieved. If skipped - latest is retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The model version of interest.
    """
    return self._get_resource(
        resource_id=model_version_id,
        route=MODEL_VERSIONS,
        response_model=ModelVersionResponse,
        params={"hydrate": hydrate},
    )
get_or_create_run(self, pipeline_run)

Gets or creates a pipeline run.

If a run with the same ID or name already exists, it is returned. Otherwise, a new run is created.

Parameters:

Name Type Description Default
pipeline_run PipelineRunRequest

The pipeline run to get or create.

required

Returns:

Type Description
Tuple[zenml.models.v2.core.pipeline_run.PipelineRunResponse, bool]

The pipeline run, and a boolean indicating whether the run was created or not.

Source code in zenml/zen_stores/rest_zen_store.py
def get_or_create_run(
    self, pipeline_run: PipelineRunRequest
) -> Tuple[PipelineRunResponse, bool]:
    """Gets or creates a pipeline run.

    If a run with the same ID or name already exists, it is returned.
    Otherwise, a new run is created.

    Args:
        pipeline_run: The pipeline run to get or create.

    Returns:
        The pipeline run, and a boolean indicating whether the run was
        created or not.
    """
    return self._get_or_create_workspace_scoped_resource(
        resource=pipeline_run,
        route=RUNS,
        response_model=PipelineRunResponse,
    )
get_pipeline(self, pipeline_id, hydrate=True)

Get a pipeline with a given ID.

Parameters:

Name Type Description Default
pipeline_id UUID

ID of the pipeline.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineResponse

The pipeline.

Source code in zenml/zen_stores/rest_zen_store.py
def get_pipeline(
    self, pipeline_id: UUID, hydrate: bool = True
) -> PipelineResponse:
    """Get a pipeline with a given ID.

    Args:
        pipeline_id: ID of the pipeline.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The pipeline.
    """
    return self._get_resource(
        resource_id=pipeline_id,
        route=PIPELINES,
        response_model=PipelineResponse,
        params={"hydrate": hydrate},
    )
get_run(self, run_name_or_id, hydrate=True)

Gets a pipeline run.

Parameters:

Name Type Description Default
run_name_or_id Union[uuid.UUID, str]

The name or ID of the pipeline run to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineRunResponse

The pipeline run.

Source code in zenml/zen_stores/rest_zen_store.py
def get_run(
    self, run_name_or_id: Union[UUID, str], hydrate: bool = True
) -> PipelineRunResponse:
    """Gets a pipeline run.

    Args:
        run_name_or_id: The name or ID of the pipeline run to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The pipeline run.
    """
    return self._get_resource(
        resource_id=run_name_or_id,
        route=RUNS,
        response_model=PipelineRunResponse,
        params={"hydrate": hydrate},
    )
get_run_metadata(self, run_metadata_id, hydrate=True)

Gets run metadata with the given ID.

Parameters:

Name Type Description Default
run_metadata_id UUID

The ID of the run metadata to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
RunMetadataResponse

The run metadata.

Source code in zenml/zen_stores/rest_zen_store.py
def get_run_metadata(
    self, run_metadata_id: UUID, hydrate: bool = True
) -> RunMetadataResponse:
    """Gets run metadata with the given ID.

    Args:
        run_metadata_id: The ID of the run metadata to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The run metadata.
    """
    return self._get_resource(
        resource_id=run_metadata_id,
        route=RUN_METADATA,
        response_model=RunMetadataResponse,
        params={"hydrate": hydrate},
    )
get_run_step(self, step_run_id, hydrate=True)

Get a step run by ID.

Parameters:

Name Type Description Default
step_run_id UUID

The ID of the step run to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
StepRunResponse

The step run.

Source code in zenml/zen_stores/rest_zen_store.py
def get_run_step(
    self, step_run_id: UUID, hydrate: bool = True
) -> StepRunResponse:
    """Get a step run by ID.

    Args:
        step_run_id: The ID of the step run to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The step run.
    """
    return self._get_resource(
        resource_id=step_run_id,
        route=STEPS,
        response_model=StepRunResponse,
        params={"hydrate": hydrate},
    )
get_schedule(self, schedule_id, hydrate=True)

Get a schedule with a given ID.

Parameters:

Name Type Description Default
schedule_id UUID

ID of the schedule.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ScheduleResponse

The schedule.

Source code in zenml/zen_stores/rest_zen_store.py
def get_schedule(
    self, schedule_id: UUID, hydrate: bool = True
) -> ScheduleResponse:
    """Get a schedule with a given ID.

    Args:
        schedule_id: ID of the schedule.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The schedule.
    """
    return self._get_resource(
        resource_id=schedule_id,
        route=SCHEDULES,
        response_model=ScheduleResponse,
        params={"hydrate": hydrate},
    )
get_secret(self, secret_id, hydrate=True)

Get a secret by ID.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to fetch.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
SecretResponse

The secret.

Source code in zenml/zen_stores/rest_zen_store.py
def get_secret(
    self, secret_id: UUID, hydrate: bool = True
) -> SecretResponse:
    """Get a secret by ID.

    Args:
        secret_id: The ID of the secret to fetch.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The secret.
    """
    return self._get_resource(
        resource_id=secret_id,
        route=SECRETS,
        response_model=SecretResponse,
        params={"hydrate": hydrate},
    )
get_service_account(self, service_account_name_or_id, hydrate=True)

Gets a specific service account.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or ID of the service account to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ServiceAccountResponse

The requested service account, if it was found.

Source code in zenml/zen_stores/rest_zen_store.py
def get_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
    hydrate: bool = True,
) -> ServiceAccountResponse:
    """Gets a specific service account.

    Args:
        service_account_name_or_id: The name or ID of the service account to
            get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested service account, if it was found.
    """
    return self._get_resource(
        resource_id=service_account_name_or_id,
        route=SERVICE_ACCOUNTS,
        response_model=ServiceAccountResponse,
        params={"hydrate": hydrate},
    )
get_service_connector(self, service_connector_id, hydrate=True)

Gets a specific service connector.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ServiceConnectorResponse

The requested service connector, if it was found.

Source code in zenml/zen_stores/rest_zen_store.py
def get_service_connector(
    self, service_connector_id: UUID, hydrate: bool = True
) -> ServiceConnectorResponse:
    """Gets a specific service connector.

    Args:
        service_connector_id: The ID of the service connector to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested service connector, if it was found.
    """
    connector_model = self._get_resource(
        resource_id=service_connector_id,
        route=SERVICE_CONNECTORS,
        response_model=ServiceConnectorResponse,
        params={"expand_secrets": False, "hydrate": hydrate},
    )
    self._populate_connector_type(connector_model)
    return connector_model
get_service_connector_client(self, service_connector_id, resource_type=None, resource_id=None)

Get a service connector client for a service connector and given resource.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the base service connector to use.

required
resource_type Optional[str]

The type of resource to get a client for.

None
resource_id Optional[str]

The ID of the resource to get a client for.

None

Returns:

Type Description
ServiceConnectorResponse

A service connector client that can be used to access the given resource.

Source code in zenml/zen_stores/rest_zen_store.py
def get_service_connector_client(
    self,
    service_connector_id: UUID,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
) -> ServiceConnectorResponse:
    """Get a service connector client for a service connector and given resource.

    Args:
        service_connector_id: The ID of the base service connector to use.
        resource_type: The type of resource to get a client for.
        resource_id: The ID of the resource to get a client for.

    Returns:
        A service connector client that can be used to access the given
        resource.
    """
    params = {}
    if resource_type:
        params["resource_type"] = resource_type
    if resource_id:
        params["resource_id"] = resource_id
    response_body = self.get(
        f"{SERVICE_CONNECTORS}/{str(service_connector_id)}{SERVICE_CONNECTOR_CLIENT}",
        params=params,
    )

    connector = ServiceConnectorResponse.parse_obj(response_body)
    self._populate_connector_type(connector)
    return connector
get_service_connector_type(self, connector_type)

Returns the requested service connector type.

Parameters:

Name Type Description Default
connector_type str

the service connector type identifier.

required

Returns:

Type Description
ServiceConnectorTypeModel

The requested service connector type.

Source code in zenml/zen_stores/rest_zen_store.py
def get_service_connector_type(
    self,
    connector_type: str,
) -> ServiceConnectorTypeModel:
    """Returns the requested service connector type.

    Args:
        connector_type: the service connector type identifier.

    Returns:
        The requested service connector type.
    """
    # Use the local registry to get the service connector type, if it
    # exists.
    local_connector_type: Optional[ServiceConnectorTypeModel] = None
    if service_connector_registry.is_registered(connector_type):
        local_connector_type = (
            service_connector_registry.get_service_connector_type(
                connector_type
            )
        )
    try:
        response_body = self.get(
            f"{SERVICE_CONNECTOR_TYPES}/{connector_type}",
        )
        remote_connector_type = ServiceConnectorTypeModel.parse_obj(
            response_body
        )
        if local_connector_type:
            # If locally available, return the local connector type but
            # mark it as being remotely available.
            local_connector_type.remote = True
            return local_connector_type

        # Mark the remote connector type as being only remotely available
        remote_connector_type.local = False
        remote_connector_type.remote = True

        return remote_connector_type
    except KeyError:
        # If the service connector type is not found, check the local
        # registry.
        return service_connector_registry.get_service_connector_type(
            connector_type
        )
get_stack(self, stack_id, hydrate=True)

Get a stack by its unique ID.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
StackResponse

The stack with the given ID.

Source code in zenml/zen_stores/rest_zen_store.py
def get_stack(self, stack_id: UUID, hydrate: bool = True) -> StackResponse:
    """Get a stack by its unique ID.

    Args:
        stack_id: The ID of the stack to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack with the given ID.
    """
    return self._get_resource(
        resource_id=stack_id,
        route=STACKS,
        response_model=StackResponse,
        params={"hydrate": hydrate},
    )
get_stack_component(self, component_id, hydrate=True)

Get a stack component by ID.

Parameters:

Name Type Description Default
component_id UUID

The ID of the stack component to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ComponentResponse

The stack component.

Source code in zenml/zen_stores/rest_zen_store.py
def get_stack_component(
    self, component_id: UUID, hydrate: bool = True
) -> ComponentResponse:
    """Get a stack component by ID.

    Args:
        component_id: The ID of the stack component to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack component.
    """
    return self._get_resource(
        resource_id=component_id,
        route=STACK_COMPONENTS,
        response_model=ComponentResponse,
        params={"hydrate": hydrate},
    )
get_store_info(self)

Get information about the server.

Returns:

Type Description
ServerModel

Information about the server.

Source code in zenml/zen_stores/rest_zen_store.py
def get_store_info(self) -> ServerModel:
    """Get information about the server.

    Returns:
        Information about the server.
    """
    body = self.get(INFO)
    return ServerModel.parse_obj(body)
get_tag(self, tag_name_or_id, hydrate=True)

Get an existing tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to be retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TagResponse

The tag of interest.

Source code in zenml/zen_stores/rest_zen_store.py
def get_tag(
    self, tag_name_or_id: Union[str, UUID], hydrate: bool = True
) -> TagResponse:
    """Get an existing tag.

    Args:
        tag_name_or_id: name or id of the tag to be retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The tag of interest.
    """
    return self._get_resource(
        resource_id=tag_name_or_id,
        route=TAGS,
        response_model=TagResponse,
        params={"hydrate": hydrate},
    )
get_trigger(self, trigger_id, hydrate=True)

Get a trigger by ID.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TriggerResponse

The trigger.

Source code in zenml/zen_stores/rest_zen_store.py
def get_trigger(
    self,
    trigger_id: UUID,
    hydrate: bool = True,
) -> TriggerResponse:
    """Get a trigger by ID.

    Args:
        trigger_id: The ID of the trigger to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The trigger.
    """
    return self._get_resource(
        resource_id=trigger_id,
        route=TRIGGERS,
        response_model=TriggerResponse,
        params={"hydrate": hydrate},
    )
get_trigger_execution(self, trigger_execution_id, hydrate=True)

Get an trigger execution by ID.

Parameters:

Name Type Description Default
trigger_execution_id UUID

The ID of the trigger execution to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TriggerExecutionResponse

The trigger execution.

Source code in zenml/zen_stores/rest_zen_store.py
def get_trigger_execution(
    self,
    trigger_execution_id: UUID,
    hydrate: bool = True,
) -> TriggerExecutionResponse:
    """Get an trigger execution by ID.

    Args:
        trigger_execution_id: The ID of the trigger execution to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The trigger execution.
    """
    return self._get_resource(
        resource_id=trigger_execution_id,
        route=TRIGGER_EXECUTIONS,
        response_model=TriggerExecutionResponse,
        params={"hydrate": hydrate},
    )
get_user(self, user_name_or_id=None, include_private=False, hydrate=True)

Gets a specific user, when no id is specified get the active user.

The include_private parameter is ignored here as it is handled implicitly by the /current-user endpoint that is queried when no user_name_or_id is set. Raises a KeyError in case a user with that id does not exist.

Parameters:

Name Type Description Default
user_name_or_id Union[str, uuid.UUID]

The name or ID of the user to get.

None
include_private bool

Whether to include private user information.

False
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
UserResponse

The requested user, if it was found.

Source code in zenml/zen_stores/rest_zen_store.py
def get_user(
    self,
    user_name_or_id: Optional[Union[str, UUID]] = None,
    include_private: bool = False,
    hydrate: bool = True,
) -> UserResponse:
    """Gets a specific user, when no id is specified get the active user.

    The `include_private` parameter is ignored here as it is handled
    implicitly by the /current-user endpoint that is queried when no
    user_name_or_id is set. Raises a KeyError in case a user with that id
    does not exist.

    Args:
        user_name_or_id: The name or ID of the user to get.
        include_private: Whether to include private user information.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested user, if it was found.
    """
    if user_name_or_id:
        return self._get_resource(
            resource_id=user_name_or_id,
            route=USERS,
            response_model=UserResponse,
            params={"hydrate": hydrate},
        )
    else:
        body = self.get(CURRENT_USER, params={"hydrate": hydrate})
        return UserResponse.parse_obj(body)
get_workspace(self, workspace_name_or_id, hydrate=True)

Get an existing workspace by name or ID.

Parameters:

Name Type Description Default
workspace_name_or_id Union[uuid.UUID, str]

Name or ID of the workspace to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
WorkspaceResponse

The requested workspace.

Source code in zenml/zen_stores/rest_zen_store.py
def get_workspace(
    self, workspace_name_or_id: Union[UUID, str], hydrate: bool = True
) -> WorkspaceResponse:
    """Get an existing workspace by name or ID.

    Args:
        workspace_name_or_id: Name or ID of the workspace to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested workspace.
    """
    return self._get_resource(
        resource_id=workspace_name_or_id,
        route=WORKSPACES,
        response_model=WorkspaceResponse,
        params={"hydrate": hydrate},
    )
list_api_keys(self, service_account_id, filter_model, hydrate=False)

List all API keys for a service account matching the given filter criteria.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to list the API keys.

required
filter_model APIKeyFilter

All filter parameters including pagination params

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[APIKeyResponse]

A list of all API keys matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_api_keys(
    self,
    service_account_id: UUID,
    filter_model: APIKeyFilter,
    hydrate: bool = False,
) -> Page[APIKeyResponse]:
    """List all API keys for a service account matching the given filter criteria.

    Args:
        service_account_id: The ID of the service account for which to list
            the API keys.
        filter_model: All filter parameters including pagination
            params
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all API keys matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
        response_model=APIKeyResponse,
        filter_model=filter_model,
        params={"hydrate": hydrate},
    )
list_artifact_versions(self, artifact_version_filter_model, hydrate=False)

List all artifact versions matching the given filter criteria.

Parameters:

Name Type Description Default
artifact_version_filter_model ArtifactVersionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ArtifactVersionResponse]

A list of all artifact versions matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_artifact_versions(
    self,
    artifact_version_filter_model: ArtifactVersionFilter,
    hydrate: bool = False,
) -> Page[ArtifactVersionResponse]:
    """List all artifact versions matching the given filter criteria.

    Args:
        artifact_version_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all artifact versions matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=ARTIFACT_VERSIONS,
        response_model=ArtifactVersionResponse,
        filter_model=artifact_version_filter_model,
        params={"hydrate": hydrate},
    )
list_artifacts(self, filter_model, hydrate=False)

List all artifacts matching the given filter criteria.

Parameters:

Name Type Description Default
filter_model ArtifactFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ArtifactResponse]

A list of all artifacts matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_artifacts(
    self, filter_model: ArtifactFilter, hydrate: bool = False
) -> Page[ArtifactResponse]:
    """List all artifacts matching the given filter criteria.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all artifacts matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=ARTIFACTS,
        response_model=ArtifactResponse,
        filter_model=filter_model,
        params={"hydrate": hydrate},
    )
list_authorized_devices(self, filter_model, hydrate=False)

List all OAuth 2.0 authorized devices for a user.

Parameters:

Name Type Description Default
filter_model OAuthDeviceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[OAuthDeviceResponse]

A page of all matching OAuth 2.0 authorized devices.

Source code in zenml/zen_stores/rest_zen_store.py
def list_authorized_devices(
    self, filter_model: OAuthDeviceFilter, hydrate: bool = False
) -> Page[OAuthDeviceResponse]:
    """List all OAuth 2.0 authorized devices for a user.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all matching OAuth 2.0 authorized devices.
    """
    return self._list_paginated_resources(
        route=DEVICES,
        response_model=OAuthDeviceResponse,
        filter_model=filter_model,
        params={"hydrate": hydrate},
    )
list_builds(self, build_filter_model, hydrate=False)

List all builds matching the given filter criteria.

Parameters:

Name Type Description Default
build_filter_model PipelineBuildFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineBuildResponse]

A page of all builds matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_builds(
    self,
    build_filter_model: PipelineBuildFilter,
    hydrate: bool = False,
) -> Page[PipelineBuildResponse]:
    """List all builds matching the given filter criteria.

    Args:
        build_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all builds matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=PIPELINE_BUILDS,
        response_model=PipelineBuildResponse,
        filter_model=build_filter_model,
        params={"hydrate": hydrate},
    )
list_code_repositories(self, filter_model, hydrate=False)

List all code repositories.

Parameters:

Name Type Description Default
filter_model CodeRepositoryFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[CodeRepositoryResponse]

A page of all code repositories.

Source code in zenml/zen_stores/rest_zen_store.py
def list_code_repositories(
    self,
    filter_model: CodeRepositoryFilter,
    hydrate: bool = False,
) -> Page[CodeRepositoryResponse]:
    """List all code repositories.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all code repositories.
    """
    return self._list_paginated_resources(
        route=CODE_REPOSITORIES,
        response_model=CodeRepositoryResponse,
        filter_model=filter_model,
        params={"hydrate": hydrate},
    )
list_deployments(self, deployment_filter_model, hydrate=False)

List all deployments matching the given filter criteria.

Parameters:

Name Type Description Default
deployment_filter_model PipelineDeploymentFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineDeploymentResponse]

A page of all deployments matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_deployments(
    self,
    deployment_filter_model: PipelineDeploymentFilter,
    hydrate: bool = False,
) -> Page[PipelineDeploymentResponse]:
    """List all deployments matching the given filter criteria.

    Args:
        deployment_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all deployments matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=PIPELINE_DEPLOYMENTS,
        response_model=PipelineDeploymentResponse,
        filter_model=deployment_filter_model,
        params={"hydrate": hydrate},
    )
list_event_sources(self, event_source_filter_model, hydrate=False)

List all event_sources matching the given filter criteria.

Parameters:

Name Type Description Default
event_source_filter_model EventSourceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[EventSourceResponse]

A list of all event_sources matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_event_sources(
    self,
    event_source_filter_model: EventSourceFilter,
    hydrate: bool = False,
) -> Page[EventSourceResponse]:
    """List all event_sources matching the given filter criteria.

    Args:
        event_source_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all event_sources matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=EVENT_SOURCES,
        response_model=EventSourceResponse,
        filter_model=event_source_filter_model,
        params={"hydrate": hydrate},
    )
list_flavors(self, flavor_filter_model, hydrate=False)

List all stack component flavors matching the given filter criteria.

Parameters:

Name Type Description Default
flavor_filter_model FlavorFilter

All filter parameters including pagination params

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[FlavorResponse]

List of all the stack component flavors matching the given criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_flavors(
    self,
    flavor_filter_model: FlavorFilter,
    hydrate: bool = False,
) -> Page[FlavorResponse]:
    """List all stack component flavors matching the given filter criteria.

    Args:
        flavor_filter_model: All filter parameters including pagination
            params
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        List of all the stack component flavors matching the given criteria.
    """
    return self._list_paginated_resources(
        route=FLAVORS,
        response_model=FlavorResponse,
        filter_model=flavor_filter_model,
        params={"hydrate": hydrate},
    )

Get all model version to artifact links by filter.

Parameters:

Name Type Description Default
model_version_artifact_link_filter_model ModelVersionArtifactFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionArtifactResponse]

A page of all model version to artifact links.

Source code in zenml/zen_stores/rest_zen_store.py
def list_model_version_artifact_links(
    self,
    model_version_artifact_link_filter_model: ModelVersionArtifactFilter,
    hydrate: bool = False,
) -> Page[ModelVersionArtifactResponse]:
    """Get all model version to artifact links by filter.

    Args:
        model_version_artifact_link_filter_model: All filter parameters
            including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model version to artifact links.
    """
    return self._list_paginated_resources(
        route=MODEL_VERSION_ARTIFACTS,
        response_model=ModelVersionArtifactResponse,
        filter_model=model_version_artifact_link_filter_model,
        params={"hydrate": hydrate},
    )

Get all model version to pipeline run links by filter.

Parameters:

Name Type Description Default
model_version_pipeline_run_link_filter_model ModelVersionPipelineRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionPipelineRunResponse]

A page of all model version to pipeline run links.

Source code in zenml/zen_stores/rest_zen_store.py
def list_model_version_pipeline_run_links(
    self,
    model_version_pipeline_run_link_filter_model: ModelVersionPipelineRunFilter,
    hydrate: bool = False,
) -> Page[ModelVersionPipelineRunResponse]:
    """Get all model version to pipeline run links by filter.

    Args:
        model_version_pipeline_run_link_filter_model: All filter parameters
            including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model version to pipeline run links.
    """
    return self._list_paginated_resources(
        route=MODEL_VERSION_PIPELINE_RUNS,
        response_model=ModelVersionPipelineRunResponse,
        filter_model=model_version_pipeline_run_link_filter_model,
        params={"hydrate": hydrate},
    )
list_model_versions(self, model_version_filter_model, model_name_or_id=None, hydrate=False)

Get all model versions by filter.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model containing the model versions.

None
model_version_filter_model ModelVersionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionResponse]

A page of all model versions.

Source code in zenml/zen_stores/rest_zen_store.py
def list_model_versions(
    self,
    model_version_filter_model: ModelVersionFilter,
    model_name_or_id: Optional[Union[str, UUID]] = None,
    hydrate: bool = False,
) -> Page[ModelVersionResponse]:
    """Get all model versions by filter.

    Args:
        model_name_or_id: name or id of the model containing the model
            versions.
        model_version_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model versions.
    """
    if model_name_or_id:
        return self._list_paginated_resources(
            route=f"{MODELS}/{model_name_or_id}{MODEL_VERSIONS}",
            response_model=ModelVersionResponse,
            filter_model=model_version_filter_model,
            params={"hydrate": hydrate},
        )
    else:
        return self._list_paginated_resources(
            route=MODEL_VERSIONS,
            response_model=ModelVersionResponse,
            filter_model=model_version_filter_model,
            params={"hydrate": hydrate},
        )
list_models(self, model_filter_model, hydrate=False)

Get all models by filter.

Parameters:

Name Type Description Default
model_filter_model ModelFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelResponse]

A page of all models.

Source code in zenml/zen_stores/rest_zen_store.py
def list_models(
    self,
    model_filter_model: ModelFilter,
    hydrate: bool = False,
) -> Page[ModelResponse]:
    """Get all models by filter.

    Args:
        model_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all models.
    """
    return self._list_paginated_resources(
        route=MODELS,
        response_model=ModelResponse,
        filter_model=model_filter_model,
        params={"hydrate": hydrate},
    )
list_pipelines(self, pipeline_filter_model, hydrate=False)

List all pipelines matching the given filter criteria.

Parameters:

Name Type Description Default
pipeline_filter_model PipelineFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineResponse]

A list of all pipelines matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_pipelines(
    self,
    pipeline_filter_model: PipelineFilter,
    hydrate: bool = False,
) -> Page[PipelineResponse]:
    """List all pipelines matching the given filter criteria.

    Args:
        pipeline_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all pipelines matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=PIPELINES,
        response_model=PipelineResponse,
        filter_model=pipeline_filter_model,
        params={"hydrate": hydrate},
    )
list_run_metadata(self, run_metadata_filter_model, hydrate=False)

List run metadata.

Parameters:

Name Type Description Default
run_metadata_filter_model RunMetadataFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[RunMetadataResponse]

The run metadata.

Source code in zenml/zen_stores/rest_zen_store.py
def list_run_metadata(
    self,
    run_metadata_filter_model: RunMetadataFilter,
    hydrate: bool = False,
) -> Page[RunMetadataResponse]:
    """List run metadata.

    Args:
        run_metadata_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The run metadata.
    """
    return self._list_paginated_resources(
        route=RUN_METADATA,
        response_model=RunMetadataResponse,
        filter_model=run_metadata_filter_model,
        params={"hydrate": hydrate},
    )
list_run_steps(self, step_run_filter_model, hydrate=False)

List all step runs matching the given filter criteria.

Parameters:

Name Type Description Default
step_run_filter_model StepRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[StepRunResponse]

A list of all step runs matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_run_steps(
    self,
    step_run_filter_model: StepRunFilter,
    hydrate: bool = False,
) -> Page[StepRunResponse]:
    """List all step runs matching the given filter criteria.

    Args:
        step_run_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all step runs matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=STEPS,
        response_model=StepRunResponse,
        filter_model=step_run_filter_model,
        params={"hydrate": hydrate},
    )
list_runs(self, runs_filter_model, hydrate=False)

List all pipeline runs matching the given filter criteria.

Parameters:

Name Type Description Default
runs_filter_model PipelineRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineRunResponse]

A list of all pipeline runs matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_runs(
    self,
    runs_filter_model: PipelineRunFilter,
    hydrate: bool = False,
) -> Page[PipelineRunResponse]:
    """List all pipeline runs matching the given filter criteria.

    Args:
        runs_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all pipeline runs matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=RUNS,
        response_model=PipelineRunResponse,
        filter_model=runs_filter_model,
        params={"hydrate": hydrate},
    )
list_schedules(self, schedule_filter_model, hydrate=False)

List all schedules in the workspace.

Parameters:

Name Type Description Default
schedule_filter_model ScheduleFilter

All filter parameters including pagination params

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ScheduleResponse]

A list of schedules.

Source code in zenml/zen_stores/rest_zen_store.py
def list_schedules(
    self,
    schedule_filter_model: ScheduleFilter,
    hydrate: bool = False,
) -> Page[ScheduleResponse]:
    """List all schedules in the workspace.

    Args:
        schedule_filter_model: All filter parameters including pagination
            params
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of schedules.
    """
    return self._list_paginated_resources(
        route=SCHEDULES,
        response_model=ScheduleResponse,
        filter_model=schedule_filter_model,
        params={"hydrate": hydrate},
    )
list_secrets(self, secret_filter_model, hydrate=False)

List all secrets matching the given filter criteria.

Note that returned secrets do not include any secret values. To fetch the secret values, use get_secret.

Parameters:

Name Type Description Default
secret_filter_model SecretFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[SecretResponse]

A list of all secrets matching the filter criteria, with pagination information and sorted according to the filter criteria. The returned secrets do not include any secret values, only metadata. To fetch the secret values, use get_secret individually with each secret.

Source code in zenml/zen_stores/rest_zen_store.py
def list_secrets(
    self, secret_filter_model: SecretFilter, hydrate: bool = False
) -> Page[SecretResponse]:
    """List all secrets matching the given filter criteria.

    Note that returned secrets do not include any secret values. To fetch
    the secret values, use `get_secret`.

    Args:
        secret_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all secrets matching the filter criteria, with pagination
        information and sorted according to the filter criteria. The
        returned secrets do not include any secret values, only metadata. To
        fetch the secret values, use `get_secret` individually with each
        secret.
    """
    return self._list_paginated_resources(
        route=SECRETS,
        response_model=SecretResponse,
        filter_model=secret_filter_model,
        params={"hydrate": hydrate},
    )
list_service_accounts(self, filter_model, hydrate=False)

List all service accounts.

Parameters:

Name Type Description Default
filter_model ServiceAccountFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ServiceAccountResponse]

A list of filtered service accounts.

Source code in zenml/zen_stores/rest_zen_store.py
def list_service_accounts(
    self, filter_model: ServiceAccountFilter, hydrate: bool = False
) -> Page[ServiceAccountResponse]:
    """List all service accounts.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of filtered service accounts.
    """
    return self._list_paginated_resources(
        route=SERVICE_ACCOUNTS,
        response_model=ServiceAccountResponse,
        filter_model=filter_model,
        params={"hydrate": hydrate},
    )
list_service_connector_resources(self, workspace_name_or_id, connector_type=None, resource_type=None, resource_id=None)

List resources that can be accessed by service connectors.

Parameters:

Name Type Description Default
workspace_name_or_id Union[str, uuid.UUID]

The name or ID of the workspace to scope to.

required
connector_type Optional[str]

The type of service connector to scope to.

None
resource_type Optional[str]

The type of resource to scope to.

None
resource_id Optional[str]

The ID of the resource to scope to.

None

Returns:

Type Description
List[zenml.models.v2.misc.service_connector_type.ServiceConnectorResourcesModel]

The matching list of resources that available service connectors have access to.

Source code in zenml/zen_stores/rest_zen_store.py
def list_service_connector_resources(
    self,
    workspace_name_or_id: Union[str, UUID],
    connector_type: Optional[str] = None,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
) -> List[ServiceConnectorResourcesModel]:
    """List resources that can be accessed by service connectors.

    Args:
        workspace_name_or_id: The name or ID of the workspace to scope to.
        connector_type: The type of service connector to scope to.
        resource_type: The type of resource to scope to.
        resource_id: The ID of the resource to scope to.

    Returns:
        The matching list of resources that available service
        connectors have access to.
    """
    params = {}
    if connector_type:
        params["connector_type"] = connector_type
    if resource_type:
        params["resource_type"] = resource_type
    if resource_id:
        params["resource_id"] = resource_id
    response_body = self.get(
        f"{WORKSPACES}/{workspace_name_or_id}{SERVICE_CONNECTORS}{SERVICE_CONNECTOR_RESOURCES}",
        params=params,
    )

    assert isinstance(response_body, list)
    resource_list = [
        ServiceConnectorResourcesModel.parse_obj(item)
        for item in response_body
    ]

    self._populate_connector_type(*resource_list)

    # For service connectors with types that are only locally available,
    # we need to retrieve the resource list locally
    for idx, resources in enumerate(resource_list):
        if isinstance(resources.connector_type, str):
            # Skip connector types that are neither locally nor remotely
            # available
            continue
        if resources.connector_type.remote:
            # Skip connector types that are remotely available
            continue

        # Retrieve the resource list locally
        assert resources.id is not None
        connector = self.get_service_connector(resources.id)
        connector_instance = (
            service_connector_registry.instantiate_connector(
                model=connector
            )
        )

        try:
            local_resources = connector_instance.verify(
                resource_type=resource_type,
                resource_id=resource_id,
            )
        except (ValueError, AuthorizationException) as e:
            logger.error(
                f'Failed to fetch {resource_type or "available"} '
                f"resources from service connector {connector.name}/"
                f"{connector.id}: {e}"
            )
            continue

        resource_list[idx] = local_resources

    return resource_list
list_service_connector_types(self, connector_type=None, resource_type=None, auth_method=None)

Get a list of service connector types.

Parameters:

Name Type Description Default
connector_type Optional[str]

Filter by connector type.

None
resource_type Optional[str]

Filter by resource type.

None
auth_method Optional[str]

Filter by authentication method.

None

Returns:

Type Description
List[zenml.models.v2.misc.service_connector_type.ServiceConnectorTypeModel]

List of service connector types.

Source code in zenml/zen_stores/rest_zen_store.py
def list_service_connector_types(
    self,
    connector_type: Optional[str] = None,
    resource_type: Optional[str] = None,
    auth_method: Optional[str] = None,
) -> List[ServiceConnectorTypeModel]:
    """Get a list of service connector types.

    Args:
        connector_type: Filter by connector type.
        resource_type: Filter by resource type.
        auth_method: Filter by authentication method.

    Returns:
        List of service connector types.
    """
    params = {}
    if connector_type:
        params["connector_type"] = connector_type
    if resource_type:
        params["resource_type"] = resource_type
    if auth_method:
        params["auth_method"] = auth_method
    response_body = self.get(
        SERVICE_CONNECTOR_TYPES,
        params=params,
    )

    assert isinstance(response_body, list)
    remote_connector_types = [
        ServiceConnectorTypeModel.parse_obj(item) for item in response_body
    ]

    # Mark the remote connector types as being only remotely available
    for c in remote_connector_types:
        c.local = False
        c.remote = True

    local_connector_types = (
        service_connector_registry.list_service_connector_types(
            connector_type=connector_type,
            resource_type=resource_type,
            auth_method=auth_method,
        )
    )

    # Add the connector types in the local registry to the list of
    # connector types available remotely. Overwrite those that have
    # the same connector type but mark them as being remotely available.
    connector_types_map = {
        connector_type.connector_type: connector_type
        for connector_type in remote_connector_types
    }

    for connector in local_connector_types:
        if connector.connector_type in connector_types_map:
            connector.remote = True
        connector_types_map[connector.connector_type] = connector

    return list(connector_types_map.values())
list_service_connectors(self, filter_model, hydrate=False)

List all service connectors.

Parameters:

Name Type Description Default
filter_model ServiceConnectorFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ServiceConnectorResponse]

A page of all service connectors.

Source code in zenml/zen_stores/rest_zen_store.py
def list_service_connectors(
    self,
    filter_model: ServiceConnectorFilter,
    hydrate: bool = False,
) -> Page[ServiceConnectorResponse]:
    """List all service connectors.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all service connectors.
    """
    connector_models = self._list_paginated_resources(
        route=SERVICE_CONNECTORS,
        response_model=ServiceConnectorResponse,
        filter_model=filter_model,
        params={"expand_secrets": False, "hydrate": hydrate},
    )
    self._populate_connector_type(*connector_models.items)
    return connector_models
list_stack_components(self, component_filter_model, hydrate=False)

List all stack components matching the given filter criteria.

Parameters:

Name Type Description Default
component_filter_model ComponentFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ComponentResponse]

A list of all stack components matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_stack_components(
    self,
    component_filter_model: ComponentFilter,
    hydrate: bool = False,
) -> Page[ComponentResponse]:
    """List all stack components matching the given filter criteria.

    Args:
        component_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all stack components matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=STACK_COMPONENTS,
        response_model=ComponentResponse,
        filter_model=component_filter_model,
        params={"hydrate": hydrate},
    )
list_stacks(self, stack_filter_model, hydrate=False)

List all stacks matching the given filter criteria.

Parameters:

Name Type Description Default
stack_filter_model StackFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[StackResponse]

A list of all stacks matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_stacks(
    self, stack_filter_model: StackFilter, hydrate: bool = False
) -> Page[StackResponse]:
    """List all stacks matching the given filter criteria.

    Args:
        stack_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all stacks matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=STACKS,
        response_model=StackResponse,
        filter_model=stack_filter_model,
        params={"hydrate": hydrate},
    )
list_tags(self, tag_filter_model, hydrate=False)

Get all tags by filter.

Parameters:

Name Type Description Default
tag_filter_model TagFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TagResponse]

A page of all tags.

Source code in zenml/zen_stores/rest_zen_store.py
def list_tags(
    self,
    tag_filter_model: TagFilter,
    hydrate: bool = False,
) -> Page[TagResponse]:
    """Get all tags by filter.

    Args:
        tag_filter_model: All filter parameters including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all tags.
    """
    return self._list_paginated_resources(
        route=TAGS,
        response_model=TagResponse,
        filter_model=tag_filter_model,
        params={"hydrate": hydrate},
    )
list_trigger_executions(self, trigger_execution_filter_model, hydrate=False)

List all trigger executions matching the given filter criteria.

Parameters:

Name Type Description Default
trigger_execution_filter_model TriggerExecutionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TriggerExecutionResponse]

A list of all trigger executions matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_trigger_executions(
    self,
    trigger_execution_filter_model: TriggerExecutionFilter,
    hydrate: bool = False,
) -> Page[TriggerExecutionResponse]:
    """List all trigger executions matching the given filter criteria.

    Args:
        trigger_execution_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all trigger executions matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=TRIGGER_EXECUTIONS,
        response_model=TriggerExecutionResponse,
        filter_model=trigger_execution_filter_model,
        params={"hydrate": hydrate},
    )
list_triggers(self, trigger_filter_model, hydrate=False)

List all triggers matching the given filter criteria.

Parameters:

Name Type Description Default
trigger_filter_model TriggerFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TriggerResponse]

A list of all triggers matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_triggers(
    self,
    trigger_filter_model: TriggerFilter,
    hydrate: bool = False,
) -> Page[TriggerResponse]:
    """List all triggers matching the given filter criteria.

    Args:
        trigger_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all triggers matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=TRIGGERS,
        response_model=TriggerResponse,
        filter_model=trigger_filter_model,
        params={"hydrate": hydrate},
    )
list_users(self, user_filter_model, hydrate=False)

List all users.

Parameters:

Name Type Description Default
user_filter_model UserFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[UserResponse]

A list of all users.

Source code in zenml/zen_stores/rest_zen_store.py
def list_users(
    self,
    user_filter_model: UserFilter,
    hydrate: bool = False,
) -> Page[UserResponse]:
    """List all users.

    Args:
        user_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all users.
    """
    return self._list_paginated_resources(
        route=USERS,
        response_model=UserResponse,
        filter_model=user_filter_model,
        params={"hydrate": hydrate},
    )
list_workspaces(self, workspace_filter_model, hydrate=False)

List all workspace matching the given filter criteria.

Parameters:

Name Type Description Default
workspace_filter_model WorkspaceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[WorkspaceResponse]

A list of all workspace matching the filter criteria.

Source code in zenml/zen_stores/rest_zen_store.py
def list_workspaces(
    self,
    workspace_filter_model: WorkspaceFilter,
    hydrate: bool = False,
) -> Page[WorkspaceResponse]:
    """List all workspace matching the given filter criteria.

    Args:
        workspace_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all workspace matching the filter criteria.
    """
    return self._list_paginated_resources(
        route=WORKSPACES,
        response_model=WorkspaceResponse,
        filter_model=workspace_filter_model,
        params={"hydrate": hydrate},
    )
post(self, path, body, params=None, **kwargs)

Make a POST request to the given endpoint path.

Parameters:

Name Type Description Default
path str

The path to the endpoint.

required
body BaseModel

The body to send.

required
params Optional[Dict[str, Any]]

The query parameters to pass to the endpoint.

None
kwargs Any

Additional keyword arguments to pass to the request.

{}

Returns:

Type Description
Union[Dict[str, Any], List[Any], str, int, float, bool]

The response body.

Source code in zenml/zen_stores/rest_zen_store.py
def post(
    self,
    path: str,
    body: BaseModel,
    params: Optional[Dict[str, Any]] = None,
    **kwargs: Any,
) -> Json:
    """Make a POST request to the given endpoint path.

    Args:
        path: The path to the endpoint.
        body: The body to send.
        params: The query parameters to pass to the endpoint.
        kwargs: Additional keyword arguments to pass to the request.

    Returns:
        The response body.
    """
    logger.debug(f"Sending POST request to {path}...")
    return self._request(
        "POST",
        self.url + API + VERSION_1 + path,
        data=body.json(),
        params=params,
        **kwargs,
    )
prune_artifact_versions(self, only_versions=True)

Prunes unused artifact versions and their artifacts.

Parameters:

Name Type Description Default
only_versions bool

Only delete artifact versions, keeping artifacts

True
Source code in zenml/zen_stores/rest_zen_store.py
def prune_artifact_versions(
    self,
    only_versions: bool = True,
) -> None:
    """Prunes unused artifact versions and their artifacts.

    Args:
        only_versions: Only delete artifact versions, keeping artifacts
    """
    self.delete(
        path=ARTIFACT_VERSIONS, params={"only_versions": only_versions}
    )
put(self, path, body=None, params=None, **kwargs)

Make a PUT request to the given endpoint path.

Parameters:

Name Type Description Default
path str

The path to the endpoint.

required
body Optional[pydantic.main.BaseModel]

The body to send.

None
params Optional[Dict[str, Any]]

The query parameters to pass to the endpoint.

None
kwargs Any

Additional keyword arguments to pass to the request.

{}

Returns:

Type Description
Union[Dict[str, Any], List[Any], str, int, float, bool]

The response body.

Source code in zenml/zen_stores/rest_zen_store.py
def put(
    self,
    path: str,
    body: Optional[BaseModel] = None,
    params: Optional[Dict[str, Any]] = None,
    **kwargs: Any,
) -> Json:
    """Make a PUT request to the given endpoint path.

    Args:
        path: The path to the endpoint.
        body: The body to send.
        params: The query parameters to pass to the endpoint.
        kwargs: Additional keyword arguments to pass to the request.

    Returns:
        The response body.
    """
    logger.debug(f"Sending PUT request to {path}...")
    data = body.json(exclude_unset=True) if body else None
    return self._request(
        "PUT",
        self.url + API + VERSION_1 + path,
        data=data,
        params=params,
        **kwargs,
    )
restore_secrets(self, ignore_errors=False, delete_secrets=False)

Restore all secrets from the configured backup secrets store.

Parameters:

Name Type Description Default
ignore_errors bool

Whether to ignore individual errors during the restore process and attempt to restore all secrets.

False
delete_secrets bool

Whether to delete the secrets that have been successfully restored from the backup secrets store. Setting this flag effectively moves all secrets from the backup secrets store to the primary secrets store.

False
Source code in zenml/zen_stores/rest_zen_store.py
def restore_secrets(
    self, ignore_errors: bool = False, delete_secrets: bool = False
) -> None:
    """Restore all secrets from the configured backup secrets store.

    Args:
        ignore_errors: Whether to ignore individual errors during the
            restore process and attempt to restore all secrets.
        delete_secrets: Whether to delete the secrets that have been
            successfully restored from the backup secrets store. Setting
            this flag effectively moves all secrets from the backup secrets
            store to the primary secrets store.
    """
    params: Dict[str, Any] = {
        "ignore_errors": ignore_errors,
        "delete_secrets": delete_secrets,
    }
    self.put(
        f"{SECRETS_OPERATIONS}{SECRETS_RESTORE}",
        params=params,
    )
rotate_api_key(self, service_account_id, api_key_name_or_id, rotate_request)

Rotate an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to rotate the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to rotate.

required
rotate_request APIKeyRotateRequest

The rotate request on the API key.

required

Returns:

Type Description
APIKeyResponse

The updated API key.

Source code in zenml/zen_stores/rest_zen_store.py
def rotate_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    rotate_request: APIKeyRotateRequest,
) -> APIKeyResponse:
    """Rotate an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            rotate the API key.
        api_key_name_or_id: The name or ID of the API key to rotate.
        rotate_request: The rotate request on the API key.

    Returns:
        The updated API key.
    """
    response_body = self.put(
        f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}/{str(api_key_name_or_id)}{API_KEY_ROTATE}",
        body=rotate_request,
    )
    return APIKeyResponse.parse_obj(response_body)
set_api_key(self, api_key)

Set the API key to use for authentication.

Parameters:

Name Type Description Default
api_key str

The API key to use for authentication.

required
Source code in zenml/zen_stores/rest_zen_store.py
def set_api_key(self, api_key: str) -> None:
    """Set the API key to use for authentication.

    Args:
        api_key: The API key to use for authentication.
    """
    self.config.api_key = api_key
    self.clear_session()
    # TODO: find a way to persist the API key in the configuration file
    #  without calling _write_config() here.
    # This is the only place where we need to explicitly call
    # _write_config() to persist the global configuration.
    GlobalConfiguration()._write_config()
update_api_key(self, service_account_id, api_key_name_or_id, api_key_update)

Update an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to update the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to update.

required
api_key_update APIKeyUpdate

The update request on the API key.

required

Returns:

Type Description
APIKeyResponse

The updated API key.

Source code in zenml/zen_stores/rest_zen_store.py
def update_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    api_key_update: APIKeyUpdate,
) -> APIKeyResponse:
    """Update an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            update the API key.
        api_key_name_or_id: The name or ID of the API key to update.
        api_key_update: The update request on the API key.

    Returns:
        The updated API key.
    """
    return self._update_resource(
        resource_id=api_key_name_or_id,
        resource_update=api_key_update,
        route=f"{SERVICE_ACCOUNTS}/{str(service_account_id)}{API_KEYS}",
        response_model=APIKeyResponse,
    )
update_artifact(self, artifact_id, artifact_update)

Updates an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to update.

required
artifact_update ArtifactUpdate

The update to be applied to the artifact.

required

Returns:

Type Description
ArtifactResponse

The updated artifact.

Source code in zenml/zen_stores/rest_zen_store.py
def update_artifact(
    self, artifact_id: UUID, artifact_update: ArtifactUpdate
) -> ArtifactResponse:
    """Updates an artifact.

    Args:
        artifact_id: The ID of the artifact to update.
        artifact_update: The update to be applied to the artifact.

    Returns:
        The updated artifact.
    """
    return self._update_resource(
        resource_id=artifact_id,
        resource_update=artifact_update,
        response_model=ArtifactResponse,
        route=ARTIFACTS,
    )
update_artifact_version(self, artifact_version_id, artifact_version_update)

Updates an artifact version.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to update.

required
artifact_version_update ArtifactVersionUpdate

The update to be applied to the artifact version.

required

Returns:

Type Description
ArtifactVersionResponse

The updated artifact version.

Source code in zenml/zen_stores/rest_zen_store.py
def update_artifact_version(
    self,
    artifact_version_id: UUID,
    artifact_version_update: ArtifactVersionUpdate,
) -> ArtifactVersionResponse:
    """Updates an artifact version.

    Args:
        artifact_version_id: The ID of the artifact version to update.
        artifact_version_update: The update to be applied to the artifact
            version.

    Returns:
        The updated artifact version.
    """
    return self._update_resource(
        resource_id=artifact_version_id,
        resource_update=artifact_version_update,
        response_model=ArtifactVersionResponse,
        route=ARTIFACT_VERSIONS,
    )
update_authorized_device(self, device_id, update)

Updates an existing OAuth 2.0 authorized device for internal use.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to update.

required
update OAuthDeviceUpdate

The update to be applied to the device.

required

Returns:

Type Description
OAuthDeviceResponse

The updated OAuth 2.0 authorized device.

Source code in zenml/zen_stores/rest_zen_store.py
def update_authorized_device(
    self, device_id: UUID, update: OAuthDeviceUpdate
) -> OAuthDeviceResponse:
    """Updates an existing OAuth 2.0 authorized device for internal use.

    Args:
        device_id: The ID of the device to update.
        update: The update to be applied to the device.

    Returns:
        The updated OAuth 2.0 authorized device.
    """
    return self._update_resource(
        resource_id=device_id,
        resource_update=update,
        response_model=OAuthDeviceResponse,
        route=DEVICES,
    )
update_code_repository(self, code_repository_id, update)

Updates an existing code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to update.

required
update CodeRepositoryUpdate

The update to be applied to the code repository.

required

Returns:

Type Description
CodeRepositoryResponse

The updated code repository.

Source code in zenml/zen_stores/rest_zen_store.py
def update_code_repository(
    self, code_repository_id: UUID, update: CodeRepositoryUpdate
) -> CodeRepositoryResponse:
    """Updates an existing code repository.

    Args:
        code_repository_id: The ID of the code repository to update.
        update: The update to be applied to the code repository.

    Returns:
        The updated code repository.
    """
    return self._update_resource(
        resource_id=code_repository_id,
        resource_update=update,
        response_model=CodeRepositoryResponse,
        route=CODE_REPOSITORIES,
    )
update_event_source(self, event_source_id, event_source_update)

Update an existing event_source.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to update.

required
event_source_update EventSourceUpdate

The update to be applied to the event_source.

required

Returns:

Type Description
EventSourceResponse

The updated event_source.

Source code in zenml/zen_stores/rest_zen_store.py
def update_event_source(
    self,
    event_source_id: UUID,
    event_source_update: EventSourceUpdate,
) -> EventSourceResponse:
    """Update an existing event_source.

    Args:
        event_source_id: The ID of the event_source to update.
        event_source_update: The update to be applied to the event_source.

    Returns:
        The updated event_source.
    """
    return self._update_resource(
        resource_id=event_source_id,
        resource_update=event_source_update,
        route=EVENT_SOURCES,
        response_model=EventSourceResponse,
    )
update_flavor(self, flavor_id, flavor_update)

Updates an existing user.

Parameters:

Name Type Description Default
flavor_id UUID

The id of the flavor to update.

required
flavor_update FlavorUpdate

The update to be applied to the flavor.

required

Returns:

Type Description
FlavorResponse

The updated flavor.

Source code in zenml/zen_stores/rest_zen_store.py
def update_flavor(
    self, flavor_id: UUID, flavor_update: FlavorUpdate
) -> FlavorResponse:
    """Updates an existing user.

    Args:
        flavor_id: The id of the flavor to update.
        flavor_update: The update to be applied to the flavor.

    Returns:
        The updated flavor.
    """
    return self._update_resource(
        resource_id=flavor_id,
        resource_update=flavor_update,
        route=FLAVORS,
        response_model=FlavorResponse,
    )
update_model(self, model_id, model_update)

Updates an existing model.

Parameters:

Name Type Description Default
model_id UUID

UUID of the model to be updated.

required
model_update ModelUpdate

the Model to be updated.

required

Returns:

Type Description
ModelResponse

The updated model.

Source code in zenml/zen_stores/rest_zen_store.py
def update_model(
    self,
    model_id: UUID,
    model_update: ModelUpdate,
) -> ModelResponse:
    """Updates an existing model.

    Args:
        model_id: UUID of the model to be updated.
        model_update: the Model to be updated.

    Returns:
        The updated model.
    """
    return self._update_resource(
        resource_id=model_id,
        resource_update=model_update,
        route=MODELS,
        response_model=ModelResponse,
    )
update_model_version(self, model_version_id, model_version_update_model)

Get all model versions by filter.

Parameters:

Name Type Description Default
model_version_id UUID

The ID of model version to be updated.

required
model_version_update_model ModelVersionUpdate

The model version to be updated.

required

Returns:

Type Description
ModelVersionResponse

An updated model version.

Source code in zenml/zen_stores/rest_zen_store.py
def update_model_version(
    self,
    model_version_id: UUID,
    model_version_update_model: ModelVersionUpdate,
) -> ModelVersionResponse:
    """Get all model versions by filter.

    Args:
        model_version_id: The ID of model version to be updated.
        model_version_update_model: The model version to be updated.

    Returns:
        An updated model version.

    """
    return self._update_resource(
        resource_id=model_version_id,
        resource_update=model_version_update_model,
        route=MODEL_VERSIONS,
        response_model=ModelVersionResponse,
    )
update_pipeline(self, pipeline_id, pipeline_update)

Updates a pipeline.

Parameters:

Name Type Description Default
pipeline_id UUID

The ID of the pipeline to be updated.

required
pipeline_update PipelineUpdate

The update to be applied.

required

Returns:

Type Description
PipelineResponse

The updated pipeline.

Source code in zenml/zen_stores/rest_zen_store.py
def update_pipeline(
    self, pipeline_id: UUID, pipeline_update: PipelineUpdate
) -> PipelineResponse:
    """Updates a pipeline.

    Args:
        pipeline_id: The ID of the pipeline to be updated.
        pipeline_update: The update to be applied.

    Returns:
        The updated pipeline.
    """
    return self._update_resource(
        resource_id=pipeline_id,
        resource_update=pipeline_update,
        route=PIPELINES,
        response_model=PipelineResponse,
    )
update_run(self, run_id, run_update)

Updates a pipeline run.

Parameters:

Name Type Description Default
run_id UUID

The ID of the pipeline run to update.

required
run_update PipelineRunUpdate

The update to be applied to the pipeline run.

required

Returns:

Type Description
PipelineRunResponse

The updated pipeline run.

Source code in zenml/zen_stores/rest_zen_store.py
def update_run(
    self, run_id: UUID, run_update: PipelineRunUpdate
) -> PipelineRunResponse:
    """Updates a pipeline run.

    Args:
        run_id: The ID of the pipeline run to update.
        run_update: The update to be applied to the pipeline run.


    Returns:
        The updated pipeline run.
    """
    return self._update_resource(
        resource_id=run_id,
        resource_update=run_update,
        response_model=PipelineRunResponse,
        route=RUNS,
    )
update_run_step(self, step_run_id, step_run_update)

Updates a step run.

Parameters:

Name Type Description Default
step_run_id UUID

The ID of the step to update.

required
step_run_update StepRunUpdate

The update to be applied to the step.

required

Returns:

Type Description
StepRunResponse

The updated step run.

Source code in zenml/zen_stores/rest_zen_store.py
def update_run_step(
    self,
    step_run_id: UUID,
    step_run_update: StepRunUpdate,
) -> StepRunResponse:
    """Updates a step run.

    Args:
        step_run_id: The ID of the step to update.
        step_run_update: The update to be applied to the step.

    Returns:
        The updated step run.
    """
    return self._update_resource(
        resource_id=step_run_id,
        resource_update=step_run_update,
        response_model=StepRunResponse,
        route=STEPS,
    )
update_schedule(self, schedule_id, schedule_update)

Updates a schedule.

Parameters:

Name Type Description Default
schedule_id UUID

The ID of the schedule to be updated.

required
schedule_update ScheduleUpdate

The update to be applied.

required

Returns:

Type Description
ScheduleResponse

The updated schedule.

Source code in zenml/zen_stores/rest_zen_store.py
def update_schedule(
    self,
    schedule_id: UUID,
    schedule_update: ScheduleUpdate,
) -> ScheduleResponse:
    """Updates a schedule.

    Args:
        schedule_id: The ID of the schedule to be updated.
        schedule_update: The update to be applied.

    Returns:
        The updated schedule.
    """
    return self._update_resource(
        resource_id=schedule_id,
        resource_update=schedule_update,
        route=SCHEDULES,
        response_model=ScheduleResponse,
    )
update_secret(self, secret_id, secret_update)

Updates a secret.

Secret values that are specified as None in the update that are present in the existing secret are removed from the existing secret. Values that are present in both secrets are overwritten. All other values in both the existing secret and the update are kept (merged).

If the update includes a change of name or scope, the scoping rules enforced in the secrets store are used to validate the update:

  • only one workspace-scoped secret with the given name can exist in the target workspace.
  • only one user-scoped secret with the given name can exist in the target workspace for the target user.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_update SecretUpdate

The update to be applied.

required

Returns:

Type Description
SecretResponse

The updated secret.

Source code in zenml/zen_stores/rest_zen_store.py
def update_secret(
    self, secret_id: UUID, secret_update: SecretUpdate
) -> SecretResponse:
    """Updates a secret.

    Secret values that are specified as `None` in the update that are
    present in the existing secret are removed from the existing secret.
    Values that are present in both secrets are overwritten. All other
    values in both the existing secret and the update are kept (merged).

    If the update includes a change of name or scope, the scoping rules
    enforced in the secrets store are used to validate the update:

      - only one workspace-scoped secret with the given name can exist
        in the target workspace.
      - only one user-scoped secret with the given name can exist in the
        target workspace for the target user.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_update: The update to be applied.

    Returns:
        The updated secret.
    """
    return self._update_resource(
        resource_id=secret_id,
        resource_update=secret_update,
        route=SECRETS,
        response_model=SecretResponse,
        # The default endpoint behavior is to replace all secret values
        # with the values in the update. We want to merge the values
        # instead.
        params=dict(patch_values=True),
    )
update_service_account(self, service_account_name_or_id, service_account_update)

Updates an existing service account.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or the ID of the service account to update.

required
service_account_update ServiceAccountUpdate

The update to be applied to the service account.

required

Returns:

Type Description
ServiceAccountResponse

The updated service account.

Source code in zenml/zen_stores/rest_zen_store.py
def update_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
    service_account_update: ServiceAccountUpdate,
) -> ServiceAccountResponse:
    """Updates an existing service account.

    Args:
        service_account_name_or_id: The name or the ID of the service
            account to update.
        service_account_update: The update to be applied to the service
            account.

    Returns:
        The updated service account.
    """
    return self._update_resource(
        resource_id=service_account_name_or_id,
        resource_update=service_account_update,
        route=SERVICE_ACCOUNTS,
        response_model=ServiceAccountResponse,
    )
update_service_connector(self, service_connector_id, update)

Updates an existing service connector.

The update model contains the fields to be updated. If a field value is set to None in the model, the field is not updated, but there are special rules concerning some fields:

  • the configuration and secrets fields together represent a full valid configuration update, not just a partial update. If either is set (i.e. not None) in the update, their values are merged together and will replace the existing configuration and secrets values.
  • the resource_id field value is also a full replacement value: if set to None, the resource ID is removed from the service connector.
  • the expiration_seconds field value is also a full replacement value: if set to None, the expiration is removed from the service connector.
  • the secret_id field value in the update is ignored, given that secrets are managed internally by the ZenML store.
  • the labels field is also a full labels update: if set (i.e. not None), all existing labels are removed and replaced by the new labels in the update.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to update.

required
update ServiceConnectorUpdate

The update to be applied to the service connector.

required

Returns:

Type Description
ServiceConnectorResponse

The updated service connector.

Source code in zenml/zen_stores/rest_zen_store.py
def update_service_connector(
    self, service_connector_id: UUID, update: ServiceConnectorUpdate
) -> ServiceConnectorResponse:
    """Updates an existing service connector.

    The update model contains the fields to be updated. If a field value is
    set to None in the model, the field is not updated, but there are
    special rules concerning some fields:

    * the `configuration` and `secrets` fields together represent a full
    valid configuration update, not just a partial update. If either is
    set (i.e. not None) in the update, their values are merged together and
    will replace the existing configuration and secrets values.
    * the `resource_id` field value is also a full replacement value: if set
    to `None`, the resource ID is removed from the service connector.
    * the `expiration_seconds` field value is also a full replacement value:
    if set to `None`, the expiration is removed from the service connector.
    * the `secret_id` field value in the update is ignored, given that
    secrets are managed internally by the ZenML store.
    * the `labels` field is also a full labels update: if set (i.e. not
    `None`), all existing labels are removed and replaced by the new labels
    in the update.

    Args:
        service_connector_id: The ID of the service connector to update.
        update: The update to be applied to the service connector.

    Returns:
        The updated service connector.
    """
    connector_model = self._update_resource(
        resource_id=service_connector_id,
        resource_update=update,
        response_model=ServiceConnectorResponse,
        route=SERVICE_CONNECTORS,
    )
    self._populate_connector_type(connector_model)
    return connector_model
update_stack(self, stack_id, stack_update)

Update a stack.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack update.

required
stack_update StackUpdate

The update request on the stack.

required

Returns:

Type Description
StackResponse

The updated stack.

Source code in zenml/zen_stores/rest_zen_store.py
def update_stack(
    self, stack_id: UUID, stack_update: StackUpdate
) -> StackResponse:
    """Update a stack.

    Args:
        stack_id: The ID of the stack update.
        stack_update: The update request on the stack.

    Returns:
        The updated stack.
    """
    return self._update_resource(
        resource_id=stack_id,
        resource_update=stack_update,
        route=STACKS,
        response_model=StackResponse,
    )
update_stack_component(self, component_id, component_update)

Update an existing stack component.

Parameters:

Name Type Description Default
component_id UUID

The ID of the stack component to update.

required
component_update ComponentUpdate

The update to be applied to the stack component.

required

Returns:

Type Description
ComponentResponse

The updated stack component.

Source code in zenml/zen_stores/rest_zen_store.py
def update_stack_component(
    self,
    component_id: UUID,
    component_update: ComponentUpdate,
) -> ComponentResponse:
    """Update an existing stack component.

    Args:
        component_id: The ID of the stack component to update.
        component_update: The update to be applied to the stack component.

    Returns:
        The updated stack component.
    """
    return self._update_resource(
        resource_id=component_id,
        resource_update=component_update,
        route=STACK_COMPONENTS,
        response_model=ComponentResponse,
    )
update_tag(self, tag_name_or_id, tag_update_model)

Update tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to be updated.

required
tag_update_model TagUpdate

Tag to use for the update.

required

Returns:

Type Description
TagResponse

An updated tag.

Source code in zenml/zen_stores/rest_zen_store.py
def update_tag(
    self,
    tag_name_or_id: Union[str, UUID],
    tag_update_model: TagUpdate,
) -> TagResponse:
    """Update tag.

    Args:
        tag_name_or_id: name or id of the tag to be updated.
        tag_update_model: Tag to use for the update.

    Returns:
        An updated tag.
    """
    tag = self.get_tag(tag_name_or_id)
    return self._update_resource(
        resource_id=tag.id,
        resource_update=tag_update_model,
        route=TAGS,
        response_model=TagResponse,
    )
update_trigger(self, trigger_id, trigger_update)

Update an existing trigger.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger to update.

required
trigger_update TriggerUpdate

The update to be applied to the trigger.

required

Returns:

Type Description
TriggerResponse

The updated trigger.

Source code in zenml/zen_stores/rest_zen_store.py
def update_trigger(
    self,
    trigger_id: UUID,
    trigger_update: TriggerUpdate,
) -> TriggerResponse:
    """Update an existing trigger.

    Args:
        trigger_id: The ID of the trigger to update.
        trigger_update: The update to be applied to the trigger.

    Returns:
        The updated trigger.
    """
    return self._update_resource(
        resource_id=trigger_id,
        resource_update=trigger_update,
        route=TRIGGERS,
        response_model=TriggerResponse,
    )
update_user(self, user_id, user_update)

Updates an existing user.

Parameters:

Name Type Description Default
user_id UUID

The id of the user to update.

required
user_update UserUpdate

The update to be applied to the user.

required

Returns:

Type Description
UserResponse

The updated user.

Source code in zenml/zen_stores/rest_zen_store.py
def update_user(
    self, user_id: UUID, user_update: UserUpdate
) -> UserResponse:
    """Updates an existing user.

    Args:
        user_id: The id of the user to update.
        user_update: The update to be applied to the user.

    Returns:
        The updated user.
    """
    return self._update_resource(
        resource_id=user_id,
        resource_update=user_update,
        route=USERS,
        response_model=UserResponse,
    )
update_workspace(self, workspace_id, workspace_update)

Update an existing workspace.

Parameters:

Name Type Description Default
workspace_id UUID

The ID of the workspace to be updated.

required
workspace_update WorkspaceUpdate

The update to be applied to the workspace.

required

Returns:

Type Description
WorkspaceResponse

The updated workspace.

Source code in zenml/zen_stores/rest_zen_store.py
def update_workspace(
    self, workspace_id: UUID, workspace_update: WorkspaceUpdate
) -> WorkspaceResponse:
    """Update an existing workspace.

    Args:
        workspace_id: The ID of the workspace to be updated.
        workspace_update: The update to be applied to the workspace.

    Returns:
        The updated workspace.
    """
    return self._update_resource(
        resource_id=workspace_id,
        resource_update=workspace_update,
        route=WORKSPACES,
        response_model=WorkspaceResponse,
    )
verify_service_connector(self, service_connector_id, resource_type=None, resource_id=None, list_resources=True)

Verifies if a service connector instance has access to one or more resources.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to verify.

required
resource_type Optional[str]

The type of resource to verify access to.

None
resource_id Optional[str]

The ID of the resource to verify access to.

None
list_resources bool

If True, the list of all resources accessible through the service connector and matching the supplied resource type and ID are returned.

True

Returns:

Type Description
ServiceConnectorResourcesModel

The list of resources that the service connector has access to, scoped to the supplied resource type and ID, if provided.

Source code in zenml/zen_stores/rest_zen_store.py
def verify_service_connector(
    self,
    service_connector_id: UUID,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
    list_resources: bool = True,
) -> ServiceConnectorResourcesModel:
    """Verifies if a service connector instance has access to one or more resources.

    Args:
        service_connector_id: The ID of the service connector to verify.
        resource_type: The type of resource to verify access to.
        resource_id: The ID of the resource to verify access to.
        list_resources: If True, the list of all resources accessible
            through the service connector and matching the supplied resource
            type and ID are returned.

    Returns:
        The list of resources that the service connector has access to,
        scoped to the supplied resource type and ID, if provided.
    """
    params: Dict[str, Any] = {"list_resources": list_resources}
    if resource_type:
        params["resource_type"] = resource_type
    if resource_id:
        params["resource_id"] = resource_id
    response_body = self.put(
        f"{SERVICE_CONNECTORS}/{str(service_connector_id)}{SERVICE_CONNECTOR_VERIFY}",
        params=params,
    )

    resources = ServiceConnectorResourcesModel.parse_obj(response_body)
    self._populate_connector_type(resources)
    return resources
verify_service_connector_config(self, service_connector, list_resources=True)

Verifies if a service connector configuration has access to resources.

Parameters:

Name Type Description Default
service_connector ServiceConnectorRequest

The service connector configuration to verify.

required
list_resources bool

If True, the list of all resources accessible through the service connector and matching the supplied resource type and ID are returned.

True

Returns:

Type Description
ServiceConnectorResourcesModel

The list of resources that the service connector configuration has access to.

Source code in zenml/zen_stores/rest_zen_store.py
def verify_service_connector_config(
    self,
    service_connector: ServiceConnectorRequest,
    list_resources: bool = True,
) -> ServiceConnectorResourcesModel:
    """Verifies if a service connector configuration has access to resources.

    Args:
        service_connector: The service connector configuration to verify.
        list_resources: If True, the list of all resources accessible
            through the service connector and matching the supplied resource
            type and ID are returned.

    Returns:
        The list of resources that the service connector configuration has
        access to.
    """
    response_body = self.post(
        f"{SERVICE_CONNECTORS}{SERVICE_CONNECTOR_VERIFY}",
        body=service_connector,
        params={"list_resources": list_resources},
    )

    resources = ServiceConnectorResourcesModel.parse_obj(response_body)
    self._populate_connector_type(resources)
    return resources

RestZenStoreConfiguration (StoreConfiguration) pydantic-model

REST ZenML store configuration.

Attributes:

Name Type Description
type StoreType

The type of the store.

username Optional[str]

The username to use to connect to the Zen server.

password Optional[str]

The password to use to connect to the Zen server.

api_key Optional[str]

The service account API key to use to connect to the Zen server.

api_token Optional[str]

The API token to use to connect to the Zen server. Generated by the client and stored in the configuration file on the first login and every time the API key is refreshed.

verify_ssl Union[bool, str]

Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use or the CA bundle value itself.

http_timeout int

The timeout to use for all requests.

Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStoreConfiguration(StoreConfiguration):
    """REST ZenML store configuration.

    Attributes:
        type: The type of the store.
        username: The username to use to connect to the Zen server.
        password: The password to use to connect to the Zen server.
        api_key: The service account API key to use to connect to the Zen
            server.
        api_token: The API token to use to connect to the Zen server. Generated
            by the client and stored in the configuration file on the first
            login and every time the API key is refreshed.
        verify_ssl: Either a boolean, in which case it controls whether we
            verify the server's TLS certificate, or a string, in which case it
            must be a path to a CA bundle to use or the CA bundle value itself.
        http_timeout: The timeout to use for all requests.

    """

    type: StoreType = StoreType.REST

    username: Optional[str] = None
    password: Optional[str] = None
    api_key: Optional[str] = None
    api_token: Optional[str] = None
    verify_ssl: Union[bool, str] = True
    http_timeout: int = DEFAULT_HTTP_TIMEOUT

    @root_validator
    def validate_credentials(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validates the credentials provided in the values dictionary.

        Args:
            values: A dictionary containing the values to be validated.

        Raises:
            ValueError: If neither api_token nor username nor api_key is set.

        Returns:
            The values dictionary.
        """
        # Check if the values dictionary contains either an API token, an API
        # key or a username as non-empty strings.
        if (
            values.get("api_token")
            or values.get("username")
            or values.get("api_key")
        ):
            return values
        raise ValueError(
            "Neither api_token nor username nor api_key is set in the "
            "store config."
        )

    @validator("url")
    def validate_url(cls, url: str) -> str:
        """Validates that the URL is a well-formed REST store URL.

        Args:
            url: The URL to be validated.

        Returns:
            The validated URL without trailing slashes.

        Raises:
            ValueError: If the URL is not a well-formed REST store URL.
        """
        url = url.rstrip("/")
        scheme = re.search("^([a-z0-9]+://)", url)
        if scheme is None or scheme.group() not in ("https://", "http://"):
            raise ValueError(
                "Invalid URL for REST store: {url}. Should be in the form "
                "https://hostname[:port] or http://hostname[:port]."
            )

        # When running inside a container, if the URL uses localhost, the
        # target service will not be available. We try to replace localhost
        # with one of the special Docker or K3D internal hostnames.
        url = replace_localhost_with_internal_hostname(url)

        return url

    @validator("verify_ssl")
    def validate_verify_ssl(
        cls, verify_ssl: Union[bool, str]
    ) -> Union[bool, str]:
        """Validates that the verify_ssl either points to a file or is a bool.

        Args:
            verify_ssl: The verify_ssl value to be validated.

        Returns:
            The validated verify_ssl value.
        """
        secret_folder = Path(
            GlobalConfiguration().local_stores_path,
            "certificates",
        )
        if isinstance(verify_ssl, bool) or verify_ssl.startswith(
            str(secret_folder)
        ):
            return verify_ssl

        if os.path.isfile(verify_ssl):
            with open(verify_ssl, "r") as f:
                verify_ssl = f.read()

        fileio.makedirs(str(secret_folder))
        file_path = Path(secret_folder, "ca_bundle.pem")
        with open(file_path, "w") as f:
            f.write(verify_ssl)
        file_path.chmod(0o600)
        verify_ssl = str(file_path)

        return verify_ssl

    @classmethod
    def supports_url_scheme(cls, url: str) -> bool:
        """Check if a URL scheme is supported by this store.

        Args:
            url: The URL to check.

        Returns:
            True if the URL scheme is supported, False otherwise.
        """
        return urlparse(url).scheme in ("http", "https")

    def expand_certificates(self) -> None:
        """Expands the certificates in the verify_ssl field."""
        # Load the certificate values back into the configuration
        if isinstance(self.verify_ssl, str) and os.path.isfile(
            self.verify_ssl
        ):
            with open(self.verify_ssl, "r") as f:
                self.verify_ssl = f.read()

    class Config:
        """Pydantic configuration class."""

        # Don't validate attributes when assigning them. This is necessary
        # because the `verify_ssl` attribute can be expanded to the contents
        # of the certificate file.
        validate_assignment = False
        # Forbid extra attributes set in the class.
        extra = "forbid"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/rest_zen_store.py
class Config:
    """Pydantic configuration class."""

    # Don't validate attributes when assigning them. This is necessary
    # because the `verify_ssl` attribute can be expanded to the contents
    # of the certificate file.
    validate_assignment = False
    # Forbid extra attributes set in the class.
    extra = "forbid"
expand_certificates(self)

Expands the certificates in the verify_ssl field.

Source code in zenml/zen_stores/rest_zen_store.py
def expand_certificates(self) -> None:
    """Expands the certificates in the verify_ssl field."""
    # Load the certificate values back into the configuration
    if isinstance(self.verify_ssl, str) and os.path.isfile(
        self.verify_ssl
    ):
        with open(self.verify_ssl, "r") as f:
            self.verify_ssl = f.read()
supports_url_scheme(url) classmethod

Check if a URL scheme is supported by this store.

Parameters:

Name Type Description Default
url str

The URL to check.

required

Returns:

Type Description
bool

True if the URL scheme is supported, False otherwise.

Source code in zenml/zen_stores/rest_zen_store.py
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
    """Check if a URL scheme is supported by this store.

    Args:
        url: The URL to check.

    Returns:
        True if the URL scheme is supported, False otherwise.
    """
    return urlparse(url).scheme in ("http", "https")
validate_credentials(values) classmethod

Validates the credentials provided in the values dictionary.

Parameters:

Name Type Description Default
values Dict[str, Any]

A dictionary containing the values to be validated.

required

Exceptions:

Type Description
ValueError

If neither api_token nor username nor api_key is set.

Returns:

Type Description
Dict[str, Any]

The values dictionary.

Source code in zenml/zen_stores/rest_zen_store.py
@root_validator
def validate_credentials(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Validates the credentials provided in the values dictionary.

    Args:
        values: A dictionary containing the values to be validated.

    Raises:
        ValueError: If neither api_token nor username nor api_key is set.

    Returns:
        The values dictionary.
    """
    # Check if the values dictionary contains either an API token, an API
    # key or a username as non-empty strings.
    if (
        values.get("api_token")
        or values.get("username")
        or values.get("api_key")
    ):
        return values
    raise ValueError(
        "Neither api_token nor username nor api_key is set in the "
        "store config."
    )
validate_url(url) classmethod

Validates that the URL is a well-formed REST store URL.

Parameters:

Name Type Description Default
url str

The URL to be validated.

required

Returns:

Type Description
str

The validated URL without trailing slashes.

Exceptions:

Type Description
ValueError

If the URL is not a well-formed REST store URL.

Source code in zenml/zen_stores/rest_zen_store.py
@validator("url")
def validate_url(cls, url: str) -> str:
    """Validates that the URL is a well-formed REST store URL.

    Args:
        url: The URL to be validated.

    Returns:
        The validated URL without trailing slashes.

    Raises:
        ValueError: If the URL is not a well-formed REST store URL.
    """
    url = url.rstrip("/")
    scheme = re.search("^([a-z0-9]+://)", url)
    if scheme is None or scheme.group() not in ("https://", "http://"):
        raise ValueError(
            "Invalid URL for REST store: {url}. Should be in the form "
            "https://hostname[:port] or http://hostname[:port]."
        )

    # When running inside a container, if the URL uses localhost, the
    # target service will not be available. We try to replace localhost
    # with one of the special Docker or K3D internal hostnames.
    url = replace_localhost_with_internal_hostname(url)

    return url
validate_verify_ssl(verify_ssl) classmethod

Validates that the verify_ssl either points to a file or is a bool.

Parameters:

Name Type Description Default
verify_ssl Union[bool, str]

The verify_ssl value to be validated.

required

Returns:

Type Description
Union[bool, str]

The validated verify_ssl value.

Source code in zenml/zen_stores/rest_zen_store.py
@validator("verify_ssl")
def validate_verify_ssl(
    cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
    """Validates that the verify_ssl either points to a file or is a bool.

    Args:
        verify_ssl: The verify_ssl value to be validated.

    Returns:
        The validated verify_ssl value.
    """
    secret_folder = Path(
        GlobalConfiguration().local_stores_path,
        "certificates",
    )
    if isinstance(verify_ssl, bool) or verify_ssl.startswith(
        str(secret_folder)
    ):
        return verify_ssl

    if os.path.isfile(verify_ssl):
        with open(verify_ssl, "r") as f:
            verify_ssl = f.read()

    fileio.makedirs(str(secret_folder))
    file_path = Path(secret_folder, "ca_bundle.pem")
    with open(file_path, "w") as f:
        f.write(verify_ssl)
    file_path.chmod(0o600)
    verify_ssl = str(file_path)

    return verify_ssl

schemas special

SQL Model Implementations.

api_key_schemas

SQLModel implementation of user tables.

APIKeySchema (NamedSchema) pydantic-model

SQL Model for API keys.

Source code in zenml/zen_stores/schemas/api_key_schemas.py
class APIKeySchema(NamedSchema, table=True):
    """SQL Model for API keys."""

    __tablename__ = "api_key"

    description: str = Field(sa_column=Column(TEXT))
    key: str
    previous_key: Optional[str] = Field(default=None, nullable=True)
    retain_period: int = Field(default=0)
    active: bool = Field(default=True)
    last_login: Optional[datetime] = None
    last_rotated: Optional[datetime] = None

    service_account_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="service_account_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    service_account: "UserSchema" = Relationship(back_populates="api_keys")

    @classmethod
    def _generate_jwt_secret_key(cls) -> str:
        """Generate a random API key.

        Returns:
            A random API key.
        """
        return token_hex(32)

    @classmethod
    def _get_hashed_key(cls, key: str) -> str:
        """Hashes the input key and returns the hash value.

        Args:
            key: The key value to hash.

        Returns:
            The key hash value.
        """
        context = CryptContext(schemes=["bcrypt"], deprecated="auto")
        return context.hash(key)

    @classmethod
    def from_request(
        cls,
        service_account_id: UUID,
        request: APIKeyRequest,
    ) -> Tuple["APIKeySchema", str]:
        """Convert a `APIKeyRequest` to a `APIKeySchema`.

        Args:
            service_account_id: The service account id to associate the key
                with.
            request: The request model to convert.

        Returns:
            The converted schema and the un-hashed API key.
        """
        key = cls._generate_jwt_secret_key()
        hashed_key = cls._get_hashed_key(key)
        now = datetime.utcnow()
        return (
            cls(
                name=request.name,
                description=request.description or "",
                key=hashed_key,
                service_account_id=service_account_id,
                created=now,
                updated=now,
            ),
            key,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> APIKeyResponse:
        """Convert a `APIKeySchema` to an `APIKeyResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic

            **kwargs: Keyword arguments to filter models.

        Returns:
            The created APIKeyResponse.
        """
        metadata = None
        if include_metadata:
            metadata = APIKeyResponseMetadata(
                description=self.description,
                retain_period_minutes=self.retain_period,
                last_login=self.last_login,
                last_rotated=self.last_rotated,
            )

        body = APIKeyResponseBody(
            created=self.created,
            updated=self.updated,
            active=self.active,
            service_account=self.service_account.to_service_account_model(),
        )

        return APIKeyResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )

    def to_internal_model(
        self, hydrate: bool = False
    ) -> APIKeyInternalResponse:
        """Convert a `APIKeySchema` to an `APIKeyInternalResponse`.

        The internal response model includes the hashed key values.

        Args:
            hydrate: bool to decide whether to return a hydrated version of the
                model.

        Returns:
            The created APIKeyInternalResponse.
        """
        model = self.to_model(include_metadata=hydrate)
        model.get_body().key = self.key

        return APIKeyInternalResponse(
            id=self.id,
            name=self.name,
            previous_key=self.previous_key,
            body=model.body,
            metadata=model.metadata,
        )

    def update(self, update: APIKeyUpdate) -> "APIKeySchema":
        """Update an `APIKeySchema` with an `APIKeyUpdate`.

        Args:
            update: The update model.

        Returns:
            The updated `APIKeySchema`.
        """
        for field, value in update.dict(exclude_none=True).items():
            if hasattr(self, field):
                setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    def internal_update(self, update: APIKeyInternalUpdate) -> "APIKeySchema":
        """Update an `APIKeySchema` with an `APIKeyInternalUpdate`.

        The internal update can also update the last used timestamp.

        Args:
            update: The update model.

        Returns:
            The updated `APIKeySchema`.
        """
        self.update(update)

        if update.update_last_login:
            self.last_login = self.updated

        return self

    def rotate(
        self,
        rotate_request: APIKeyRotateRequest,
    ) -> Tuple["APIKeySchema", str]:
        """Rotate the key for an `APIKeySchema`.

        Args:
            rotate_request: The rotate request model.

        Returns:
            The updated `APIKeySchema` and the new un-hashed key.
        """
        self.updated = datetime.utcnow()
        self.previous_key = self.key
        self.retain_period = rotate_request.retain_period_minutes
        new_key = self._generate_jwt_secret_key()
        self.key = self._get_hashed_key(new_key)
        self.last_rotated = self.updated

        return self, new_key
from_request(service_account_id, request) classmethod

Convert a APIKeyRequest to a APIKeySchema.

Parameters:

Name Type Description Default
service_account_id UUID

The service account id to associate the key with.

required
request APIKeyRequest

The request model to convert.

required

Returns:

Type Description
Tuple[APIKeySchema, str]

The converted schema and the un-hashed API key.

Source code in zenml/zen_stores/schemas/api_key_schemas.py
@classmethod
def from_request(
    cls,
    service_account_id: UUID,
    request: APIKeyRequest,
) -> Tuple["APIKeySchema", str]:
    """Convert a `APIKeyRequest` to a `APIKeySchema`.

    Args:
        service_account_id: The service account id to associate the key
            with.
        request: The request model to convert.

    Returns:
        The converted schema and the un-hashed API key.
    """
    key = cls._generate_jwt_secret_key()
    hashed_key = cls._get_hashed_key(key)
    now = datetime.utcnow()
    return (
        cls(
            name=request.name,
            description=request.description or "",
            key=hashed_key,
            service_account_id=service_account_id,
            created=now,
            updated=now,
        ),
        key,
    )
internal_update(self, update)

Update an APIKeySchema with an APIKeyInternalUpdate.

The internal update can also update the last used timestamp.

Parameters:

Name Type Description Default
update APIKeyInternalUpdate

The update model.

required

Returns:

Type Description
APIKeySchema

The updated APIKeySchema.

Source code in zenml/zen_stores/schemas/api_key_schemas.py
def internal_update(self, update: APIKeyInternalUpdate) -> "APIKeySchema":
    """Update an `APIKeySchema` with an `APIKeyInternalUpdate`.

    The internal update can also update the last used timestamp.

    Args:
        update: The update model.

    Returns:
        The updated `APIKeySchema`.
    """
    self.update(update)

    if update.update_last_login:
        self.last_login = self.updated

    return self
rotate(self, rotate_request)

Rotate the key for an APIKeySchema.

Parameters:

Name Type Description Default
rotate_request APIKeyRotateRequest

The rotate request model.

required

Returns:

Type Description
Tuple[APIKeySchema, str]

The updated APIKeySchema and the new un-hashed key.

Source code in zenml/zen_stores/schemas/api_key_schemas.py
def rotate(
    self,
    rotate_request: APIKeyRotateRequest,
) -> Tuple["APIKeySchema", str]:
    """Rotate the key for an `APIKeySchema`.

    Args:
        rotate_request: The rotate request model.

    Returns:
        The updated `APIKeySchema` and the new un-hashed key.
    """
    self.updated = datetime.utcnow()
    self.previous_key = self.key
    self.retain_period = rotate_request.retain_period_minutes
    new_key = self._generate_jwt_secret_key()
    self.key = self._get_hashed_key(new_key)
    self.last_rotated = self.updated

    return self, new_key
to_internal_model(self, hydrate=False)

Convert a APIKeySchema to an APIKeyInternalResponse.

The internal response model includes the hashed key values.

Parameters:

Name Type Description Default
hydrate bool

bool to decide whether to return a hydrated version of the model.

False

Returns:

Type Description
APIKeyInternalResponse

The created APIKeyInternalResponse.

Source code in zenml/zen_stores/schemas/api_key_schemas.py
def to_internal_model(
    self, hydrate: bool = False
) -> APIKeyInternalResponse:
    """Convert a `APIKeySchema` to an `APIKeyInternalResponse`.

    The internal response model includes the hashed key values.

    Args:
        hydrate: bool to decide whether to return a hydrated version of the
            model.

    Returns:
        The created APIKeyInternalResponse.
    """
    model = self.to_model(include_metadata=hydrate)
    model.get_body().key = self.key

    return APIKeyInternalResponse(
        id=self.id,
        name=self.name,
        previous_key=self.previous_key,
        body=model.body,
        metadata=model.metadata,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a APIKeySchema to an APIKeyResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}
**kwargs Any

Keyword arguments to filter models.

{}

Returns:

Type Description
APIKeyResponse

The created APIKeyResponse.

Source code in zenml/zen_stores/schemas/api_key_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> APIKeyResponse:
    """Convert a `APIKeySchema` to an `APIKeyResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic

        **kwargs: Keyword arguments to filter models.

    Returns:
        The created APIKeyResponse.
    """
    metadata = None
    if include_metadata:
        metadata = APIKeyResponseMetadata(
            description=self.description,
            retain_period_minutes=self.retain_period,
            last_login=self.last_login,
            last_rotated=self.last_rotated,
        )

    body = APIKeyResponseBody(
        created=self.created,
        updated=self.updated,
        active=self.active,
        service_account=self.service_account.to_service_account_model(),
    )

    return APIKeyResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, update)

Update an APIKeySchema with an APIKeyUpdate.

Parameters:

Name Type Description Default
update APIKeyUpdate

The update model.

required

Returns:

Type Description
APIKeySchema

The updated APIKeySchema.

Source code in zenml/zen_stores/schemas/api_key_schemas.py
def update(self, update: APIKeyUpdate) -> "APIKeySchema":
    """Update an `APIKeySchema` with an `APIKeyUpdate`.

    Args:
        update: The update model.

    Returns:
        The updated `APIKeySchema`.
    """
    for field, value in update.dict(exclude_none=True).items():
        if hasattr(self, field):
            setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self

artifact_schemas

SQLModel implementation of artifact table.

ArtifactSchema (NamedSchema) pydantic-model

SQL Model for artifacts.

Source code in zenml/zen_stores/schemas/artifact_schemas.py
class ArtifactSchema(NamedSchema, table=True):
    """SQL Model for artifacts."""

    __tablename__ = "artifact"

    # Fields
    has_custom_name: bool
    versions: List["ArtifactVersionSchema"] = Relationship(
        back_populates="artifact",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    tags: List["TagResourceSchema"] = Relationship(
        back_populates="artifact",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(TagResourceSchema.resource_type=='{TaggableResourceTypes.ARTIFACT.value}', foreign(TagResourceSchema.resource_id)==ArtifactSchema.id)",
            cascade="delete",
            overlaps="tags",
        ),
    )

    @classmethod
    def from_request(
        cls,
        artifact_request: ArtifactRequest,
    ) -> "ArtifactSchema":
        """Convert an `ArtifactRequest` to an `ArtifactSchema`.

        Args:
            artifact_request: The request model to convert.

        Returns:
            The converted schema.
        """
        return cls(
            name=artifact_request.name,
            has_custom_name=artifact_request.has_custom_name,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> ArtifactResponse:
        """Convert an `ArtifactSchema` to an `ArtifactResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic



        Returns:
            The created `ArtifactResponse`.
        """
        latest_id, latest_name = None, None
        if self.versions:
            latest_version = max(self.versions, key=lambda x: x.created)
            latest_id, latest_name = latest_version.id, latest_version.version

        # Create the body of the model
        body = ArtifactResponseBody(
            created=self.created,
            updated=self.updated,
            tags=[t.tag.to_model() for t in self.tags],
            latest_version_name=latest_name,
            latest_version_id=latest_id,
        )

        # Create the metadata of the model
        metadata = None
        if include_metadata:
            metadata = ArtifactResponseMetadata(
                has_custom_name=self.has_custom_name,
            )

        return ArtifactResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )

    def update(self, artifact_update: ArtifactUpdate) -> "ArtifactSchema":
        """Update an `ArtifactSchema` with an `ArtifactUpdate`.

        Args:
            artifact_update: The update model to apply.

        Returns:
            The updated `ArtifactSchema`.
        """
        self.updated = datetime.utcnow()
        if artifact_update.name:
            self.name = artifact_update.name
            self.has_custom_name = True
        if artifact_update.has_custom_name is not None:
            self.has_custom_name = artifact_update.has_custom_name
        return self
from_request(artifact_request) classmethod

Convert an ArtifactRequest to an ArtifactSchema.

Parameters:

Name Type Description Default
artifact_request ArtifactRequest

The request model to convert.

required

Returns:

Type Description
ArtifactSchema

The converted schema.

Source code in zenml/zen_stores/schemas/artifact_schemas.py
@classmethod
def from_request(
    cls,
    artifact_request: ArtifactRequest,
) -> "ArtifactSchema":
    """Convert an `ArtifactRequest` to an `ArtifactSchema`.

    Args:
        artifact_request: The request model to convert.

    Returns:
        The converted schema.
    """
    return cls(
        name=artifact_request.name,
        has_custom_name=artifact_request.has_custom_name,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an ArtifactSchema to an ArtifactResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ArtifactResponse

The created ArtifactResponse.

Source code in zenml/zen_stores/schemas/artifact_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> ArtifactResponse:
    """Convert an `ArtifactSchema` to an `ArtifactResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic



    Returns:
        The created `ArtifactResponse`.
    """
    latest_id, latest_name = None, None
    if self.versions:
        latest_version = max(self.versions, key=lambda x: x.created)
        latest_id, latest_name = latest_version.id, latest_version.version

    # Create the body of the model
    body = ArtifactResponseBody(
        created=self.created,
        updated=self.updated,
        tags=[t.tag.to_model() for t in self.tags],
        latest_version_name=latest_name,
        latest_version_id=latest_id,
    )

    # Create the metadata of the model
    metadata = None
    if include_metadata:
        metadata = ArtifactResponseMetadata(
            has_custom_name=self.has_custom_name,
        )

    return ArtifactResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, artifact_update)

Update an ArtifactSchema with an ArtifactUpdate.

Parameters:

Name Type Description Default
artifact_update ArtifactUpdate

The update model to apply.

required

Returns:

Type Description
ArtifactSchema

The updated ArtifactSchema.

Source code in zenml/zen_stores/schemas/artifact_schemas.py
def update(self, artifact_update: ArtifactUpdate) -> "ArtifactSchema":
    """Update an `ArtifactSchema` with an `ArtifactUpdate`.

    Args:
        artifact_update: The update model to apply.

    Returns:
        The updated `ArtifactSchema`.
    """
    self.updated = datetime.utcnow()
    if artifact_update.name:
        self.name = artifact_update.name
        self.has_custom_name = True
    if artifact_update.has_custom_name is not None:
        self.has_custom_name = artifact_update.has_custom_name
    return self
ArtifactVersionSchema (BaseSchema) pydantic-model

SQL Model for artifact versions.

Source code in zenml/zen_stores/schemas/artifact_schemas.py
class ArtifactVersionSchema(BaseSchema, table=True):
    """SQL Model for artifact versions."""

    __tablename__ = "artifact_version"

    # Fields
    version: str
    version_number: Optional[int]
    type: ArtifactType
    uri: str = Field(sa_column=Column(TEXT, nullable=False))
    materializer: str = Field(sa_column=Column(TEXT, nullable=False))
    data_type: str = Field(sa_column=Column(TEXT, nullable=False))
    tags: List["TagResourceSchema"] = Relationship(
        back_populates="artifact_version",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(TagResourceSchema.resource_type=='{TaggableResourceTypes.ARTIFACT_VERSION.value}', foreign(TagResourceSchema.resource_id)==ArtifactVersionSchema.id)",
            cascade="delete",
            overlaps="tags",
        ),
    )

    # Foreign keys
    artifact_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=ArtifactSchema.__tablename__,
        source_column="artifact_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    artifact_store_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=StackComponentSchema.__tablename__,
        source_column="artifact_store_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )

    # Relationships
    artifact: "ArtifactSchema" = Relationship(back_populates="versions")
    user: Optional["UserSchema"] = Relationship(
        back_populates="artifact_versions"
    )
    workspace: "WorkspaceSchema" = Relationship(
        back_populates="artifact_versions"
    )
    run_metadata: List["RunMetadataSchema"] = Relationship(
        back_populates="artifact_version",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.ARTIFACT_VERSION.value}', foreign(RunMetadataSchema.resource_id)==ArtifactVersionSchema.id)",
            cascade="delete",
            overlaps="run_metadata",
        ),
    )
    output_of_step_runs: List["StepRunOutputArtifactSchema"] = Relationship(
        back_populates="artifact_version",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    input_of_step_runs: List["StepRunInputArtifactSchema"] = Relationship(
        back_populates="artifact_version",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    visualizations: List["ArtifactVisualizationSchema"] = Relationship(
        back_populates="artifact_version",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    model_versions_artifacts_links: List["ModelVersionArtifactSchema"] = (
        Relationship(
            back_populates="artifact_version",
            sa_relationship_kwargs={"cascade": "delete"},
        )
    )

    @classmethod
    def from_request(
        cls,
        artifact_version_request: ArtifactVersionRequest,
    ) -> "ArtifactVersionSchema":
        """Convert an `ArtifactVersionRequest` to an `ArtifactVersionSchema`.

        Args:
            artifact_version_request: The request model to convert.

        Returns:
            The converted schema.
        """
        try:
            version_number = int(artifact_version_request.version)
        except ValueError:
            version_number = None
        return cls(
            artifact_id=artifact_version_request.artifact_id,
            version=str(artifact_version_request.version),
            version_number=version_number,
            artifact_store_id=artifact_version_request.artifact_store_id,
            workspace_id=artifact_version_request.workspace,
            user_id=artifact_version_request.user,
            type=artifact_version_request.type,
            uri=artifact_version_request.uri,
            materializer=artifact_version_request.materializer.json(),
            data_type=artifact_version_request.data_type.json(),
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> ArtifactVersionResponse:
        """Convert an `ArtifactVersionSchema` to an `ArtifactVersionResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic



        Returns:
            The created `ArtifactVersionResponse`.
        """
        try:
            materializer = Source.parse_raw(self.materializer)
        except ValidationError:
            # This is an old source which was an importable source path
            materializer = Source.from_import_path(self.materializer)

        try:
            data_type = Source.parse_raw(self.data_type)
        except ValidationError:
            # This is an old source which was an importable source path
            data_type = Source.from_import_path(self.data_type)

        producer_step_run_id, producer_pipeline_run_id = None, None
        if self.output_of_step_runs:
            step_run = self.output_of_step_runs[0].step_run
            if step_run.status == ExecutionStatus.COMPLETED:
                producer_step_run_id = step_run.id
                producer_pipeline_run_id = step_run.pipeline_run_id
            else:
                producer_step_run_id = step_run.original_step_run_id

        # Create the body of the model
        body = ArtifactVersionResponseBody(
            artifact=self.artifact.to_model(),
            version=self.version_number or self.version,
            user=self.user.to_model() if self.user else None,
            uri=self.uri,
            type=self.type,
            materializer=materializer,
            data_type=data_type,
            created=self.created,
            updated=self.updated,
            tags=[t.tag.to_model() for t in self.tags],
            producer_pipeline_run_id=producer_pipeline_run_id,
        )

        # Create the metadata of the model
        metadata = None
        if include_metadata:
            metadata = ArtifactVersionResponseMetadata(
                workspace=self.workspace.to_model(),
                artifact_store_id=self.artifact_store_id,
                producer_step_run_id=producer_step_run_id,
                visualizations=[v.to_model() for v in self.visualizations],
                run_metadata={m.key: m.to_model() for m in self.run_metadata},
            )

        return ArtifactVersionResponse(
            id=self.id,
            body=body,
            metadata=metadata,
        )

    def update(
        self, artifact_version_update: ArtifactVersionUpdate
    ) -> "ArtifactVersionSchema":
        """Update an `ArtifactVersionSchema` with an `ArtifactVersionUpdate`.

        Args:
            artifact_version_update: The update model to apply.

        Returns:
            The updated `ArtifactVersionSchema`.
        """
        self.updated = datetime.utcnow()
        return self
from_request(artifact_version_request) classmethod

Convert an ArtifactVersionRequest to an ArtifactVersionSchema.

Parameters:

Name Type Description Default
artifact_version_request ArtifactVersionRequest

The request model to convert.

required

Returns:

Type Description
ArtifactVersionSchema

The converted schema.

Source code in zenml/zen_stores/schemas/artifact_schemas.py
@classmethod
def from_request(
    cls,
    artifact_version_request: ArtifactVersionRequest,
) -> "ArtifactVersionSchema":
    """Convert an `ArtifactVersionRequest` to an `ArtifactVersionSchema`.

    Args:
        artifact_version_request: The request model to convert.

    Returns:
        The converted schema.
    """
    try:
        version_number = int(artifact_version_request.version)
    except ValueError:
        version_number = None
    return cls(
        artifact_id=artifact_version_request.artifact_id,
        version=str(artifact_version_request.version),
        version_number=version_number,
        artifact_store_id=artifact_version_request.artifact_store_id,
        workspace_id=artifact_version_request.workspace,
        user_id=artifact_version_request.user,
        type=artifact_version_request.type,
        uri=artifact_version_request.uri,
        materializer=artifact_version_request.materializer.json(),
        data_type=artifact_version_request.data_type.json(),
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an ArtifactVersionSchema to an ArtifactVersionResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ArtifactVersionResponse

The created ArtifactVersionResponse.

Source code in zenml/zen_stores/schemas/artifact_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> ArtifactVersionResponse:
    """Convert an `ArtifactVersionSchema` to an `ArtifactVersionResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic



    Returns:
        The created `ArtifactVersionResponse`.
    """
    try:
        materializer = Source.parse_raw(self.materializer)
    except ValidationError:
        # This is an old source which was an importable source path
        materializer = Source.from_import_path(self.materializer)

    try:
        data_type = Source.parse_raw(self.data_type)
    except ValidationError:
        # This is an old source which was an importable source path
        data_type = Source.from_import_path(self.data_type)

    producer_step_run_id, producer_pipeline_run_id = None, None
    if self.output_of_step_runs:
        step_run = self.output_of_step_runs[0].step_run
        if step_run.status == ExecutionStatus.COMPLETED:
            producer_step_run_id = step_run.id
            producer_pipeline_run_id = step_run.pipeline_run_id
        else:
            producer_step_run_id = step_run.original_step_run_id

    # Create the body of the model
    body = ArtifactVersionResponseBody(
        artifact=self.artifact.to_model(),
        version=self.version_number or self.version,
        user=self.user.to_model() if self.user else None,
        uri=self.uri,
        type=self.type,
        materializer=materializer,
        data_type=data_type,
        created=self.created,
        updated=self.updated,
        tags=[t.tag.to_model() for t in self.tags],
        producer_pipeline_run_id=producer_pipeline_run_id,
    )

    # Create the metadata of the model
    metadata = None
    if include_metadata:
        metadata = ArtifactVersionResponseMetadata(
            workspace=self.workspace.to_model(),
            artifact_store_id=self.artifact_store_id,
            producer_step_run_id=producer_step_run_id,
            visualizations=[v.to_model() for v in self.visualizations],
            run_metadata={m.key: m.to_model() for m in self.run_metadata},
        )

    return ArtifactVersionResponse(
        id=self.id,
        body=body,
        metadata=metadata,
    )
update(self, artifact_version_update)

Update an ArtifactVersionSchema with an ArtifactVersionUpdate.

Parameters:

Name Type Description Default
artifact_version_update ArtifactVersionUpdate

The update model to apply.

required

Returns:

Type Description
ArtifactVersionSchema

The updated ArtifactVersionSchema.

Source code in zenml/zen_stores/schemas/artifact_schemas.py
def update(
    self, artifact_version_update: ArtifactVersionUpdate
) -> "ArtifactVersionSchema":
    """Update an `ArtifactVersionSchema` with an `ArtifactVersionUpdate`.

    Args:
        artifact_version_update: The update model to apply.

    Returns:
        The updated `ArtifactVersionSchema`.
    """
    self.updated = datetime.utcnow()
    return self

artifact_visualization_schemas

SQLModel implementation of artifact visualization table.

ArtifactVisualizationSchema (BaseSchema) pydantic-model

SQL Model for visualizations of artifacts.

Source code in zenml/zen_stores/schemas/artifact_visualization_schemas.py
class ArtifactVisualizationSchema(BaseSchema, table=True):
    """SQL Model for visualizations of artifacts."""

    __tablename__ = "artifact_visualization"

    # Fields
    type: VisualizationType
    uri: str = Field(sa_column=Column(TEXT, nullable=False))

    # Foreign Keys
    artifact_version_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=ArtifactVersionSchema.__tablename__,
        source_column="artifact_version_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )

    # Relationships
    artifact_version: ArtifactVersionSchema = Relationship(
        back_populates="visualizations"
    )

    @classmethod
    def from_model(
        cls,
        artifact_visualization_request: ArtifactVisualizationRequest,
        artifact_version_id: UUID,
    ) -> "ArtifactVisualizationSchema":
        """Convert a `ArtifactVisualizationRequest` to a `ArtifactVisualizationSchema`.

        Args:
            artifact_visualization_request: The visualization.
            artifact_version_id: The UUID of the artifact version.

        Returns:
            The `ArtifactVisualizationSchema`.
        """
        return cls(
            type=artifact_visualization_request.type,
            uri=artifact_visualization_request.uri,
            artifact_version_id=artifact_version_id,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> ArtifactVisualizationResponse:
        """Convert an `ArtifactVisualizationSchema` to a `Visualization`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic



        Returns:
            The `Visualization`.
        """
        body = ArtifactVisualizationResponseBody(
            type=self.type,
            uri=self.uri,
            created=self.created,
            updated=self.updated,
        )

        metadata = None
        if include_metadata:
            metadata = ArtifactVisualizationResponseMetadata(
                artifact_version_id=self.artifact_version_id,
            )

        return ArtifactVisualizationResponse(
            id=self.id,
            body=body,
            metadata=metadata,
        )
from_model(artifact_visualization_request, artifact_version_id) classmethod

Convert a ArtifactVisualizationRequest to a ArtifactVisualizationSchema.

Parameters:

Name Type Description Default
artifact_visualization_request ArtifactVisualizationRequest

The visualization.

required
artifact_version_id UUID

The UUID of the artifact version.

required

Returns:

Type Description
ArtifactVisualizationSchema

The ArtifactVisualizationSchema.

Source code in zenml/zen_stores/schemas/artifact_visualization_schemas.py
@classmethod
def from_model(
    cls,
    artifact_visualization_request: ArtifactVisualizationRequest,
    artifact_version_id: UUID,
) -> "ArtifactVisualizationSchema":
    """Convert a `ArtifactVisualizationRequest` to a `ArtifactVisualizationSchema`.

    Args:
        artifact_visualization_request: The visualization.
        artifact_version_id: The UUID of the artifact version.

    Returns:
        The `ArtifactVisualizationSchema`.
    """
    return cls(
        type=artifact_visualization_request.type,
        uri=artifact_visualization_request.uri,
        artifact_version_id=artifact_version_id,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an ArtifactVisualizationSchema to a Visualization.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ArtifactVisualizationResponse

The Visualization.

Source code in zenml/zen_stores/schemas/artifact_visualization_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> ArtifactVisualizationResponse:
    """Convert an `ArtifactVisualizationSchema` to a `Visualization`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic



    Returns:
        The `Visualization`.
    """
    body = ArtifactVisualizationResponseBody(
        type=self.type,
        uri=self.uri,
        created=self.created,
        updated=self.updated,
    )

    metadata = None
    if include_metadata:
        metadata = ArtifactVisualizationResponseMetadata(
            artifact_version_id=self.artifact_version_id,
        )

    return ArtifactVisualizationResponse(
        id=self.id,
        body=body,
        metadata=metadata,
    )

base_schemas

Base classes for SQLModel schemas.

BaseSchema (SQLModel) pydantic-model

Base SQL Model for ZenML entities.

Source code in zenml/zen_stores/schemas/base_schemas.py
class BaseSchema(SQLModel):
    """Base SQL Model for ZenML entities."""

    id: UUID = Field(default_factory=uuid4, primary_key=True)
    created: datetime = Field(default_factory=datetime.utcnow)
    updated: datetime = Field(default_factory=datetime.utcnow)

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> Any:
        """In case the Schema has a corresponding Model, this allows conversion to that model.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic

        Raises:
            NotImplementedError: When the base class fails to implement this.
        """
        raise NotImplementedError(
            "No 'to_model()' method implemented for this"
            f"schema: '{self.__class__.__name__}'."
        )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

In case the Schema has a corresponding Model, this allows conversion to that model.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Exceptions:

Type Description
NotImplementedError

When the base class fails to implement this.

Source code in zenml/zen_stores/schemas/base_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> Any:
    """In case the Schema has a corresponding Model, this allows conversion to that model.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic

    Raises:
        NotImplementedError: When the base class fails to implement this.
    """
    raise NotImplementedError(
        "No 'to_model()' method implemented for this"
        f"schema: '{self.__class__.__name__}'."
    )
NamedSchema (BaseSchema) pydantic-model

Base Named SQL Model.

Source code in zenml/zen_stores/schemas/base_schemas.py
class NamedSchema(BaseSchema):
    """Base Named SQL Model."""

    name: str

code_repository_schemas

SQL Model Implementations for code repositories.

CodeReferenceSchema (BaseSchema) pydantic-model

SQL Model for code references.

Source code in zenml/zen_stores/schemas/code_repository_schemas.py
class CodeReferenceSchema(BaseSchema, table=True):
    """SQL Model for code references."""

    __tablename__ = "code_reference"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship()

    code_repository_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=CodeRepositorySchema.__tablename__,
        source_column="code_repository_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    code_repository: "CodeRepositorySchema" = Relationship()

    commit: str
    subdirectory: str

    @classmethod
    def from_request(
        cls, request: "CodeReferenceRequest", workspace_id: UUID
    ) -> "CodeReferenceSchema":
        """Convert a `CodeReferenceRequest` to a `CodeReferenceSchema`.

        Args:
            request: The request model to convert.
            workspace_id: The workspace ID.

        Returns:
            The converted schema.
        """
        return cls(
            workspace_id=workspace_id,
            commit=request.commit,
            subdirectory=request.subdirectory,
            code_repository_id=request.code_repository,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "CodeReferenceResponse":
        """Convert a `CodeReferenceSchema` to a `CodeReferenceResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic

            kwargs: Additional keyword arguments.

        Returns:
            The converted model.
        """
        body = CodeReferenceResponseBody(
            commit=self.commit,
            subdirectory=self.subdirectory,
            code_repository=self.code_repository.to_model(),
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = CodeReferenceResponseMetadata()

        return CodeReferenceResponse(
            id=self.id,
            body=body,
            metadata=metadata,
        )
from_request(request, workspace_id) classmethod

Convert a CodeReferenceRequest to a CodeReferenceSchema.

Parameters:

Name Type Description Default
request CodeReferenceRequest

The request model to convert.

required
workspace_id UUID

The workspace ID.

required

Returns:

Type Description
CodeReferenceSchema

The converted schema.

Source code in zenml/zen_stores/schemas/code_repository_schemas.py
@classmethod
def from_request(
    cls, request: "CodeReferenceRequest", workspace_id: UUID
) -> "CodeReferenceSchema":
    """Convert a `CodeReferenceRequest` to a `CodeReferenceSchema`.

    Args:
        request: The request model to convert.
        workspace_id: The workspace ID.

    Returns:
        The converted schema.
    """
    return cls(
        workspace_id=workspace_id,
        commit=request.commit,
        subdirectory=request.subdirectory,
        code_repository_id=request.code_repository,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a CodeReferenceSchema to a CodeReferenceResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}
kwargs Any

Additional keyword arguments.

{}

Returns:

Type Description
CodeReferenceResponse

The converted model.

Source code in zenml/zen_stores/schemas/code_repository_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "CodeReferenceResponse":
    """Convert a `CodeReferenceSchema` to a `CodeReferenceResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic

        kwargs: Additional keyword arguments.

    Returns:
        The converted model.
    """
    body = CodeReferenceResponseBody(
        commit=self.commit,
        subdirectory=self.subdirectory,
        code_repository=self.code_repository.to_model(),
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = CodeReferenceResponseMetadata()

    return CodeReferenceResponse(
        id=self.id,
        body=body,
        metadata=metadata,
    )
CodeRepositorySchema (NamedSchema) pydantic-model

SQL Model for code repositories.

Source code in zenml/zen_stores/schemas/code_repository_schemas.py
class CodeRepositorySchema(NamedSchema, table=True):
    """SQL Model for code repositories."""

    __tablename__ = "code_repository"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(
        back_populates="code_repositories"
    )

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )

    user: Optional["UserSchema"] = Relationship(
        back_populates="code_repositories"
    )

    config: str = Field(sa_column=Column(TEXT, nullable=False))
    source: str = Field(sa_column=Column(TEXT, nullable=False))
    logo_url: Optional[str] = Field()
    description: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))

    @classmethod
    def from_request(
        cls, request: "CodeRepositoryRequest"
    ) -> "CodeRepositorySchema":
        """Convert a `CodeRepositoryRequest` to a `CodeRepositorySchema`.

        Args:
            request: The request model to convert.

        Returns:
            The converted schema.
        """
        return cls(
            name=request.name,
            workspace_id=request.workspace,
            user_id=request.user,
            config=json.dumps(request.config),
            source=request.source.json(),
            description=request.description,
            logo_url=request.logo_url,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "CodeRepositoryResponse":
        """Convert a `CodeRepositorySchema` to a `CodeRepositoryResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created CodeRepositoryResponse.
        """
        body = CodeRepositoryResponseBody(
            user=self.user.to_model() if self.user else None,
            source=json.loads(self.source),
            logo_url=self.logo_url,
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = CodeRepositoryResponseMetadata(
                workspace=self.workspace.to_model(),
                config=json.loads(self.config),
                description=self.description,
            )
        return CodeRepositoryResponse(
            id=self.id,
            name=self.name,
            metadata=metadata,
            body=body,
        )

    def update(self, update: "CodeRepositoryUpdate") -> "CodeRepositorySchema":
        """Update a `CodeRepositorySchema` with a `CodeRepositoryUpdate`.

        Args:
            update: The update model.

        Returns:
            The updated `CodeRepositorySchema`.
        """
        if update.name:
            self.name = update.name

        if update.description:
            self.description = update.description

        if update.logo_url:
            self.logo_url = update.logo_url

        self.updated = datetime.utcnow()
        return self
from_request(request) classmethod

Convert a CodeRepositoryRequest to a CodeRepositorySchema.

Parameters:

Name Type Description Default
request CodeRepositoryRequest

The request model to convert.

required

Returns:

Type Description
CodeRepositorySchema

The converted schema.

Source code in zenml/zen_stores/schemas/code_repository_schemas.py
@classmethod
def from_request(
    cls, request: "CodeRepositoryRequest"
) -> "CodeRepositorySchema":
    """Convert a `CodeRepositoryRequest` to a `CodeRepositorySchema`.

    Args:
        request: The request model to convert.

    Returns:
        The converted schema.
    """
    return cls(
        name=request.name,
        workspace_id=request.workspace,
        user_id=request.user,
        config=json.dumps(request.config),
        source=request.source.json(),
        description=request.description,
        logo_url=request.logo_url,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a CodeRepositorySchema to a CodeRepositoryResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
CodeRepositoryResponse

The created CodeRepositoryResponse.

Source code in zenml/zen_stores/schemas/code_repository_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "CodeRepositoryResponse":
    """Convert a `CodeRepositorySchema` to a `CodeRepositoryResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created CodeRepositoryResponse.
    """
    body = CodeRepositoryResponseBody(
        user=self.user.to_model() if self.user else None,
        source=json.loads(self.source),
        logo_url=self.logo_url,
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = CodeRepositoryResponseMetadata(
            workspace=self.workspace.to_model(),
            config=json.loads(self.config),
            description=self.description,
        )
    return CodeRepositoryResponse(
        id=self.id,
        name=self.name,
        metadata=metadata,
        body=body,
    )
update(self, update)

Update a CodeRepositorySchema with a CodeRepositoryUpdate.

Parameters:

Name Type Description Default
update CodeRepositoryUpdate

The update model.

required

Returns:

Type Description
CodeRepositorySchema

The updated CodeRepositorySchema.

Source code in zenml/zen_stores/schemas/code_repository_schemas.py
def update(self, update: "CodeRepositoryUpdate") -> "CodeRepositorySchema":
    """Update a `CodeRepositorySchema` with a `CodeRepositoryUpdate`.

    Args:
        update: The update model.

    Returns:
        The updated `CodeRepositorySchema`.
    """
    if update.name:
        self.name = update.name

    if update.description:
        self.description = update.description

    if update.logo_url:
        self.logo_url = update.logo_url

    self.updated = datetime.utcnow()
    return self

component_schemas

SQL Model Implementations for Stack Components.

StackComponentSchema (NamedSchema) pydantic-model

SQL Model for stack components.

Source code in zenml/zen_stores/schemas/component_schemas.py
class StackComponentSchema(NamedSchema, table=True):
    """SQL Model for stack components."""

    __tablename__ = "stack_component"

    type: StackComponentType
    flavor: str
    configuration: bytes
    labels: Optional[bytes]
    component_spec_path: Optional[str]

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="components")

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="components")

    stacks: List["StackSchema"] = Relationship(
        back_populates="components", link_model=StackCompositionSchema
    )
    schedules: List["ScheduleSchema"] = Relationship(
        back_populates="orchestrator",
    )

    run_metadata: List["RunMetadataSchema"] = Relationship(
        back_populates="stack_component",
    )

    run_or_step_logs: List["LogsSchema"] = Relationship(
        back_populates="artifact_store",
        sa_relationship_kwargs={"cascade": "delete", "uselist": True},
    )

    connector_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=ServiceConnectorSchema.__tablename__,
        source_column="connector_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    connector: Optional["ServiceConnectorSchema"] = Relationship(
        back_populates="components"
    )

    connector_resource_id: Optional[str]

    def update(
        self, component_update: "ComponentUpdate"
    ) -> "StackComponentSchema":
        """Updates a `StackComponentSchema` from a `ComponentUpdate`.

        Args:
            component_update: The `ComponentUpdate` to update from.

        Returns:
            The updated `StackComponentSchema`.
        """
        for field, value in component_update.dict(
            exclude_unset=True, exclude={"workspace", "user", "connector"}
        ).items():
            if field == "configuration":
                self.configuration = base64.b64encode(
                    json.dumps(component_update.configuration).encode("utf-8")
                )
            elif field == "labels":
                self.labels = base64.b64encode(
                    json.dumps(component_update.labels).encode("utf-8")
                )
            else:
                setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "ComponentResponse":
        """Creates a `ComponentModel` from an instance of a `StackComponentSchema`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            A `ComponentModel`
        """
        body = ComponentResponseBody(
            type=self.type,
            flavor=self.flavor,
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = ComponentResponseMetadata(
                workspace=self.workspace.to_model(),
                configuration=json.loads(
                    base64.b64decode(self.configuration).decode()
                ),
                labels=json.loads(base64.b64decode(self.labels).decode())
                if self.labels
                else None,
                component_spec_path=self.component_spec_path,
                connector_resource_id=self.connector_resource_id,
                connector=self.connector.to_model()
                if self.connector
                else None,
            )
        return ComponentResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Creates a ComponentModel from an instance of a StackComponentSchema.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ComponentResponse

A ComponentModel

Source code in zenml/zen_stores/schemas/component_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "ComponentResponse":
    """Creates a `ComponentModel` from an instance of a `StackComponentSchema`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        A `ComponentModel`
    """
    body = ComponentResponseBody(
        type=self.type,
        flavor=self.flavor,
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = ComponentResponseMetadata(
            workspace=self.workspace.to_model(),
            configuration=json.loads(
                base64.b64decode(self.configuration).decode()
            ),
            labels=json.loads(base64.b64decode(self.labels).decode())
            if self.labels
            else None,
            component_spec_path=self.component_spec_path,
            connector_resource_id=self.connector_resource_id,
            connector=self.connector.to_model()
            if self.connector
            else None,
        )
    return ComponentResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, component_update)

Updates a StackComponentSchema from a ComponentUpdate.

Parameters:

Name Type Description Default
component_update ComponentUpdate

The ComponentUpdate to update from.

required

Returns:

Type Description
StackComponentSchema

The updated StackComponentSchema.

Source code in zenml/zen_stores/schemas/component_schemas.py
def update(
    self, component_update: "ComponentUpdate"
) -> "StackComponentSchema":
    """Updates a `StackComponentSchema` from a `ComponentUpdate`.

    Args:
        component_update: The `ComponentUpdate` to update from.

    Returns:
        The updated `StackComponentSchema`.
    """
    for field, value in component_update.dict(
        exclude_unset=True, exclude={"workspace", "user", "connector"}
    ).items():
        if field == "configuration":
            self.configuration = base64.b64encode(
                json.dumps(component_update.configuration).encode("utf-8")
            )
        elif field == "labels":
            self.labels = base64.b64encode(
                json.dumps(component_update.labels).encode("utf-8")
            )
        else:
            setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self

device_schemas

SQLModel implementation for authorized OAuth2 devices.

OAuthDeviceSchema (BaseSchema) pydantic-model

SQL Model for authorized OAuth2 devices.

Source code in zenml/zen_stores/schemas/device_schemas.py
class OAuthDeviceSchema(BaseSchema, table=True):
    """SQL Model for authorized OAuth2 devices."""

    __tablename__ = "auth_devices"

    client_id: UUID
    user_code: str
    device_code: str
    status: OAuthDeviceStatus
    failed_auth_attempts: int = 0
    expires: Optional[datetime] = None
    last_login: Optional[datetime] = None
    trusted_device: bool = False
    os: Optional[str] = None
    ip_address: Optional[str] = None
    hostname: Optional[str] = None
    python_version: Optional[str] = None
    zenml_version: Optional[str] = None
    city: Optional[str] = None
    region: Optional[str] = None
    country: Optional[str] = None

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="auth_devices")

    @classmethod
    def _generate_user_code(cls) -> str:
        """Generate a user code for an OAuth2 device.

        Returns:
            The generated user code.
        """
        return token_hex(16)

    @classmethod
    def _generate_device_code(cls) -> str:
        """Generate a device code.

        Returns:
            The generated device code.
        """
        return token_hex(32)

    @classmethod
    def _get_hashed_code(cls, code: str) -> str:
        """Hashes the input code and returns the hash value.

        Args:
            code: The code value to hash.

        Returns:
            The code hash value.
        """
        context = CryptContext(schemes=["bcrypt"], deprecated="auto")
        return context.hash(code)

    @classmethod
    def from_request(
        cls, request: OAuthDeviceInternalRequest
    ) -> Tuple["OAuthDeviceSchema", str, str]:
        """Create an authorized device DB entry from a device authorization request.

        Args:
            request: The device authorization request.

        Returns:
            The created `OAuthDeviceSchema`, the user code and the device code.
        """
        user_code = cls._generate_user_code()
        device_code = cls._generate_device_code()
        hashed_user_code = cls._get_hashed_code(user_code)
        hashed_device_code = cls._get_hashed_code(device_code)
        now = datetime.utcnow()
        return (
            cls(
                client_id=request.client_id,
                user_code=hashed_user_code,
                device_code=hashed_device_code,
                status=OAuthDeviceStatus.PENDING,
                failed_auth_attempts=0,
                expires=now + timedelta(seconds=request.expires_in),
                os=request.os,
                ip_address=request.ip_address,
                hostname=request.hostname,
                python_version=request.python_version,
                zenml_version=request.zenml_version,
                city=request.city,
                region=request.region,
                country=request.country,
                created=now,
                updated=now,
            ),
            user_code,
            device_code,
        )

    def update(self, device_update: OAuthDeviceUpdate) -> "OAuthDeviceSchema":
        """Update an authorized device from a device update model.

        Args:
            device_update: The device update model.

        Returns:
            The updated `OAuthDeviceSchema`.
        """
        for field, value in device_update.dict(exclude_none=True).items():
            if hasattr(self, field):
                setattr(self, field, value)

        if device_update.locked is True:
            self.status = OAuthDeviceStatus.LOCKED
        elif device_update.locked is False:
            self.status = OAuthDeviceStatus.ACTIVE

        self.updated = datetime.utcnow()
        return self

    def internal_update(
        self, device_update: OAuthDeviceInternalUpdate
    ) -> Tuple["OAuthDeviceSchema", Optional[str], Optional[str]]:
        """Update an authorized device from an internal device update model.

        Args:
            device_update: The internal device update model.

        Returns:
            The updated `OAuthDeviceSchema` and the new user code and device
            code, if they were generated.
        """
        now = datetime.utcnow()
        user_code: Optional[str] = None
        device_code: Optional[str] = None

        # This call also takes care of setting fields that have the same
        # name in the internal model and the schema.
        self.update(device_update)

        if device_update.expires_in is not None:
            if device_update.expires_in <= 0:
                self.expires = None
            else:
                self.expires = now + timedelta(
                    seconds=device_update.expires_in
                )
        if device_update.update_last_login:
            self.last_login = now
        if device_update.generate_new_codes:
            user_code = self._generate_user_code()
            device_code = self._generate_device_code()
            self.user_code = self._get_hashed_code(user_code)
            self.device_code = self._get_hashed_code(device_code)
        self.updated = now
        return self, user_code, device_code

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> OAuthDeviceResponse:
        """Convert a device schema to a device response model.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The converted device response model.
        """
        metadata = None
        if include_metadata:
            metadata = OAuthDeviceResponseMetadata(
                python_version=self.python_version,
                zenml_version=self.zenml_version,
                city=self.city,
                region=self.region,
                country=self.country,
                failed_auth_attempts=self.failed_auth_attempts,
                last_login=self.last_login,
            )

        body = OAuthDeviceResponseBody(
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
            client_id=self.client_id,
            expires=self.expires,
            trusted_device=self.trusted_device,
            status=self.status,
            os=self.os,
            ip_address=self.ip_address,
            hostname=self.hostname,
        )
        return OAuthDeviceResponse(
            id=self.id,
            body=body,
            metadata=metadata,
        )

    def to_internal_model(
        self, hydrate: bool = False
    ) -> OAuthDeviceInternalResponse:
        """Convert a device schema to an internal device response model.

        Args:
            hydrate: bool to decide whether to return a hydrated version of the
                model.

        Returns:
            The converted internal device response model.
        """
        device_model = self.to_model(include_metadata=hydrate)
        return OAuthDeviceInternalResponse(
            id=device_model.id,
            body=device_model.body,
            metadata=device_model.metadata,
            user_code=self.user_code,
            device_code=self.device_code,
        )
from_request(request) classmethod

Create an authorized device DB entry from a device authorization request.

Parameters:

Name Type Description Default
request OAuthDeviceInternalRequest

The device authorization request.

required

Returns:

Type Description
Tuple[OAuthDeviceSchema, str, str]

The created OAuthDeviceSchema, the user code and the device code.

Source code in zenml/zen_stores/schemas/device_schemas.py
@classmethod
def from_request(
    cls, request: OAuthDeviceInternalRequest
) -> Tuple["OAuthDeviceSchema", str, str]:
    """Create an authorized device DB entry from a device authorization request.

    Args:
        request: The device authorization request.

    Returns:
        The created `OAuthDeviceSchema`, the user code and the device code.
    """
    user_code = cls._generate_user_code()
    device_code = cls._generate_device_code()
    hashed_user_code = cls._get_hashed_code(user_code)
    hashed_device_code = cls._get_hashed_code(device_code)
    now = datetime.utcnow()
    return (
        cls(
            client_id=request.client_id,
            user_code=hashed_user_code,
            device_code=hashed_device_code,
            status=OAuthDeviceStatus.PENDING,
            failed_auth_attempts=0,
            expires=now + timedelta(seconds=request.expires_in),
            os=request.os,
            ip_address=request.ip_address,
            hostname=request.hostname,
            python_version=request.python_version,
            zenml_version=request.zenml_version,
            city=request.city,
            region=request.region,
            country=request.country,
            created=now,
            updated=now,
        ),
        user_code,
        device_code,
    )
internal_update(self, device_update)

Update an authorized device from an internal device update model.

Parameters:

Name Type Description Default
device_update OAuthDeviceInternalUpdate

The internal device update model.

required

Returns:

Type Description
Tuple[OAuthDeviceSchema, Optional[str], Optional[str]]

The updated OAuthDeviceSchema and the new user code and device code, if they were generated.

Source code in zenml/zen_stores/schemas/device_schemas.py
def internal_update(
    self, device_update: OAuthDeviceInternalUpdate
) -> Tuple["OAuthDeviceSchema", Optional[str], Optional[str]]:
    """Update an authorized device from an internal device update model.

    Args:
        device_update: The internal device update model.

    Returns:
        The updated `OAuthDeviceSchema` and the new user code and device
        code, if they were generated.
    """
    now = datetime.utcnow()
    user_code: Optional[str] = None
    device_code: Optional[str] = None

    # This call also takes care of setting fields that have the same
    # name in the internal model and the schema.
    self.update(device_update)

    if device_update.expires_in is not None:
        if device_update.expires_in <= 0:
            self.expires = None
        else:
            self.expires = now + timedelta(
                seconds=device_update.expires_in
            )
    if device_update.update_last_login:
        self.last_login = now
    if device_update.generate_new_codes:
        user_code = self._generate_user_code()
        device_code = self._generate_device_code()
        self.user_code = self._get_hashed_code(user_code)
        self.device_code = self._get_hashed_code(device_code)
    self.updated = now
    return self, user_code, device_code
to_internal_model(self, hydrate=False)

Convert a device schema to an internal device response model.

Parameters:

Name Type Description Default
hydrate bool

bool to decide whether to return a hydrated version of the model.

False

Returns:

Type Description
OAuthDeviceInternalResponse

The converted internal device response model.

Source code in zenml/zen_stores/schemas/device_schemas.py
def to_internal_model(
    self, hydrate: bool = False
) -> OAuthDeviceInternalResponse:
    """Convert a device schema to an internal device response model.

    Args:
        hydrate: bool to decide whether to return a hydrated version of the
            model.

    Returns:
        The converted internal device response model.
    """
    device_model = self.to_model(include_metadata=hydrate)
    return OAuthDeviceInternalResponse(
        id=device_model.id,
        body=device_model.body,
        metadata=device_model.metadata,
        user_code=self.user_code,
        device_code=self.device_code,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a device schema to a device response model.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
OAuthDeviceResponse

The converted device response model.

Source code in zenml/zen_stores/schemas/device_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> OAuthDeviceResponse:
    """Convert a device schema to a device response model.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The converted device response model.
    """
    metadata = None
    if include_metadata:
        metadata = OAuthDeviceResponseMetadata(
            python_version=self.python_version,
            zenml_version=self.zenml_version,
            city=self.city,
            region=self.region,
            country=self.country,
            failed_auth_attempts=self.failed_auth_attempts,
            last_login=self.last_login,
        )

    body = OAuthDeviceResponseBody(
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
        client_id=self.client_id,
        expires=self.expires,
        trusted_device=self.trusted_device,
        status=self.status,
        os=self.os,
        ip_address=self.ip_address,
        hostname=self.hostname,
    )
    return OAuthDeviceResponse(
        id=self.id,
        body=body,
        metadata=metadata,
    )
update(self, device_update)

Update an authorized device from a device update model.

Parameters:

Name Type Description Default
device_update OAuthDeviceUpdate

The device update model.

required

Returns:

Type Description
OAuthDeviceSchema

The updated OAuthDeviceSchema.

Source code in zenml/zen_stores/schemas/device_schemas.py
def update(self, device_update: OAuthDeviceUpdate) -> "OAuthDeviceSchema":
    """Update an authorized device from a device update model.

    Args:
        device_update: The device update model.

    Returns:
        The updated `OAuthDeviceSchema`.
    """
    for field, value in device_update.dict(exclude_none=True).items():
        if hasattr(self, field):
            setattr(self, field, value)

    if device_update.locked is True:
        self.status = OAuthDeviceStatus.LOCKED
    elif device_update.locked is False:
        self.status = OAuthDeviceStatus.ACTIVE

    self.updated = datetime.utcnow()
    return self

event_source_schemas

SQL Model Implementations for event sources.

EventSourceSchema (NamedSchema) pydantic-model

SQL Model for tag.

Source code in zenml/zen_stores/schemas/event_source_schemas.py
class EventSourceSchema(NamedSchema, table=True):
    """SQL Model for tag."""

    __tablename__ = "event_source"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="event_sources")

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="event_sources")

    triggers: List["TriggerSchema"] = Relationship(
        back_populates="event_source"
    )

    flavor: str = Field(nullable=False)
    plugin_subtype: str = Field(nullable=False)
    description: str = Field(sa_column=Column(TEXT, nullable=True))

    configuration: bytes
    is_active: bool = Field(nullable=False)

    @classmethod
    def from_request(cls, request: EventSourceRequest) -> "EventSourceSchema":
        """Convert an `EventSourceRequest` to an `EventSourceSchema`.

        Args:
            request: The request model to convert.

        Returns:
            The converted schema.
        """
        return cls(
            workspace_id=request.workspace,
            user_id=request.user,
            flavor=request.flavor,
            plugin_subtype=request.plugin_subtype,
            name=request.name,
            description=request.description,
            configuration=base64.b64encode(
                json.dumps(
                    request.configuration,
                    sort_keys=False,
                    default=pydantic_encoder,
                ).encode("utf-8")
            ),
            is_active=True,  # Makes no sense to create an inactive event source
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> EventSourceResponse:
        """Convert an `EventSourceSchema` to an `EventSourceResponse`.

        Args:
            include_metadata: Flag deciding whether to include the output model(s)
                metadata fields in the response.
            include_resources: Flag deciding whether to include the output model(s)
                metadata fields in the response.
            **kwargs: Keyword arguments to allow schema specific logic

        Returns:
            The created `EventSourceResponse`.
        """
        from zenml.models import TriggerResponse

        body = EventSourceResponseBody(
            created=self.created,
            updated=self.updated,
            user=self.user.to_model() if self.user else None,
            flavor=self.flavor,
            plugin_subtype=self.plugin_subtype,
            is_active=self.is_active,
        )
        resources = None
        if include_resources:
            triggers = cast(
                Page[TriggerResponse],
                get_page_from_list(
                    items_list=self.triggers,
                    response_model=TriggerResponse,
                    include_resources=include_resources,
                    include_metadata=include_metadata,
                ),
            )
            resources = EventSourceResponseResources(
                triggers=triggers,
            )
        metadata = None
        if include_metadata:
            metadata = EventSourceResponseMetadata(
                workspace=self.workspace.to_model(),
                description=self.description,
                configuration=json.loads(
                    base64.b64decode(self.configuration).decode()
                ),
            )
        return EventSourceResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
            resources=resources,
        )

    def update(self, update: EventSourceUpdate) -> "EventSourceSchema":
        """Updates a `EventSourceSchema` from a `EventSourceUpdate`.

        Args:
            update: The `EventSourceUpdate` to update from.

        Returns:
            The updated `EventSourceSchema`.
        """
        for field, value in update.dict(
            exclude_unset=True, exclude_none=True
        ).items():
            if field == "configuration":
                self.configuration = base64.b64encode(
                    json.dumps(
                        update.configuration, default=pydantic_encoder
                    ).encode("utf-8")
                )
            else:
                setattr(self, field, value)
        self.updated = datetime.utcnow()
        return self
from_request(request) classmethod

Convert an EventSourceRequest to an EventSourceSchema.

Parameters:

Name Type Description Default
request EventSourceRequest

The request model to convert.

required

Returns:

Type Description
EventSourceSchema

The converted schema.

Source code in zenml/zen_stores/schemas/event_source_schemas.py
@classmethod
def from_request(cls, request: EventSourceRequest) -> "EventSourceSchema":
    """Convert an `EventSourceRequest` to an `EventSourceSchema`.

    Args:
        request: The request model to convert.

    Returns:
        The converted schema.
    """
    return cls(
        workspace_id=request.workspace,
        user_id=request.user,
        flavor=request.flavor,
        plugin_subtype=request.plugin_subtype,
        name=request.name,
        description=request.description,
        configuration=base64.b64encode(
            json.dumps(
                request.configuration,
                sort_keys=False,
                default=pydantic_encoder,
            ).encode("utf-8")
        ),
        is_active=True,  # Makes no sense to create an inactive event source
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an EventSourceSchema to an EventSourceResponse.

Parameters:

Name Type Description Default
include_metadata bool

Flag deciding whether to include the output model(s) metadata fields in the response.

False
include_resources bool

Flag deciding whether to include the output model(s) metadata fields in the response.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
EventSourceResponse

The created EventSourceResponse.

Source code in zenml/zen_stores/schemas/event_source_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> EventSourceResponse:
    """Convert an `EventSourceSchema` to an `EventSourceResponse`.

    Args:
        include_metadata: Flag deciding whether to include the output model(s)
            metadata fields in the response.
        include_resources: Flag deciding whether to include the output model(s)
            metadata fields in the response.
        **kwargs: Keyword arguments to allow schema specific logic

    Returns:
        The created `EventSourceResponse`.
    """
    from zenml.models import TriggerResponse

    body = EventSourceResponseBody(
        created=self.created,
        updated=self.updated,
        user=self.user.to_model() if self.user else None,
        flavor=self.flavor,
        plugin_subtype=self.plugin_subtype,
        is_active=self.is_active,
    )
    resources = None
    if include_resources:
        triggers = cast(
            Page[TriggerResponse],
            get_page_from_list(
                items_list=self.triggers,
                response_model=TriggerResponse,
                include_resources=include_resources,
                include_metadata=include_metadata,
            ),
        )
        resources = EventSourceResponseResources(
            triggers=triggers,
        )
    metadata = None
    if include_metadata:
        metadata = EventSourceResponseMetadata(
            workspace=self.workspace.to_model(),
            description=self.description,
            configuration=json.loads(
                base64.b64decode(self.configuration).decode()
            ),
        )
    return EventSourceResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
        resources=resources,
    )
update(self, update)

Updates a EventSourceSchema from a EventSourceUpdate.

Parameters:

Name Type Description Default
update EventSourceUpdate

The EventSourceUpdate to update from.

required

Returns:

Type Description
EventSourceSchema

The updated EventSourceSchema.

Source code in zenml/zen_stores/schemas/event_source_schemas.py
def update(self, update: EventSourceUpdate) -> "EventSourceSchema":
    """Updates a `EventSourceSchema` from a `EventSourceUpdate`.

    Args:
        update: The `EventSourceUpdate` to update from.

    Returns:
        The updated `EventSourceSchema`.
    """
    for field, value in update.dict(
        exclude_unset=True, exclude_none=True
    ).items():
        if field == "configuration":
            self.configuration = base64.b64encode(
                json.dumps(
                    update.configuration, default=pydantic_encoder
                ).encode("utf-8")
            )
        else:
            setattr(self, field, value)
    self.updated = datetime.utcnow()
    return self

flavor_schemas

SQL Model Implementations for Flavors.

FlavorSchema (NamedSchema) pydantic-model

SQL Model for flavors.

Attributes:

Name Type Description
type StackComponentType

The type of the flavor.

source str

The source of the flavor.

config_schema str

The config schema of the flavor.

integration Optional[str]

The integration associated with the flavor.

Source code in zenml/zen_stores/schemas/flavor_schemas.py
class FlavorSchema(NamedSchema, table=True):
    """SQL Model for flavors.

    Attributes:
        type: The type of the flavor.
        source: The source of the flavor.
        config_schema: The config schema of the flavor.
        integration: The integration associated with the flavor.
    """

    __tablename__ = "flavor"

    type: StackComponentType
    source: str
    config_schema: str = Field(sa_column=Column(TEXT, nullable=False))
    integration: Optional[str] = Field(default="")
    connector_type: Optional[str]
    connector_resource_type: Optional[str]
    connector_resource_id_attr: Optional[str]

    workspace_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=True,
    )
    workspace: Optional["WorkspaceSchema"] = Relationship(
        back_populates="flavors"
    )

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="flavors")

    logo_url: Optional[str] = Field()

    docs_url: Optional[str] = Field()

    sdk_docs_url: Optional[str] = Field()

    is_custom: bool = Field(default=True)

    def update(self, flavor_update: "FlavorUpdate") -> "FlavorSchema":
        """Update a `FlavorSchema` from a `FlavorUpdate`.

        Args:
            flavor_update: The `FlavorUpdate` from which to update the schema.

        Returns:
            The updated `FlavorSchema`.
        """
        for field, value in flavor_update.dict(
            exclude_unset=True, exclude={"workspace", "user"}
        ).items():
            if field == "config_schema":
                setattr(self, field, json.dumps(value))
            else:
                setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "FlavorResponse":
        """Converts a flavor schema to a flavor model.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The flavor model.
        """
        body = FlavorResponseBody(
            user=self.user.to_model() if self.user else None,
            type=self.type,
            integration=self.integration,
            logo_url=self.logo_url,
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = FlavorResponseMetadata(
                workspace=self.workspace.to_model()
                if self.workspace
                else None,
                config_schema=json.loads(self.config_schema),
                connector_type=self.connector_type,
                connector_resource_type=self.connector_resource_type,
                connector_resource_id_attr=self.connector_resource_id_attr,
                source=self.source,
                docs_url=self.docs_url,
                sdk_docs_url=self.sdk_docs_url,
                is_custom=self.is_custom,
            )
        return FlavorResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Converts a flavor schema to a flavor model.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
FlavorResponse

The flavor model.

Source code in zenml/zen_stores/schemas/flavor_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "FlavorResponse":
    """Converts a flavor schema to a flavor model.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The flavor model.
    """
    body = FlavorResponseBody(
        user=self.user.to_model() if self.user else None,
        type=self.type,
        integration=self.integration,
        logo_url=self.logo_url,
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = FlavorResponseMetadata(
            workspace=self.workspace.to_model()
            if self.workspace
            else None,
            config_schema=json.loads(self.config_schema),
            connector_type=self.connector_type,
            connector_resource_type=self.connector_resource_type,
            connector_resource_id_attr=self.connector_resource_id_attr,
            source=self.source,
            docs_url=self.docs_url,
            sdk_docs_url=self.sdk_docs_url,
            is_custom=self.is_custom,
        )
    return FlavorResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, flavor_update)

Update a FlavorSchema from a FlavorUpdate.

Parameters:

Name Type Description Default
flavor_update FlavorUpdate

The FlavorUpdate from which to update the schema.

required

Returns:

Type Description
FlavorSchema

The updated FlavorSchema.

Source code in zenml/zen_stores/schemas/flavor_schemas.py
def update(self, flavor_update: "FlavorUpdate") -> "FlavorSchema":
    """Update a `FlavorSchema` from a `FlavorUpdate`.

    Args:
        flavor_update: The `FlavorUpdate` from which to update the schema.

    Returns:
        The updated `FlavorSchema`.
    """
    for field, value in flavor_update.dict(
        exclude_unset=True, exclude={"workspace", "user"}
    ).items():
        if field == "config_schema":
            setattr(self, field, json.dumps(value))
        else:
            setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self

identity_schemas

SQLModel implementation for the server information table.

IdentitySchema (SQLModel) pydantic-model

SQL Model for the client/server identity.

Source code in zenml/zen_stores/schemas/identity_schemas.py
class IdentitySchema(SQLModel, table=True):
    """SQL Model for the client/server identity."""

    __tablename__ = "identity"

    id: UUID = Field(primary_key=True)

logs_schemas

SQLModel implementation of pipeline logs tables.

LogsSchema (BaseSchema) pydantic-model

SQL Model for logs.

Source code in zenml/zen_stores/schemas/logs_schemas.py
class LogsSchema(BaseSchema, table=True):
    """SQL Model for logs."""

    __tablename__ = "logs"

    # Fields
    uri: str = Field(sa_column=Column(TEXT, nullable=False))

    # Foreign Keys
    pipeline_run_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=PipelineRunSchema.__tablename__,
        source_column="pipeline_run_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=True,
    )
    step_run_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=StepRunSchema.__tablename__,
        source_column="step_run_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=True,
    )
    artifact_store_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=StackComponentSchema.__tablename__,
        source_column="stack_component_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )

    # Relationships
    artifact_store: Optional["StackComponentSchema"] = Relationship(
        back_populates="run_or_step_logs"
    )
    pipeline_run: Optional["PipelineRunSchema"] = Relationship(
        back_populates="logs"
    )
    step_run: Optional["StepRunSchema"] = Relationship(back_populates="logs")

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "LogsResponse":
        """Convert a `LogsSchema` to a `LogsResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `LogsResponse`.
        """
        body = LogsResponseBody(
            uri=self.uri,
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = LogsResponseMetadata(
                step_run_id=self.step_run_id,
                pipeline_run_id=self.pipeline_run_id,
                artifact_store_id=self.artifact_store_id,
            )
        return LogsResponse(
            id=self.id,
            body=body,
            metadata=metadata,
        )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a LogsSchema to a LogsResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
LogsResponse

The created LogsResponse.

Source code in zenml/zen_stores/schemas/logs_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "LogsResponse":
    """Convert a `LogsSchema` to a `LogsResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `LogsResponse`.
    """
    body = LogsResponseBody(
        uri=self.uri,
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = LogsResponseMetadata(
            step_run_id=self.step_run_id,
            pipeline_run_id=self.pipeline_run_id,
            artifact_store_id=self.artifact_store_id,
        )
    return LogsResponse(
        id=self.id,
        body=body,
        metadata=metadata,
    )

model_schemas

SQLModel implementation of model tables.

ModelSchema (NamedSchema) pydantic-model

SQL Model for model.

Source code in zenml/zen_stores/schemas/model_schemas.py
class ModelSchema(NamedSchema, table=True):
    """SQL Model for model."""

    __tablename__ = "model"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="models")

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="models")

    license: str = Field(sa_column=Column(TEXT, nullable=True))
    description: str = Field(sa_column=Column(TEXT, nullable=True))
    audience: str = Field(sa_column=Column(TEXT, nullable=True))
    use_cases: str = Field(sa_column=Column(TEXT, nullable=True))
    limitations: str = Field(sa_column=Column(TEXT, nullable=True))
    trade_offs: str = Field(sa_column=Column(TEXT, nullable=True))
    ethics: str = Field(sa_column=Column(TEXT, nullable=True))
    save_models_to_registry: bool = Field(
        sa_column=Column(BOOLEAN, nullable=False)
    )
    tags: List["TagResourceSchema"] = Relationship(
        back_populates="model",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(TagResourceSchema.resource_type=='{TaggableResourceTypes.MODEL.value}', foreign(TagResourceSchema.resource_id)==ModelSchema.id)",
            cascade="delete",
            overlaps="tags",
        ),
    )
    model_versions: List["ModelVersionSchema"] = Relationship(
        back_populates="model",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    artifact_links: List["ModelVersionArtifactSchema"] = Relationship(
        back_populates="model",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    pipeline_run_links: List["ModelVersionPipelineRunSchema"] = Relationship(
        back_populates="model",
        sa_relationship_kwargs={"cascade": "delete"},
    )

    @classmethod
    def from_request(cls, model_request: ModelRequest) -> "ModelSchema":
        """Convert an `ModelRequest` to an `ModelSchema`.

        Args:
            model_request: The request model to convert.

        Returns:
            The converted schema.
        """
        return cls(
            name=model_request.name,
            workspace_id=model_request.workspace,
            user_id=model_request.user,
            license=model_request.license,
            description=model_request.description,
            audience=model_request.audience,
            use_cases=model_request.use_cases,
            limitations=model_request.limitations,
            trade_offs=model_request.trade_offs,
            ethics=model_request.ethics,
            save_models_to_registry=model_request.save_models_to_registry,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> ModelResponse:
        """Convert an `ModelSchema` to an `ModelResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `ModelResponse`.
        """
        tags = [t.tag.to_model() for t in self.tags]

        if self.model_versions:
            version_numbers = [mv.number for mv in self.model_versions]
            latest_version_idx = version_numbers.index(max(version_numbers))
            latest_version_name = self.model_versions[latest_version_idx].name
            latest_version_id = self.model_versions[latest_version_idx].id
        else:
            latest_version_name = None
            latest_version_id = None

        metadata = None
        if include_metadata:
            metadata = ModelResponseMetadata(
                workspace=self.workspace.to_model(),
                license=self.license,
                description=self.description,
                audience=self.audience,
                use_cases=self.use_cases,
                limitations=self.limitations,
                trade_offs=self.trade_offs,
                ethics=self.ethics,
                save_models_to_registry=self.save_models_to_registry,
            )

        body = ModelResponseBody(
            user=self.user.to_model() if self.user else None,
            workspace=self.workspace.to_model(),
            created=self.created,
            updated=self.updated,
            tags=tags,
            latest_version_name=latest_version_name,
            latest_version_id=latest_version_id,
        )

        return ModelResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )

    def update(
        self,
        model_update: ModelUpdate,
    ) -> "ModelSchema":
        """Updates a `ModelSchema` from a `ModelUpdate`.

        Args:
            model_update: The `ModelUpdate` to update from.

        Returns:
            The updated `ModelSchema`.
        """
        for field, value in model_update.dict(
            exclude_unset=True, exclude_none=True
        ).items():
            setattr(self, field, value)
        self.updated = datetime.utcnow()
        return self
from_request(model_request) classmethod

Convert an ModelRequest to an ModelSchema.

Parameters:

Name Type Description Default
model_request ModelRequest

The request model to convert.

required

Returns:

Type Description
ModelSchema

The converted schema.

Source code in zenml/zen_stores/schemas/model_schemas.py
@classmethod
def from_request(cls, model_request: ModelRequest) -> "ModelSchema":
    """Convert an `ModelRequest` to an `ModelSchema`.

    Args:
        model_request: The request model to convert.

    Returns:
        The converted schema.
    """
    return cls(
        name=model_request.name,
        workspace_id=model_request.workspace,
        user_id=model_request.user,
        license=model_request.license,
        description=model_request.description,
        audience=model_request.audience,
        use_cases=model_request.use_cases,
        limitations=model_request.limitations,
        trade_offs=model_request.trade_offs,
        ethics=model_request.ethics,
        save_models_to_registry=model_request.save_models_to_registry,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an ModelSchema to an ModelResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ModelResponse

The created ModelResponse.

Source code in zenml/zen_stores/schemas/model_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> ModelResponse:
    """Convert an `ModelSchema` to an `ModelResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `ModelResponse`.
    """
    tags = [t.tag.to_model() for t in self.tags]

    if self.model_versions:
        version_numbers = [mv.number for mv in self.model_versions]
        latest_version_idx = version_numbers.index(max(version_numbers))
        latest_version_name = self.model_versions[latest_version_idx].name
        latest_version_id = self.model_versions[latest_version_idx].id
    else:
        latest_version_name = None
        latest_version_id = None

    metadata = None
    if include_metadata:
        metadata = ModelResponseMetadata(
            workspace=self.workspace.to_model(),
            license=self.license,
            description=self.description,
            audience=self.audience,
            use_cases=self.use_cases,
            limitations=self.limitations,
            trade_offs=self.trade_offs,
            ethics=self.ethics,
            save_models_to_registry=self.save_models_to_registry,
        )

    body = ModelResponseBody(
        user=self.user.to_model() if self.user else None,
        workspace=self.workspace.to_model(),
        created=self.created,
        updated=self.updated,
        tags=tags,
        latest_version_name=latest_version_name,
        latest_version_id=latest_version_id,
    )

    return ModelResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, model_update)

Updates a ModelSchema from a ModelUpdate.

Parameters:

Name Type Description Default
model_update ModelUpdate

The ModelUpdate to update from.

required

Returns:

Type Description
ModelSchema

The updated ModelSchema.

Source code in zenml/zen_stores/schemas/model_schemas.py
def update(
    self,
    model_update: ModelUpdate,
) -> "ModelSchema":
    """Updates a `ModelSchema` from a `ModelUpdate`.

    Args:
        model_update: The `ModelUpdate` to update from.

    Returns:
        The updated `ModelSchema`.
    """
    for field, value in model_update.dict(
        exclude_unset=True, exclude_none=True
    ).items():
        setattr(self, field, value)
    self.updated = datetime.utcnow()
    return self
ModelVersionArtifactSchema (BaseSchema) pydantic-model

SQL Model for linking of Model Versions and Artifacts M:M.

Source code in zenml/zen_stores/schemas/model_schemas.py
class ModelVersionArtifactSchema(BaseSchema, table=True):
    """SQL Model for linking of Model Versions and Artifacts M:M."""

    __tablename__ = "model_versions_artifacts"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(
        back_populates="model_versions_artifacts_links"
    )

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(
        back_populates="model_versions_artifacts_links"
    )

    model_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=ModelSchema.__tablename__,
        source_column="model_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    model: "ModelSchema" = Relationship(back_populates="artifact_links")
    model_version_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=ModelVersionSchema.__tablename__,
        source_column="model_version_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    model_version: "ModelVersionSchema" = Relationship(
        back_populates="artifact_links"
    )
    artifact_version_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=ArtifactVersionSchema.__tablename__,
        source_column="artifact_version_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    artifact_version: "ArtifactVersionSchema" = Relationship(
        back_populates="model_versions_artifacts_links"
    )

    is_model_artifact: bool = Field(sa_column=Column(BOOLEAN, nullable=True))
    is_deployment_artifact: bool = Field(
        sa_column=Column(BOOLEAN, nullable=True)
    )

    @classmethod
    def from_request(
        cls,
        model_version_artifact_request: ModelVersionArtifactRequest,
    ) -> "ModelVersionArtifactSchema":
        """Convert an `ModelVersionArtifactRequest` to a `ModelVersionArtifactSchema`.

        Args:
            model_version_artifact_request: The request link to convert.

        Returns:
            The converted schema.
        """
        return cls(
            workspace_id=model_version_artifact_request.workspace,
            user_id=model_version_artifact_request.user,
            model_id=model_version_artifact_request.model,
            model_version_id=model_version_artifact_request.model_version,
            artifact_version_id=model_version_artifact_request.artifact_version,
            is_model_artifact=model_version_artifact_request.is_model_artifact,
            is_deployment_artifact=model_version_artifact_request.is_deployment_artifact,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> ModelVersionArtifactResponse:
        """Convert an `ModelVersionArtifactSchema` to an `ModelVersionArtifactResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `ModelVersionArtifactResponseModel`.
        """
        return ModelVersionArtifactResponse(
            id=self.id,
            body=ModelVersionArtifactResponseBody(
                created=self.created,
                updated=self.updated,
                model=self.model_id,
                model_version=self.model_version_id,
                artifact_version=self.artifact_version.to_model(),
                is_model_artifact=self.is_model_artifact,
                is_deployment_artifact=self.is_deployment_artifact,
            ),
            metadata=BaseResponseMetadata() if include_metadata else None,
        )
from_request(model_version_artifact_request) classmethod

Convert an ModelVersionArtifactRequest to a ModelVersionArtifactSchema.

Parameters:

Name Type Description Default
model_version_artifact_request ModelVersionArtifactRequest

The request link to convert.

required

Returns:

Type Description
ModelVersionArtifactSchema

The converted schema.

Source code in zenml/zen_stores/schemas/model_schemas.py
@classmethod
def from_request(
    cls,
    model_version_artifact_request: ModelVersionArtifactRequest,
) -> "ModelVersionArtifactSchema":
    """Convert an `ModelVersionArtifactRequest` to a `ModelVersionArtifactSchema`.

    Args:
        model_version_artifact_request: The request link to convert.

    Returns:
        The converted schema.
    """
    return cls(
        workspace_id=model_version_artifact_request.workspace,
        user_id=model_version_artifact_request.user,
        model_id=model_version_artifact_request.model,
        model_version_id=model_version_artifact_request.model_version,
        artifact_version_id=model_version_artifact_request.artifact_version,
        is_model_artifact=model_version_artifact_request.is_model_artifact,
        is_deployment_artifact=model_version_artifact_request.is_deployment_artifact,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an ModelVersionArtifactSchema to an ModelVersionArtifactResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ModelVersionArtifactResponse

The created ModelVersionArtifactResponseModel.

Source code in zenml/zen_stores/schemas/model_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> ModelVersionArtifactResponse:
    """Convert an `ModelVersionArtifactSchema` to an `ModelVersionArtifactResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `ModelVersionArtifactResponseModel`.
    """
    return ModelVersionArtifactResponse(
        id=self.id,
        body=ModelVersionArtifactResponseBody(
            created=self.created,
            updated=self.updated,
            model=self.model_id,
            model_version=self.model_version_id,
            artifact_version=self.artifact_version.to_model(),
            is_model_artifact=self.is_model_artifact,
            is_deployment_artifact=self.is_deployment_artifact,
        ),
        metadata=BaseResponseMetadata() if include_metadata else None,
    )
ModelVersionPipelineRunSchema (BaseSchema) pydantic-model

SQL Model for linking of Model Versions and Pipeline Runs M:M.

Source code in zenml/zen_stores/schemas/model_schemas.py
class ModelVersionPipelineRunSchema(BaseSchema, table=True):
    """SQL Model for linking of Model Versions and Pipeline Runs M:M."""

    __tablename__ = "model_versions_runs"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(
        back_populates="model_versions_pipeline_runs_links"
    )

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(
        back_populates="model_versions_pipeline_runs_links"
    )

    model_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=ModelSchema.__tablename__,
        source_column="model_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    model: "ModelSchema" = Relationship(back_populates="pipeline_run_links")
    model_version_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=ModelVersionSchema.__tablename__,
        source_column="model_version_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    model_version: "ModelVersionSchema" = Relationship(
        back_populates="pipeline_run_links"
    )
    pipeline_run_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=PipelineRunSchema.__tablename__,
        source_column="pipeline_run_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    pipeline_run: "PipelineRunSchema" = Relationship(
        back_populates="model_versions_pipeline_runs_links"
    )

    @classmethod
    def from_request(
        cls,
        model_version_pipeline_run_request: ModelVersionPipelineRunRequest,
    ) -> "ModelVersionPipelineRunSchema":
        """Convert an `ModelVersionPipelineRunRequest` to an `ModelVersionPipelineRunSchema`.

        Args:
            model_version_pipeline_run_request: The request link to convert.

        Returns:
            The converted schema.
        """
        return cls(
            workspace_id=model_version_pipeline_run_request.workspace,
            user_id=model_version_pipeline_run_request.user,
            model_id=model_version_pipeline_run_request.model,
            model_version_id=model_version_pipeline_run_request.model_version,
            pipeline_run_id=model_version_pipeline_run_request.pipeline_run,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> ModelVersionPipelineRunResponse:
        """Convert an `ModelVersionPipelineRunSchema` to an `ModelVersionPipelineRunResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `ModelVersionPipelineRunResponse`.
        """
        return ModelVersionPipelineRunResponse(
            id=self.id,
            body=ModelVersionPipelineRunResponseBody(
                created=self.created,
                updated=self.updated,
                model=self.model_id,
                model_version=self.model_version_id,
                pipeline_run=self.pipeline_run.to_model(),
            ),
            metadata=BaseResponseMetadata() if include_metadata else None,
        )
from_request(model_version_pipeline_run_request) classmethod

Convert an ModelVersionPipelineRunRequest to an ModelVersionPipelineRunSchema.

Parameters:

Name Type Description Default
model_version_pipeline_run_request ModelVersionPipelineRunRequest

The request link to convert.

required

Returns:

Type Description
ModelVersionPipelineRunSchema

The converted schema.

Source code in zenml/zen_stores/schemas/model_schemas.py
@classmethod
def from_request(
    cls,
    model_version_pipeline_run_request: ModelVersionPipelineRunRequest,
) -> "ModelVersionPipelineRunSchema":
    """Convert an `ModelVersionPipelineRunRequest` to an `ModelVersionPipelineRunSchema`.

    Args:
        model_version_pipeline_run_request: The request link to convert.

    Returns:
        The converted schema.
    """
    return cls(
        workspace_id=model_version_pipeline_run_request.workspace,
        user_id=model_version_pipeline_run_request.user,
        model_id=model_version_pipeline_run_request.model,
        model_version_id=model_version_pipeline_run_request.model_version,
        pipeline_run_id=model_version_pipeline_run_request.pipeline_run,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an ModelVersionPipelineRunSchema to an ModelVersionPipelineRunResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ModelVersionPipelineRunResponse

The created ModelVersionPipelineRunResponse.

Source code in zenml/zen_stores/schemas/model_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> ModelVersionPipelineRunResponse:
    """Convert an `ModelVersionPipelineRunSchema` to an `ModelVersionPipelineRunResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `ModelVersionPipelineRunResponse`.
    """
    return ModelVersionPipelineRunResponse(
        id=self.id,
        body=ModelVersionPipelineRunResponseBody(
            created=self.created,
            updated=self.updated,
            model=self.model_id,
            model_version=self.model_version_id,
            pipeline_run=self.pipeline_run.to_model(),
        ),
        metadata=BaseResponseMetadata() if include_metadata else None,
    )
ModelVersionSchema (NamedSchema) pydantic-model

SQL Model for model version.

Source code in zenml/zen_stores/schemas/model_schemas.py
class ModelVersionSchema(NamedSchema, table=True):
    """SQL Model for model version."""

    __tablename__ = "model_version"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(
        back_populates="model_versions"
    )

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(
        back_populates="model_versions"
    )

    model_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=ModelSchema.__tablename__,
        source_column="model_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    model: "ModelSchema" = Relationship(back_populates="model_versions")
    artifact_links: List["ModelVersionArtifactSchema"] = Relationship(
        back_populates="model_version",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    pipeline_run_links: List["ModelVersionPipelineRunSchema"] = Relationship(
        back_populates="model_version",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    tags: List["TagResourceSchema"] = Relationship(
        back_populates="model_version",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(TagResourceSchema.resource_type=='{TaggableResourceTypes.MODEL_VERSION.value}', foreign(TagResourceSchema.resource_id)==ModelVersionSchema.id)",
            cascade="delete",
            overlaps="tags",
        ),
    )

    number: int = Field(sa_column=Column(INTEGER, nullable=False))
    description: str = Field(sa_column=Column(TEXT, nullable=True))
    stage: str = Field(sa_column=Column(TEXT, nullable=True))

    run_metadata: List["RunMetadataSchema"] = Relationship(
        back_populates="model_version",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.MODEL_VERSION.value}', foreign(RunMetadataSchema.resource_id)==ModelVersionSchema.id)",
            cascade="delete",
            overlaps="run_metadata",
        ),
    )

    @classmethod
    def from_request(
        cls, model_version_request: ModelVersionRequest
    ) -> "ModelVersionSchema":
        """Convert an `ModelVersionRequest` to an `ModelVersionSchema`.

        Args:
            model_version_request: The request model version to convert.

        Returns:
            The converted schema.
        """
        return cls(
            workspace_id=model_version_request.workspace,
            user_id=model_version_request.user,
            model_id=model_version_request.model,
            name=model_version_request.name,
            number=model_version_request.number,
            description=model_version_request.description,
            stage=model_version_request.stage,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> ModelVersionResponse:
        """Convert an `ModelVersionSchema` to an `ModelVersionResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `ModelVersionResponse`.
        """
        # Construct {name: {version: id}} dicts for all linked artifacts
        model_artifact_ids: Dict[str, Dict[str, UUID]] = {}
        deployment_artifact_ids: Dict[str, Dict[str, UUID]] = {}
        data_artifact_ids: Dict[str, Dict[str, UUID]] = {}
        for artifact_link in self.artifact_links:
            if not artifact_link.artifact_version:
                continue
            artifact_name = artifact_link.artifact_version.artifact.name
            artifact_version = str(artifact_link.artifact_version.version)
            artifact_version_id = artifact_link.artifact_version.id
            if artifact_link.is_model_artifact:
                model_artifact_ids.setdefault(artifact_name, {}).update(
                    {str(artifact_version): artifact_version_id}
                )
            elif artifact_link.is_deployment_artifact:
                deployment_artifact_ids.setdefault(artifact_name, {}).update(
                    {str(artifact_version): artifact_version_id}
                )
            else:
                data_artifact_ids.setdefault(artifact_name, {}).update(
                    {str(artifact_version): artifact_version_id}
                )

        # Construct {name: id} dict for all linked pipeline runs
        pipeline_run_ids: Dict[str, UUID] = {}
        for pipeline_run_link in self.pipeline_run_links:
            if not pipeline_run_link.pipeline_run:
                continue
            pipeline_run = pipeline_run_link.pipeline_run
            pipeline_run_ids[pipeline_run.name] = pipeline_run.id

        metadata = None

        if include_metadata:
            metadata = ModelVersionResponseMetadata(
                workspace=self.workspace.to_model(),
                description=self.description,
                run_metadata={
                    rm.key: rm.to_model(include_metadata=True)
                    for rm in self.run_metadata
                },
            )

        body = ModelVersionResponseBody(
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
            stage=self.stage,
            number=self.number,
            model=self.model.to_model(),
            model_artifact_ids=model_artifact_ids,
            data_artifact_ids=data_artifact_ids,
            deployment_artifact_ids=deployment_artifact_ids,
            pipeline_run_ids=pipeline_run_ids,
            tags=[t.tag.to_model() for t in self.tags],
        )

        return ModelVersionResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )

    def update(
        self,
        target_stage: Optional[str] = None,
        target_name: Optional[str] = None,
        target_description: Optional[str] = None,
    ) -> "ModelVersionSchema":
        """Updates a `ModelVersionSchema` to a target stage.

        Args:
            target_stage: The stage to be updated.
            target_name: The version name to be updated.
            target_description: The version description to be updated.

        Returns:
            The updated `ModelVersionSchema`.
        """
        if target_stage is not None:
            self.stage = target_stage
        if target_name is not None:
            self.name = target_name
        if target_description is not None:
            self.description = target_description
        self.updated = datetime.utcnow()
        return self
from_request(model_version_request) classmethod

Convert an ModelVersionRequest to an ModelVersionSchema.

Parameters:

Name Type Description Default
model_version_request ModelVersionRequest

The request model version to convert.

required

Returns:

Type Description
ModelVersionSchema

The converted schema.

Source code in zenml/zen_stores/schemas/model_schemas.py
@classmethod
def from_request(
    cls, model_version_request: ModelVersionRequest
) -> "ModelVersionSchema":
    """Convert an `ModelVersionRequest` to an `ModelVersionSchema`.

    Args:
        model_version_request: The request model version to convert.

    Returns:
        The converted schema.
    """
    return cls(
        workspace_id=model_version_request.workspace,
        user_id=model_version_request.user,
        model_id=model_version_request.model,
        name=model_version_request.name,
        number=model_version_request.number,
        description=model_version_request.description,
        stage=model_version_request.stage,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an ModelVersionSchema to an ModelVersionResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ModelVersionResponse

The created ModelVersionResponse.

Source code in zenml/zen_stores/schemas/model_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> ModelVersionResponse:
    """Convert an `ModelVersionSchema` to an `ModelVersionResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `ModelVersionResponse`.
    """
    # Construct {name: {version: id}} dicts for all linked artifacts
    model_artifact_ids: Dict[str, Dict[str, UUID]] = {}
    deployment_artifact_ids: Dict[str, Dict[str, UUID]] = {}
    data_artifact_ids: Dict[str, Dict[str, UUID]] = {}
    for artifact_link in self.artifact_links:
        if not artifact_link.artifact_version:
            continue
        artifact_name = artifact_link.artifact_version.artifact.name
        artifact_version = str(artifact_link.artifact_version.version)
        artifact_version_id = artifact_link.artifact_version.id
        if artifact_link.is_model_artifact:
            model_artifact_ids.setdefault(artifact_name, {}).update(
                {str(artifact_version): artifact_version_id}
            )
        elif artifact_link.is_deployment_artifact:
            deployment_artifact_ids.setdefault(artifact_name, {}).update(
                {str(artifact_version): artifact_version_id}
            )
        else:
            data_artifact_ids.setdefault(artifact_name, {}).update(
                {str(artifact_version): artifact_version_id}
            )

    # Construct {name: id} dict for all linked pipeline runs
    pipeline_run_ids: Dict[str, UUID] = {}
    for pipeline_run_link in self.pipeline_run_links:
        if not pipeline_run_link.pipeline_run:
            continue
        pipeline_run = pipeline_run_link.pipeline_run
        pipeline_run_ids[pipeline_run.name] = pipeline_run.id

    metadata = None

    if include_metadata:
        metadata = ModelVersionResponseMetadata(
            workspace=self.workspace.to_model(),
            description=self.description,
            run_metadata={
                rm.key: rm.to_model(include_metadata=True)
                for rm in self.run_metadata
            },
        )

    body = ModelVersionResponseBody(
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
        stage=self.stage,
        number=self.number,
        model=self.model.to_model(),
        model_artifact_ids=model_artifact_ids,
        data_artifact_ids=data_artifact_ids,
        deployment_artifact_ids=deployment_artifact_ids,
        pipeline_run_ids=pipeline_run_ids,
        tags=[t.tag.to_model() for t in self.tags],
    )

    return ModelVersionResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, target_stage=None, target_name=None, target_description=None)

Updates a ModelVersionSchema to a target stage.

Parameters:

Name Type Description Default
target_stage Optional[str]

The stage to be updated.

None
target_name Optional[str]

The version name to be updated.

None
target_description Optional[str]

The version description to be updated.

None

Returns:

Type Description
ModelVersionSchema

The updated ModelVersionSchema.

Source code in zenml/zen_stores/schemas/model_schemas.py
def update(
    self,
    target_stage: Optional[str] = None,
    target_name: Optional[str] = None,
    target_description: Optional[str] = None,
) -> "ModelVersionSchema":
    """Updates a `ModelVersionSchema` to a target stage.

    Args:
        target_stage: The stage to be updated.
        target_name: The version name to be updated.
        target_description: The version description to be updated.

    Returns:
        The updated `ModelVersionSchema`.
    """
    if target_stage is not None:
        self.stage = target_stage
    if target_name is not None:
        self.name = target_name
    if target_description is not None:
        self.description = target_description
    self.updated = datetime.utcnow()
    return self

pipeline_build_schemas

SQLModel implementation of pipeline build tables.

PipelineBuildSchema (BaseSchema) pydantic-model

SQL Model for pipeline builds.

Source code in zenml/zen_stores/schemas/pipeline_build_schemas.py
class PipelineBuildSchema(BaseSchema, table=True):
    """SQL Model for pipeline builds."""

    __tablename__ = "pipeline_build"

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="builds")

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="builds")

    stack_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=StackSchema.__tablename__,
        source_column="stack_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    stack: Optional["StackSchema"] = Relationship(back_populates="builds")

    pipeline_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=PipelineSchema.__tablename__,
        source_column="pipeline_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    pipeline: Optional["PipelineSchema"] = Relationship(
        back_populates="builds"
    )

    template_deployment_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target="pipeline_deployment",
        source_column="template_deployment_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    images: str = Field(
        sa_column=Column(
            String(length=MEDIUMTEXT_MAX_LENGTH).with_variant(
                MEDIUMTEXT, "mysql"
            ),
            nullable=False,
        )
    )

    is_local: bool
    contains_code: bool

    zenml_version: Optional[str]
    python_version: Optional[str]
    checksum: Optional[str]

    @classmethod
    def from_request(
        cls, request: PipelineBuildRequest
    ) -> "PipelineBuildSchema":
        """Convert a `PipelineBuildRequest` to a `PipelineBuildSchema`.

        Args:
            request: The request to convert.

        Returns:
            The created `PipelineBuildSchema`.
        """
        return cls(
            stack_id=request.stack,
            workspace_id=request.workspace,
            user_id=request.user,
            pipeline_id=request.pipeline,
            images=json.dumps(request.images, default=pydantic_encoder),
            is_local=request.is_local,
            contains_code=request.contains_code,
            zenml_version=request.zenml_version,
            python_version=request.python_version,
            checksum=request.checksum,
            template_deployment_id=request.template_deployment_id,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> PipelineBuildResponse:
        """Convert a `PipelineBuildSchema` to a `PipelineBuildResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `PipelineBuildResponse`.
        """
        body = PipelineBuildResponseBody(
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = PipelineBuildResponseMetadata(
                workspace=self.workspace.to_model(),
                pipeline=self.pipeline.to_model() if self.pipeline else None,
                stack=self.stack.to_model() if self.stack else None,
                images=json.loads(self.images),
                zenml_version=self.zenml_version,
                python_version=self.python_version,
                checksum=self.checksum,
                is_local=self.is_local,
                contains_code=self.contains_code,
                template_deployment_id=self.template_deployment_id,
            )
        return PipelineBuildResponse(
            id=self.id,
            body=body,
            metadata=metadata,
        )
from_request(request) classmethod

Convert a PipelineBuildRequest to a PipelineBuildSchema.

Parameters:

Name Type Description Default
request PipelineBuildRequest

The request to convert.

required

Returns:

Type Description
PipelineBuildSchema

The created PipelineBuildSchema.

Source code in zenml/zen_stores/schemas/pipeline_build_schemas.py
@classmethod
def from_request(
    cls, request: PipelineBuildRequest
) -> "PipelineBuildSchema":
    """Convert a `PipelineBuildRequest` to a `PipelineBuildSchema`.

    Args:
        request: The request to convert.

    Returns:
        The created `PipelineBuildSchema`.
    """
    return cls(
        stack_id=request.stack,
        workspace_id=request.workspace,
        user_id=request.user,
        pipeline_id=request.pipeline,
        images=json.dumps(request.images, default=pydantic_encoder),
        is_local=request.is_local,
        contains_code=request.contains_code,
        zenml_version=request.zenml_version,
        python_version=request.python_version,
        checksum=request.checksum,
        template_deployment_id=request.template_deployment_id,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a PipelineBuildSchema to a PipelineBuildResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
PipelineBuildResponse

The created PipelineBuildResponse.

Source code in zenml/zen_stores/schemas/pipeline_build_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> PipelineBuildResponse:
    """Convert a `PipelineBuildSchema` to a `PipelineBuildResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `PipelineBuildResponse`.
    """
    body = PipelineBuildResponseBody(
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = PipelineBuildResponseMetadata(
            workspace=self.workspace.to_model(),
            pipeline=self.pipeline.to_model() if self.pipeline else None,
            stack=self.stack.to_model() if self.stack else None,
            images=json.loads(self.images),
            zenml_version=self.zenml_version,
            python_version=self.python_version,
            checksum=self.checksum,
            is_local=self.is_local,
            contains_code=self.contains_code,
            template_deployment_id=self.template_deployment_id,
        )
    return PipelineBuildResponse(
        id=self.id,
        body=body,
        metadata=metadata,
    )

pipeline_deployment_schemas

SQLModel implementation of pipeline deployment tables.

PipelineDeploymentSchema (BaseSchema) pydantic-model

SQL Model for pipeline deployments.

Source code in zenml/zen_stores/schemas/pipeline_deployment_schemas.py
class PipelineDeploymentSchema(BaseSchema, table=True):
    """SQL Model for pipeline deployments."""

    __tablename__ = "pipeline_deployment"

    # Fields
    pipeline_configuration: str = Field(
        sa_column=Column(
            String(length=MEDIUMTEXT_MAX_LENGTH).with_variant(
                MEDIUMTEXT, "mysql"
            ),
            nullable=False,
        )
    )
    step_configurations: str = Field(
        sa_column=Column(
            String(length=MEDIUMTEXT_MAX_LENGTH).with_variant(
                MEDIUMTEXT, "mysql"
            ),
            nullable=False,
        )
    )
    client_environment: str = Field(sa_column=Column(TEXT, nullable=False))
    run_name_template: str = Field(nullable=False)
    client_version: str = Field(nullable=True)
    server_version: str = Field(nullable=True)

    # Foreign keys
    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    stack_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=StackSchema.__tablename__,
        source_column="stack_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    pipeline_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=PipelineSchema.__tablename__,
        source_column="pipeline_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    schedule_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=ScheduleSchema.__tablename__,
        source_column="schedule_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    build_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=PipelineBuildSchema.__tablename__,
        source_column="build_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    code_reference_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=CodeReferenceSchema.__tablename__,
        source_column="code_reference_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )

    # SQLModel Relationships
    user: Optional["UserSchema"] = Relationship()
    workspace: "WorkspaceSchema" = Relationship()
    stack: "StackSchema" = Relationship()
    pipeline: "PipelineSchema" = Relationship()
    schedule: Optional["ScheduleSchema"] = Relationship()
    build: Optional["PipelineBuildSchema"] = Relationship(
        sa_relationship_kwargs={
            "foreign_keys": "[PipelineDeploymentSchema.build_id]"
        }
    )
    code_reference: Optional["CodeReferenceSchema"] = Relationship()

    pipeline_runs: List["PipelineRunSchema"] = Relationship(
        sa_relationship_kwargs={"cascade": "delete"}
    )
    step_runs: List["StepRunSchema"] = Relationship(
        sa_relationship_kwargs={"cascade": "delete"}
    )

    @classmethod
    def from_request(
        cls,
        request: PipelineDeploymentRequest,
        code_reference_id: Optional[UUID],
    ) -> "PipelineDeploymentSchema":
        """Convert a `PipelineDeploymentRequest` to a `PipelineDeploymentSchema`.

        Args:
            request: The request to convert.
            code_reference_id: Optional ID of the code reference for the
                deployment.

        Returns:
            The created `PipelineDeploymentSchema`.
        """
        return cls(
            stack_id=request.stack,
            workspace_id=request.workspace,
            pipeline_id=request.pipeline,
            build_id=request.build,
            user_id=request.user,
            schedule_id=request.schedule,
            code_reference_id=code_reference_id,
            run_name_template=request.run_name_template,
            pipeline_configuration=request.pipeline_configuration.json(),
            step_configurations=json.dumps(
                request.step_configurations,
                sort_keys=False,
                default=pydantic_encoder,
            ),
            client_environment=json.dumps(request.client_environment),
            client_version=request.client_version,
            server_version=request.server_version,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> PipelineDeploymentResponse:
        """Convert a `PipelineDeploymentSchema` to a `PipelineDeploymentResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `PipelineDeploymentResponse`.
        """
        pipeline_configuration = PipelineConfiguration.parse_raw(
            self.pipeline_configuration
        )
        step_configurations = json.loads(self.step_configurations)
        for s, c in step_configurations.items():
            step_configurations[s] = Step.parse_obj(c)

        body = PipelineDeploymentResponseBody(
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = PipelineDeploymentResponseMetadata(
                workspace=self.workspace.to_model(),
                run_name_template=self.run_name_template,
                pipeline_configuration=pipeline_configuration,
                step_configurations=step_configurations,
                client_environment=json.loads(self.client_environment),
                client_version=self.client_version,
                server_version=self.server_version,
                pipeline=self.pipeline.to_model() if self.pipeline else None,
                stack=self.stack.to_model() if self.stack else None,
                build=self.build.to_model() if self.build else None,
                schedule=self.schedule.to_model() if self.schedule else None,
                code_reference=self.code_reference.to_model()
                if self.code_reference
                else None,
            )
        return PipelineDeploymentResponse(
            id=self.id,
            body=body,
            metadata=metadata,
        )
from_request(request, code_reference_id) classmethod

Convert a PipelineDeploymentRequest to a PipelineDeploymentSchema.

Parameters:

Name Type Description Default
request PipelineDeploymentRequest

The request to convert.

required
code_reference_id Optional[uuid.UUID]

Optional ID of the code reference for the deployment.

required

Returns:

Type Description
PipelineDeploymentSchema

The created PipelineDeploymentSchema.

Source code in zenml/zen_stores/schemas/pipeline_deployment_schemas.py
@classmethod
def from_request(
    cls,
    request: PipelineDeploymentRequest,
    code_reference_id: Optional[UUID],
) -> "PipelineDeploymentSchema":
    """Convert a `PipelineDeploymentRequest` to a `PipelineDeploymentSchema`.

    Args:
        request: The request to convert.
        code_reference_id: Optional ID of the code reference for the
            deployment.

    Returns:
        The created `PipelineDeploymentSchema`.
    """
    return cls(
        stack_id=request.stack,
        workspace_id=request.workspace,
        pipeline_id=request.pipeline,
        build_id=request.build,
        user_id=request.user,
        schedule_id=request.schedule,
        code_reference_id=code_reference_id,
        run_name_template=request.run_name_template,
        pipeline_configuration=request.pipeline_configuration.json(),
        step_configurations=json.dumps(
            request.step_configurations,
            sort_keys=False,
            default=pydantic_encoder,
        ),
        client_environment=json.dumps(request.client_environment),
        client_version=request.client_version,
        server_version=request.server_version,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a PipelineDeploymentSchema to a PipelineDeploymentResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
PipelineDeploymentResponse

The created PipelineDeploymentResponse.

Source code in zenml/zen_stores/schemas/pipeline_deployment_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> PipelineDeploymentResponse:
    """Convert a `PipelineDeploymentSchema` to a `PipelineDeploymentResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `PipelineDeploymentResponse`.
    """
    pipeline_configuration = PipelineConfiguration.parse_raw(
        self.pipeline_configuration
    )
    step_configurations = json.loads(self.step_configurations)
    for s, c in step_configurations.items():
        step_configurations[s] = Step.parse_obj(c)

    body = PipelineDeploymentResponseBody(
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = PipelineDeploymentResponseMetadata(
            workspace=self.workspace.to_model(),
            run_name_template=self.run_name_template,
            pipeline_configuration=pipeline_configuration,
            step_configurations=step_configurations,
            client_environment=json.loads(self.client_environment),
            client_version=self.client_version,
            server_version=self.server_version,
            pipeline=self.pipeline.to_model() if self.pipeline else None,
            stack=self.stack.to_model() if self.stack else None,
            build=self.build.to_model() if self.build else None,
            schedule=self.schedule.to_model() if self.schedule else None,
            code_reference=self.code_reference.to_model()
            if self.code_reference
            else None,
        )
    return PipelineDeploymentResponse(
        id=self.id,
        body=body,
        metadata=metadata,
    )

pipeline_run_schemas

SQLModel implementation of pipeline run tables.

PipelineRunSchema (NamedSchema) pydantic-model

SQL Model for pipeline runs.

Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
class PipelineRunSchema(NamedSchema, table=True):
    """SQL Model for pipeline runs."""

    __tablename__ = "pipeline_run"
    __table_args__ = (
        UniqueConstraint(
            "deployment_id",
            "orchestrator_run_id",
            name="unique_orchestrator_run_id_for_deployment_id",
        ),
    )

    # Fields
    orchestrator_run_id: Optional[str] = Field(nullable=True)
    start_time: Optional[datetime] = Field(nullable=True)
    end_time: Optional[datetime] = Field(nullable=True, default=None)
    status: ExecutionStatus = Field(nullable=False)
    orchestrator_environment: Optional[str] = Field(
        sa_column=Column(TEXT, nullable=True)
    )

    # Foreign keys
    deployment_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=PipelineDeploymentSchema.__tablename__,
        source_column="deployment_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=True,
    )
    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    pipeline_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=PipelineSchema.__tablename__,
        source_column="pipeline_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )

    # Relationships
    deployment: Optional["PipelineDeploymentSchema"] = Relationship(
        back_populates="pipeline_runs"
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="runs")
    user: Optional["UserSchema"] = Relationship(back_populates="runs")
    run_metadata: List["RunMetadataSchema"] = Relationship(
        back_populates="pipeline_run",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.PIPELINE_RUN.value}', foreign(RunMetadataSchema.resource_id)==PipelineRunSchema.id)",
            cascade="delete",
            overlaps="run_metadata",
        ),
    )
    logs: Optional["LogsSchema"] = Relationship(
        back_populates="pipeline_run",
        sa_relationship_kwargs={"cascade": "delete", "uselist": False},
    )
    model_versions_pipeline_runs_links: List[
        "ModelVersionPipelineRunSchema"
    ] = Relationship(
        back_populates="pipeline_run",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    step_runs: List["StepRunSchema"] = Relationship(
        sa_relationship_kwargs={"cascade": "delete"},
    )

    # Temporary fields and foreign keys to be deprecated
    pipeline_configuration: Optional[str] = Field(
        sa_column=Column(TEXT, nullable=True)
    )
    client_environment: Optional[str] = Field(
        sa_column=Column(TEXT, nullable=True)
    )

    stack_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=StackSchema.__tablename__,
        source_column="stack_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    build_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=PipelineBuildSchema.__tablename__,
        source_column="build_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    schedule_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=ScheduleSchema.__tablename__,
        source_column="schedule_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    trigger_execution_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=TriggerExecutionSchema.__tablename__,
        source_column="trigger_execution_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )

    stack: Optional["StackSchema"] = Relationship()
    build: Optional["PipelineBuildSchema"] = Relationship()
    schedule: Optional["ScheduleSchema"] = Relationship()
    pipeline: Optional["PipelineSchema"] = Relationship(back_populates="runs")
    trigger_execution: Optional["TriggerExecutionSchema"] = Relationship()

    @classmethod
    def from_request(
        cls, request: "PipelineRunRequest"
    ) -> "PipelineRunSchema":
        """Convert a `PipelineRunRequest` to a `PipelineRunSchema`.

        Args:
            request: The request to convert.

        Returns:
            The created `PipelineRunSchema`.
        """
        orchestrator_environment = json.dumps(request.orchestrator_environment)

        return cls(
            workspace_id=request.workspace,
            user_id=request.user,
            name=request.name,
            orchestrator_run_id=request.orchestrator_run_id,
            orchestrator_environment=orchestrator_environment,
            start_time=request.start_time,
            status=request.status,
            pipeline_id=request.pipeline,
            deployment_id=request.deployment,
            trigger_execution_id=request.trigger_execution_id,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "PipelineRunResponse":
        """Convert a `PipelineRunSchema` to a `PipelineRunResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `PipelineRunResponse`.

        Raises:
            RuntimeError: if the model creation fails.
        """
        orchestrator_environment = (
            json.loads(self.orchestrator_environment)
            if self.orchestrator_environment
            else {}
        )

        run_metadata = {
            metadata_schema.key: metadata_schema.to_model()
            for metadata_schema in self.run_metadata
        }

        if self.deployment is not None:
            deployment = self.deployment.to_model()

            config = deployment.pipeline_configuration
            client_environment = deployment.client_environment

            stack = deployment.stack
            pipeline = deployment.pipeline
            build = deployment.build
            schedule = deployment.schedule
            code_reference = deployment.code_reference

        elif self.pipeline_configuration is not None:
            config = PipelineConfiguration.parse_raw(
                self.pipeline_configuration
            )
            client_environment = (
                json.loads(self.client_environment)
                if self.client_environment
                else {}
            )

            stack = self.stack.to_model() if self.stack else None
            pipeline = self.pipeline.to_model() if self.pipeline else None
            build = self.build.to_model() if self.build else None
            schedule = self.schedule.to_model() if self.schedule else None
            code_reference = None

        else:
            raise RuntimeError(
                "Pipeline run model creation has failed. Each pipeline run "
                "entry should either have a deployment_id or "
                "pipeline_configuration."
            )

        body = PipelineRunResponseBody(
            user=self.user.to_model() if self.user else None,
            status=self.status,
            stack=stack,
            pipeline=pipeline,
            build=build,
            schedule=schedule,
            code_reference=code_reference,
            trigger_execution=self.trigger_execution.to_model()
            if self.trigger_execution
            else None,
            created=self.created,
            updated=self.updated,
            deployment_id=self.deployment_id,
        )
        metadata = None
        if include_metadata:
            steps = {step.name: step.to_model() for step in self.step_runs}

            metadata = PipelineRunResponseMetadata(
                workspace=self.workspace.to_model(),
                run_metadata=run_metadata,
                config=config,
                steps=steps,
                start_time=self.start_time,
                end_time=self.end_time,
                client_environment=client_environment,
                orchestrator_environment=orchestrator_environment,
                orchestrator_run_id=self.orchestrator_run_id,
            )
        return PipelineRunResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )

    def update(self, run_update: "PipelineRunUpdate") -> "PipelineRunSchema":
        """Update a `PipelineRunSchema` with a `PipelineRunUpdate`.

        Args:
            run_update: The `PipelineRunUpdate` to update with.

        Returns:
            The updated `PipelineRunSchema`.
        """
        if run_update.status:
            self.status = run_update.status
            self.end_time = run_update.end_time

        self.updated = datetime.utcnow()
        return self

    def update_placeholder(
        self, request: "PipelineRunRequest"
    ) -> "PipelineRunSchema":
        """Update a placeholder run.

        Args:
            request: The pipeline run request which should replace the
                placeholder.

        Raises:
            RuntimeError: If the DB entry does not represent a placeholder run.
            ValueError: If the run request does not match the deployment or
                pipeline ID of the placeholder run.

        Returns:
            The updated `PipelineRunSchema`.
        """
        if (
            self.orchestrator_run_id
            or self.status != ExecutionStatus.INITIALIZING
        ):
            raise RuntimeError(
                f"Unable to replace pipeline run {self.id} which is not a "
                "placeholder run."
            )

        if (
            self.deployment_id != request.deployment
            or self.pipeline_id != request.pipeline
        ):
            raise ValueError(
                "Deployment or orchestrator run ID of placeholder run do not "
                "match the IDs of the run request."
            )

        orchestrator_environment = json.dumps(request.orchestrator_environment)

        self.orchestrator_run_id = request.orchestrator_run_id
        self.orchestrator_environment = orchestrator_environment
        self.status = request.status

        self.updated = datetime.utcnow()

        return self
from_request(request) classmethod

Convert a PipelineRunRequest to a PipelineRunSchema.

Parameters:

Name Type Description Default
request PipelineRunRequest

The request to convert.

required

Returns:

Type Description
PipelineRunSchema

The created PipelineRunSchema.

Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
@classmethod
def from_request(
    cls, request: "PipelineRunRequest"
) -> "PipelineRunSchema":
    """Convert a `PipelineRunRequest` to a `PipelineRunSchema`.

    Args:
        request: The request to convert.

    Returns:
        The created `PipelineRunSchema`.
    """
    orchestrator_environment = json.dumps(request.orchestrator_environment)

    return cls(
        workspace_id=request.workspace,
        user_id=request.user,
        name=request.name,
        orchestrator_run_id=request.orchestrator_run_id,
        orchestrator_environment=orchestrator_environment,
        start_time=request.start_time,
        status=request.status,
        pipeline_id=request.pipeline,
        deployment_id=request.deployment,
        trigger_execution_id=request.trigger_execution_id,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a PipelineRunSchema to a PipelineRunResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
PipelineRunResponse

The created PipelineRunResponse.

Exceptions:

Type Description
RuntimeError

if the model creation fails.

Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "PipelineRunResponse":
    """Convert a `PipelineRunSchema` to a `PipelineRunResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `PipelineRunResponse`.

    Raises:
        RuntimeError: if the model creation fails.
    """
    orchestrator_environment = (
        json.loads(self.orchestrator_environment)
        if self.orchestrator_environment
        else {}
    )

    run_metadata = {
        metadata_schema.key: metadata_schema.to_model()
        for metadata_schema in self.run_metadata
    }

    if self.deployment is not None:
        deployment = self.deployment.to_model()

        config = deployment.pipeline_configuration
        client_environment = deployment.client_environment

        stack = deployment.stack
        pipeline = deployment.pipeline
        build = deployment.build
        schedule = deployment.schedule
        code_reference = deployment.code_reference

    elif self.pipeline_configuration is not None:
        config = PipelineConfiguration.parse_raw(
            self.pipeline_configuration
        )
        client_environment = (
            json.loads(self.client_environment)
            if self.client_environment
            else {}
        )

        stack = self.stack.to_model() if self.stack else None
        pipeline = self.pipeline.to_model() if self.pipeline else None
        build = self.build.to_model() if self.build else None
        schedule = self.schedule.to_model() if self.schedule else None
        code_reference = None

    else:
        raise RuntimeError(
            "Pipeline run model creation has failed. Each pipeline run "
            "entry should either have a deployment_id or "
            "pipeline_configuration."
        )

    body = PipelineRunResponseBody(
        user=self.user.to_model() if self.user else None,
        status=self.status,
        stack=stack,
        pipeline=pipeline,
        build=build,
        schedule=schedule,
        code_reference=code_reference,
        trigger_execution=self.trigger_execution.to_model()
        if self.trigger_execution
        else None,
        created=self.created,
        updated=self.updated,
        deployment_id=self.deployment_id,
    )
    metadata = None
    if include_metadata:
        steps = {step.name: step.to_model() for step in self.step_runs}

        metadata = PipelineRunResponseMetadata(
            workspace=self.workspace.to_model(),
            run_metadata=run_metadata,
            config=config,
            steps=steps,
            start_time=self.start_time,
            end_time=self.end_time,
            client_environment=client_environment,
            orchestrator_environment=orchestrator_environment,
            orchestrator_run_id=self.orchestrator_run_id,
        )
    return PipelineRunResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, run_update)

Update a PipelineRunSchema with a PipelineRunUpdate.

Parameters:

Name Type Description Default
run_update PipelineRunUpdate

The PipelineRunUpdate to update with.

required

Returns:

Type Description
PipelineRunSchema

The updated PipelineRunSchema.

Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
def update(self, run_update: "PipelineRunUpdate") -> "PipelineRunSchema":
    """Update a `PipelineRunSchema` with a `PipelineRunUpdate`.

    Args:
        run_update: The `PipelineRunUpdate` to update with.

    Returns:
        The updated `PipelineRunSchema`.
    """
    if run_update.status:
        self.status = run_update.status
        self.end_time = run_update.end_time

    self.updated = datetime.utcnow()
    return self
update_placeholder(self, request)

Update a placeholder run.

Parameters:

Name Type Description Default
request PipelineRunRequest

The pipeline run request which should replace the placeholder.

required

Exceptions:

Type Description
RuntimeError

If the DB entry does not represent a placeholder run.

ValueError

If the run request does not match the deployment or pipeline ID of the placeholder run.

Returns:

Type Description
PipelineRunSchema

The updated PipelineRunSchema.

Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
def update_placeholder(
    self, request: "PipelineRunRequest"
) -> "PipelineRunSchema":
    """Update a placeholder run.

    Args:
        request: The pipeline run request which should replace the
            placeholder.

    Raises:
        RuntimeError: If the DB entry does not represent a placeholder run.
        ValueError: If the run request does not match the deployment or
            pipeline ID of the placeholder run.

    Returns:
        The updated `PipelineRunSchema`.
    """
    if (
        self.orchestrator_run_id
        or self.status != ExecutionStatus.INITIALIZING
    ):
        raise RuntimeError(
            f"Unable to replace pipeline run {self.id} which is not a "
            "placeholder run."
        )

    if (
        self.deployment_id != request.deployment
        or self.pipeline_id != request.pipeline
    ):
        raise ValueError(
            "Deployment or orchestrator run ID of placeholder run do not "
            "match the IDs of the run request."
        )

    orchestrator_environment = json.dumps(request.orchestrator_environment)

    self.orchestrator_run_id = request.orchestrator_run_id
    self.orchestrator_environment = orchestrator_environment
    self.status = request.status

    self.updated = datetime.utcnow()

    return self

pipeline_schemas

SQL Model Implementations for Pipelines and Pipeline Runs.

PipelineSchema (NamedSchema) pydantic-model

SQL Model for pipelines.

Source code in zenml/zen_stores/schemas/pipeline_schemas.py
class PipelineSchema(NamedSchema, table=True):
    """SQL Model for pipelines."""

    __tablename__ = "pipeline"

    # Fields
    version: str
    version_hash: str
    docstring: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
    spec: str = Field(
        sa_column=Column(
            String(length=MEDIUMTEXT_MAX_LENGTH).with_variant(
                MEDIUMTEXT, "mysql"
            ),
            nullable=False,
        )
    )

    # Foreign keys
    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )

    # Relationships
    user: Optional["UserSchema"] = Relationship(back_populates="pipelines")
    workspace: "WorkspaceSchema" = Relationship(back_populates="pipelines")

    schedules: List["ScheduleSchema"] = Relationship(
        back_populates="pipeline",
    )
    runs: List["PipelineRunSchema"] = Relationship(back_populates="pipeline")
    builds: List["PipelineBuildSchema"] = Relationship(
        back_populates="pipeline"
    )
    deployments: List["PipelineDeploymentSchema"] = Relationship(
        back_populates="pipeline",
    )

    @classmethod
    def from_request(
        cls,
        pipeline_request: "PipelineRequest",
    ) -> "PipelineSchema":
        """Convert a `PipelineRequest` to a `PipelineSchema`.

        Args:
            pipeline_request: The request model to convert.

        Returns:
            The converted schema.
        """
        return cls(
            name=pipeline_request.name,
            version=pipeline_request.version,
            version_hash=pipeline_request.version_hash,
            workspace_id=pipeline_request.workspace,
            user_id=pipeline_request.user,
            docstring=pipeline_request.docstring,
            spec=pipeline_request.spec.json(sort_keys=True),
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        last_x_runs: int = 3,
        **kwargs: Any,
    ) -> "PipelineResponse":
        """Convert a `PipelineSchema` to a `PipelineResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic
            last_x_runs: How many runs to use for the execution status

        Returns:
            The created PipelineResponse.
        """
        body = PipelineResponseBody(
            user=self.user.to_model() if self.user else None,
            status=[run.status for run in self.runs[:last_x_runs]],
            created=self.created,
            updated=self.updated,
            version=self.version,
        )
        metadata = None
        if include_metadata:
            metadata = PipelineResponseMetadata(
                workspace=self.workspace.to_model(),
                version_hash=self.version_hash,
                spec=PipelineSpec.parse_raw(self.spec),
                docstring=self.docstring,
            )

        return PipelineResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )

    def update(self, pipeline_update: "PipelineUpdate") -> "PipelineSchema":
        """Update a `PipelineSchema` with a `PipelineUpdate`.

        Args:
            pipeline_update: The update model.

        Returns:
            The updated `PipelineSchema`.
        """
        self.updated = datetime.utcnow()
        return self
from_request(pipeline_request) classmethod

Convert a PipelineRequest to a PipelineSchema.

Parameters:

Name Type Description Default
pipeline_request PipelineRequest

The request model to convert.

required

Returns:

Type Description
PipelineSchema

The converted schema.

Source code in zenml/zen_stores/schemas/pipeline_schemas.py
@classmethod
def from_request(
    cls,
    pipeline_request: "PipelineRequest",
) -> "PipelineSchema":
    """Convert a `PipelineRequest` to a `PipelineSchema`.

    Args:
        pipeline_request: The request model to convert.

    Returns:
        The converted schema.
    """
    return cls(
        name=pipeline_request.name,
        version=pipeline_request.version,
        version_hash=pipeline_request.version_hash,
        workspace_id=pipeline_request.workspace,
        user_id=pipeline_request.user,
        docstring=pipeline_request.docstring,
        spec=pipeline_request.spec.json(sort_keys=True),
    )
to_model(self, include_metadata=False, include_resources=False, last_x_runs=3, **kwargs)

Convert a PipelineSchema to a PipelineResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}
last_x_runs int

How many runs to use for the execution status

3

Returns:

Type Description
PipelineResponse

The created PipelineResponse.

Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    last_x_runs: int = 3,
    **kwargs: Any,
) -> "PipelineResponse":
    """Convert a `PipelineSchema` to a `PipelineResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic
        last_x_runs: How many runs to use for the execution status

    Returns:
        The created PipelineResponse.
    """
    body = PipelineResponseBody(
        user=self.user.to_model() if self.user else None,
        status=[run.status for run in self.runs[:last_x_runs]],
        created=self.created,
        updated=self.updated,
        version=self.version,
    )
    metadata = None
    if include_metadata:
        metadata = PipelineResponseMetadata(
            workspace=self.workspace.to_model(),
            version_hash=self.version_hash,
            spec=PipelineSpec.parse_raw(self.spec),
            docstring=self.docstring,
        )

    return PipelineResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, pipeline_update)

Update a PipelineSchema with a PipelineUpdate.

Parameters:

Name Type Description Default
pipeline_update PipelineUpdate

The update model.

required

Returns:

Type Description
PipelineSchema

The updated PipelineSchema.

Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def update(self, pipeline_update: "PipelineUpdate") -> "PipelineSchema":
    """Update a `PipelineSchema` with a `PipelineUpdate`.

    Args:
        pipeline_update: The update model.

    Returns:
        The updated `PipelineSchema`.
    """
    self.updated = datetime.utcnow()
    return self

run_metadata_schemas

SQLModel implementation of pipeline run metadata tables.

RunMetadataSchema (BaseSchema) pydantic-model

SQL Model for run metadata.

Source code in zenml/zen_stores/schemas/run_metadata_schemas.py
class RunMetadataSchema(BaseSchema, table=True):
    """SQL Model for run metadata."""

    __tablename__ = "run_metadata"

    resource_id: UUID
    resource_type: str = Field(sa_column=Column(VARCHAR(255), nullable=False))
    pipeline_run: List["PipelineRunSchema"] = Relationship(
        back_populates="run_metadata",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.PIPELINE_RUN.value}', foreign(RunMetadataSchema.resource_id)==PipelineRunSchema.id)",
            overlaps="run_metadata,step_run,artifact_version,model_version",
        ),
    )
    step_run: List["StepRunSchema"] = Relationship(
        back_populates="run_metadata",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.STEP_RUN.value}', foreign(RunMetadataSchema.resource_id)==StepRunSchema.id)",
            overlaps="run_metadata,pipeline_run,artifact_version,model_version",
        ),
    )
    artifact_version: List["ArtifactVersionSchema"] = Relationship(
        back_populates="run_metadata",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.ARTIFACT_VERSION.value}', foreign(RunMetadataSchema.resource_id)==ArtifactVersionSchema.id)",
            overlaps="run_metadata,pipeline_run,step_run,model_version",
        ),
    )
    model_version: List["ModelVersionSchema"] = Relationship(
        back_populates="run_metadata",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.MODEL_VERSION.value}', foreign(RunMetadataSchema.resource_id)==ModelVersionSchema.id)",
            overlaps="run_metadata,pipeline_run,step_run,artifact_version",
        ),
    )
    stack_component_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=StackComponentSchema.__tablename__,
        source_column="stack_component_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    stack_component: Optional["StackComponentSchema"] = Relationship(
        back_populates="run_metadata"
    )

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="run_metadata")

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="run_metadata")

    key: str
    value: str = Field(sa_column=Column(TEXT, nullable=False))
    type: MetadataTypeEnum

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "RunMetadataResponse":
        """Convert a `RunMetadataSchema` to a `RunMetadataResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `RunMetadataResponse`.
        """
        body = RunMetadataResponseBody(
            user=self.user.to_model() if self.user else None,
            key=self.key,
            created=self.created,
            updated=self.updated,
            value=json.loads(self.value),
            type=self.type,
        )
        metadata = None
        if include_metadata:
            metadata = RunMetadataResponseMetadata(
                workspace=self.workspace.to_model(),
                resource_id=self.resource_id,
                resource_type=MetadataResourceTypes(self.resource_type),
                stack_component_id=self.stack_component_id,
            )

        return RunMetadataResponse(
            id=self.id,
            body=body,
            metadata=metadata,
        )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a RunMetadataSchema to a RunMetadataResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
RunMetadataResponse

The created RunMetadataResponse.

Source code in zenml/zen_stores/schemas/run_metadata_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "RunMetadataResponse":
    """Convert a `RunMetadataSchema` to a `RunMetadataResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `RunMetadataResponse`.
    """
    body = RunMetadataResponseBody(
        user=self.user.to_model() if self.user else None,
        key=self.key,
        created=self.created,
        updated=self.updated,
        value=json.loads(self.value),
        type=self.type,
    )
    metadata = None
    if include_metadata:
        metadata = RunMetadataResponseMetadata(
            workspace=self.workspace.to_model(),
            resource_id=self.resource_id,
            resource_type=MetadataResourceTypes(self.resource_type),
            stack_component_id=self.stack_component_id,
        )

    return RunMetadataResponse(
        id=self.id,
        body=body,
        metadata=metadata,
    )

schedule_schema

SQL Model Implementations for Pipeline Schedules.

ScheduleSchema (NamedSchema) pydantic-model

SQL Model for schedules.

Source code in zenml/zen_stores/schemas/schedule_schema.py
class ScheduleSchema(NamedSchema, table=True):
    """SQL Model for schedules."""

    __tablename__ = "schedule"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="schedules")

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="schedules")

    pipeline_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=PipelineSchema.__tablename__,
        source_column="pipeline_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=True,
    )
    pipeline: "PipelineSchema" = Relationship(back_populates="schedules")
    deployment: Optional["PipelineDeploymentSchema"] = Relationship(
        back_populates="schedule"
    )

    orchestrator_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=StackComponentSchema.__tablename__,
        source_column="orchestrator_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    orchestrator: "StackComponentSchema" = Relationship(
        back_populates="schedules"
    )

    active: bool
    cron_expression: Optional[str] = Field(nullable=True)
    start_time: Optional[datetime] = Field(nullable=True)
    end_time: Optional[datetime] = Field(nullable=True)
    interval_second: Optional[float] = Field(nullable=True)
    catchup: bool

    @classmethod
    def from_request(
        cls, schedule_request: ScheduleRequest
    ) -> "ScheduleSchema":
        """Create a `ScheduleSchema` from a `ScheduleRequest`.

        Args:
            schedule_request: The `ScheduleRequest` to create the schema from.

        Returns:
            The created `ScheduleSchema`.
        """
        if schedule_request.interval_second is not None:
            interval_second = schedule_request.interval_second.total_seconds()
        else:
            interval_second = None
        return cls(
            name=schedule_request.name,
            workspace_id=schedule_request.workspace,
            user_id=schedule_request.user,
            pipeline_id=schedule_request.pipeline_id,
            orchestrator_id=schedule_request.orchestrator_id,
            active=schedule_request.active,
            cron_expression=schedule_request.cron_expression,
            start_time=schedule_request.start_time,
            end_time=schedule_request.end_time,
            interval_second=interval_second,
            catchup=schedule_request.catchup,
        )

    def update(self, schedule_update: ScheduleUpdate) -> "ScheduleSchema":
        """Update a `ScheduleSchema` from a `ScheduleUpdateModel`.

        Args:
            schedule_update: The `ScheduleUpdateModel` to update the schema from.

        Returns:
            The updated `ScheduleSchema`.
        """
        if schedule_update.name is not None:
            self.name = schedule_update.name
        if schedule_update.active is not None:
            self.active = schedule_update.active
        if schedule_update.cron_expression is not None:
            self.cron_expression = schedule_update.cron_expression
        if schedule_update.start_time is not None:
            self.start_time = schedule_update.start_time
        if schedule_update.end_time is not None:
            self.end_time = schedule_update.end_time
        if schedule_update.interval_second is not None:
            self.interval_second = (
                schedule_update.interval_second.total_seconds()
            )
        if schedule_update.catchup is not None:
            self.catchup = schedule_update.catchup
        self.updated = datetime.utcnow()
        return self

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> ScheduleResponse:
        """Convert a `ScheduleSchema` to a `ScheduleResponseModel`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `ScheduleResponseModel`.
        """
        if self.interval_second is not None:
            interval_second = timedelta(seconds=self.interval_second)
        else:
            interval_second = None

        body = ScheduleResponseBody(
            user=self.user.to_model() if self.user else None,
            active=self.active,
            cron_expression=self.cron_expression,
            start_time=self.start_time,
            end_time=self.end_time,
            interval_second=interval_second,
            catchup=self.catchup,
            updated=self.updated,
            created=self.created,
        )
        metadata = None
        if include_metadata:
            metadata = ScheduleResponseMetadata(
                workspace=self.workspace.to_model(),
                pipeline_id=self.pipeline_id,
                orchestrator_id=self.orchestrator_id,
            )

        return ScheduleResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )
from_request(schedule_request) classmethod

Create a ScheduleSchema from a ScheduleRequest.

Parameters:

Name Type Description Default
schedule_request ScheduleRequest

The ScheduleRequest to create the schema from.

required

Returns:

Type Description
ScheduleSchema

The created ScheduleSchema.

Source code in zenml/zen_stores/schemas/schedule_schema.py
@classmethod
def from_request(
    cls, schedule_request: ScheduleRequest
) -> "ScheduleSchema":
    """Create a `ScheduleSchema` from a `ScheduleRequest`.

    Args:
        schedule_request: The `ScheduleRequest` to create the schema from.

    Returns:
        The created `ScheduleSchema`.
    """
    if schedule_request.interval_second is not None:
        interval_second = schedule_request.interval_second.total_seconds()
    else:
        interval_second = None
    return cls(
        name=schedule_request.name,
        workspace_id=schedule_request.workspace,
        user_id=schedule_request.user,
        pipeline_id=schedule_request.pipeline_id,
        orchestrator_id=schedule_request.orchestrator_id,
        active=schedule_request.active,
        cron_expression=schedule_request.cron_expression,
        start_time=schedule_request.start_time,
        end_time=schedule_request.end_time,
        interval_second=interval_second,
        catchup=schedule_request.catchup,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a ScheduleSchema to a ScheduleResponseModel.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ScheduleResponse

The created ScheduleResponseModel.

Source code in zenml/zen_stores/schemas/schedule_schema.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> ScheduleResponse:
    """Convert a `ScheduleSchema` to a `ScheduleResponseModel`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `ScheduleResponseModel`.
    """
    if self.interval_second is not None:
        interval_second = timedelta(seconds=self.interval_second)
    else:
        interval_second = None

    body = ScheduleResponseBody(
        user=self.user.to_model() if self.user else None,
        active=self.active,
        cron_expression=self.cron_expression,
        start_time=self.start_time,
        end_time=self.end_time,
        interval_second=interval_second,
        catchup=self.catchup,
        updated=self.updated,
        created=self.created,
    )
    metadata = None
    if include_metadata:
        metadata = ScheduleResponseMetadata(
            workspace=self.workspace.to_model(),
            pipeline_id=self.pipeline_id,
            orchestrator_id=self.orchestrator_id,
        )

    return ScheduleResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, schedule_update)

Update a ScheduleSchema from a ScheduleUpdateModel.

Parameters:

Name Type Description Default
schedule_update ScheduleUpdate

The ScheduleUpdateModel to update the schema from.

required

Returns:

Type Description
ScheduleSchema

The updated ScheduleSchema.

Source code in zenml/zen_stores/schemas/schedule_schema.py
def update(self, schedule_update: ScheduleUpdate) -> "ScheduleSchema":
    """Update a `ScheduleSchema` from a `ScheduleUpdateModel`.

    Args:
        schedule_update: The `ScheduleUpdateModel` to update the schema from.

    Returns:
        The updated `ScheduleSchema`.
    """
    if schedule_update.name is not None:
        self.name = schedule_update.name
    if schedule_update.active is not None:
        self.active = schedule_update.active
    if schedule_update.cron_expression is not None:
        self.cron_expression = schedule_update.cron_expression
    if schedule_update.start_time is not None:
        self.start_time = schedule_update.start_time
    if schedule_update.end_time is not None:
        self.end_time = schedule_update.end_time
    if schedule_update.interval_second is not None:
        self.interval_second = (
            schedule_update.interval_second.total_seconds()
        )
    if schedule_update.catchup is not None:
        self.catchup = schedule_update.catchup
    self.updated = datetime.utcnow()
    return self

schema_utils

Utility functions for SQLModel schemas.

build_foreign_key_field(source, target, source_column, target_column, ondelete, nullable, **sa_column_kwargs)

Build a SQLModel foreign key field.

Parameters:

Name Type Description Default
source str

Source table name.

required
target str

Target table name.

required
source_column str

Source column name.

required
target_column str

Target column name.

required
ondelete str

On delete behavior.

required
nullable bool

Whether the field is nullable.

required
**sa_column_kwargs Any

Keyword arguments for the SQLAlchemy column.

{}

Returns:

Type Description
Any

SQLModel foreign key field.

Exceptions:

Type Description
ValueError

If the ondelete and nullable arguments are not compatible.

Source code in zenml/zen_stores/schemas/schema_utils.py
def build_foreign_key_field(
    source: str,
    target: str,
    source_column: str,
    target_column: str,
    ondelete: str,
    nullable: bool,
    **sa_column_kwargs: Any,
) -> Any:
    """Build a SQLModel foreign key field.

    Args:
        source: Source table name.
        target: Target table name.
        source_column: Source column name.
        target_column: Target column name.
        ondelete: On delete behavior.
        nullable: Whether the field is nullable.
        **sa_column_kwargs: Keyword arguments for the SQLAlchemy column.

    Returns:
        SQLModel foreign key field.

    Raises:
        ValueError: If the ondelete and nullable arguments are not compatible.
    """
    if not nullable and ondelete == "SET NULL":
        raise ValueError(
            "Cannot set ondelete to SET NULL if the field is not nullable."
        )
    constraint_name = foreign_key_constraint_name(
        source=source,
        target=target,
        source_column=source_column,
    )
    return Field(
        sa_column=Column(
            ForeignKey(
                f"{target}.{target_column}",
                name=constraint_name,
                ondelete=ondelete,
            ),
            nullable=nullable,
            **sa_column_kwargs,
        ),
    )
foreign_key_constraint_name(source, target, source_column)

Defines the name of a foreign key constraint.

For simplicity, we use the naming convention used by alembic here: https://alembic.sqlalchemy.org/en/latest/batch.html#dropping-unnamed-or-named-foreign-key-constraints.

Parameters:

Name Type Description Default
source str

Source table name.

required
target str

Target table name.

required
source_column str

Source column name.

required

Returns:

Type Description
str

Name of the foreign key constraint.

Source code in zenml/zen_stores/schemas/schema_utils.py
def foreign_key_constraint_name(
    source: str, target: str, source_column: str
) -> str:
    """Defines the name of a foreign key constraint.

    For simplicity, we use the naming convention used by alembic here:
    https://alembic.sqlalchemy.org/en/latest/batch.html#dropping-unnamed-or-named-foreign-key-constraints.

    Args:
        source: Source table name.
        target: Target table name.
        source_column: Source column name.

    Returns:
        Name of the foreign key constraint.
    """
    return f"fk_{source}_{source_column}_{target}"

secret_schemas

SQL Model Implementations for Secrets.

SecretDecodeError (Exception)

Raised when a secret cannot be decoded or decrypted.

Source code in zenml/zen_stores/schemas/secret_schemas.py
class SecretDecodeError(Exception):
    """Raised when a secret cannot be decoded or decrypted."""
SecretSchema (NamedSchema) pydantic-model

SQL Model for secrets.

Attributes:

Name Type Description
name str

The name of the secret.

values Optional[bytes]

The values of the secret.

Source code in zenml/zen_stores/schemas/secret_schemas.py
class SecretSchema(NamedSchema, table=True):
    """SQL Model for secrets.

    Attributes:
        name: The name of the secret.
        values: The values of the secret.
    """

    __tablename__ = "secret"

    scope: SecretScope

    values: Optional[bytes] = Field(sa_column=Column(TEXT, nullable=True))

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="secrets")

    user_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    user: "UserSchema" = Relationship(back_populates="secrets")

    @classmethod
    def _dump_secret_values(
        cls, values: Dict[str, str], encryption_engine: Optional[AesGcmEngine]
    ) -> bytes:
        """Dump the secret values to a string.

        Args:
            values: The secret values to dump.
            encryption_engine: The encryption engine to use to encrypt the
                secret values. If None, the values will be base64 encoded.

        Raises:
            ValueError: If the secret values do not fit in the database field.

        Returns:
            The serialized encrypted secret values.
        """
        serialized_values = json.dumps(values)

        if encryption_engine is None:
            encrypted_values = base64.b64encode(
                serialized_values.encode("utf-8")
            )
        else:
            encrypted_values = encryption_engine.encrypt(serialized_values)

        if len(encrypted_values) > TEXT_FIELD_MAX_LENGTH:
            raise ValueError(
                "Database representation of secret values exceeds max "
                "length. Please use fewer values or consider using shorter "
                "secret keys and/or values."
            )

        return encrypted_values

    @classmethod
    def _load_secret_values(
        cls,
        encrypted_values: bytes,
        encryption_engine: Optional[AesGcmEngine] = None,
    ) -> Dict[str, str]:
        """Load the secret values from a base64 encoded byte string.

        Args:
            encrypted_values: The serialized encrypted secret values.
            encryption_engine: The encryption engine to use to decrypt the
                secret values. If None, the values will be base64 decoded.

        Returns:
            The loaded secret values.

        Raises:
            SecretDecodeError: If the secret values cannot be decoded or
                decrypted.
        """
        if encryption_engine is None:
            try:
                serialized_values = base64.b64decode(encrypted_values).decode()
            except ValueError as e:
                raise SecretDecodeError(
                    "Could not decode base64 encoded secret values: {str(e)}"
                ) from e
        else:
            try:
                serialized_values = encryption_engine.decrypt(encrypted_values)
            except (ValueError, InvalidCiphertextError) as e:
                raise SecretDecodeError(
                    "Could not decrypt secret values. Please check that the "
                    f"encryption key is correct: {str(e)}"
                ) from e

        try:
            return cast(
                Dict[str, str],
                json.loads(serialized_values),
            )
        except json.JSONDecodeError as e:
            raise SecretDecodeError(
                "Could not decode secret values. Please check that the "
                f"secret values are valid JSON: {str(e)}"
            ) from e

    @classmethod
    def from_request(
        cls,
        secret: SecretRequest,
    ) -> "SecretSchema":
        """Create a `SecretSchema` from a `SecretRequest`.

        Args:
            secret: The `SecretRequest` from which to create the schema.

        Returns:
            The created `SecretSchema`.
        """
        assert secret.user is not None, "User must be set for secret creation."
        return cls(
            name=secret.name,
            scope=secret.scope,
            workspace_id=secret.workspace,
            user_id=secret.user,
            # Don't store secret values implicitly in the secret. The
            # SQL secret store will call `store_secret_values` to store the
            # values separately if SQL is used as the secrets store.
            values=None,
        )

    def update(
        self,
        secret_update: SecretUpdate,
    ) -> "SecretSchema":
        """Update a `SecretSchema` from a `SecretUpdate`.

        Args:
            secret_update: The `SecretUpdate` from which to update the schema.

        Returns:
            The updated `SecretSchema`.
        """
        # Don't update the secret values implicitly in the secret. The
        # SQL secret store will call `set_secret_values` to update the
        # values separately if SQL is used as the secrets store.
        for field, value in secret_update.dict(
            exclude_unset=True, exclude={"workspace", "user", "values"}
        ).items():
            setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> SecretResponse:
        """Converts a secret schema to a secret model.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The secret model.
        """
        metadata = None
        if include_metadata:
            metadata = SecretResponseMetadata(
                workspace=self.workspace.to_model(),
            )

        # Don't load the secret values implicitly in the secret. The
        # SQL secret store will call `get_secret_values` to load the
        # values separately if SQL is used as the secrets store.
        body = SecretResponseBody(
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
            scope=self.scope,
        )
        return SecretResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )

    def get_secret_values(
        self,
        encryption_engine: Optional[AesGcmEngine] = None,
    ) -> Dict[str, str]:
        """Get the secret values for this secret.

        This method is used by the SQL secrets store to load the secret values
        from the database.

        Args:
            encryption_engine: The encryption engine to use to decrypt the
                secret values. If None, the values will be base64 decoded.

        Returns:
            The secret values

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
        """
        if not self.values:
            raise KeyError(
                f"Secret values for secret {self.id} have not been stored in "
                f"the SQL secrets store."
            )
        return self._load_secret_values(self.values, encryption_engine)

    def set_secret_values(
        self,
        secret_values: Dict[str, str],
        encryption_engine: Optional[AesGcmEngine] = None,
    ) -> None:
        """Create a `SecretSchema` from a `SecretRequest`.

        This method is used by the SQL secrets store to store the secret values
        in the database.

        Args:
            secret_values: The new secret values.
            encryption_engine: The encryption engine to use to encrypt the
                secret values. If None, the values will be base64 encoded.
        """
        self.values = self._dump_secret_values(
            secret_values, encryption_engine
        )
from_request(secret) classmethod

Create a SecretSchema from a SecretRequest.

Parameters:

Name Type Description Default
secret SecretRequest

The SecretRequest from which to create the schema.

required

Returns:

Type Description
SecretSchema

The created SecretSchema.

Source code in zenml/zen_stores/schemas/secret_schemas.py
@classmethod
def from_request(
    cls,
    secret: SecretRequest,
) -> "SecretSchema":
    """Create a `SecretSchema` from a `SecretRequest`.

    Args:
        secret: The `SecretRequest` from which to create the schema.

    Returns:
        The created `SecretSchema`.
    """
    assert secret.user is not None, "User must be set for secret creation."
    return cls(
        name=secret.name,
        scope=secret.scope,
        workspace_id=secret.workspace,
        user_id=secret.user,
        # Don't store secret values implicitly in the secret. The
        # SQL secret store will call `store_secret_values` to store the
        # values separately if SQL is used as the secrets store.
        values=None,
    )
get_secret_values(self, encryption_engine=None)

Get the secret values for this secret.

This method is used by the SQL secrets store to load the secret values from the database.

Parameters:

Name Type Description Default
encryption_engine Optional[sqlalchemy_utils.types.encrypted.encrypted_type.AesGcmEngine]

The encryption engine to use to decrypt the secret values. If None, the values will be base64 decoded.

None

Returns:

Type Description
Dict[str, str]

The secret values

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

Source code in zenml/zen_stores/schemas/secret_schemas.py
def get_secret_values(
    self,
    encryption_engine: Optional[AesGcmEngine] = None,
) -> Dict[str, str]:
    """Get the secret values for this secret.

    This method is used by the SQL secrets store to load the secret values
    from the database.

    Args:
        encryption_engine: The encryption engine to use to decrypt the
            secret values. If None, the values will be base64 decoded.

    Returns:
        The secret values

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
    """
    if not self.values:
        raise KeyError(
            f"Secret values for secret {self.id} have not been stored in "
            f"the SQL secrets store."
        )
    return self._load_secret_values(self.values, encryption_engine)
set_secret_values(self, secret_values, encryption_engine=None)

Create a SecretSchema from a SecretRequest.

This method is used by the SQL secrets store to store the secret values in the database.

Parameters:

Name Type Description Default
secret_values Dict[str, str]

The new secret values.

required
encryption_engine Optional[sqlalchemy_utils.types.encrypted.encrypted_type.AesGcmEngine]

The encryption engine to use to encrypt the secret values. If None, the values will be base64 encoded.

None
Source code in zenml/zen_stores/schemas/secret_schemas.py
def set_secret_values(
    self,
    secret_values: Dict[str, str],
    encryption_engine: Optional[AesGcmEngine] = None,
) -> None:
    """Create a `SecretSchema` from a `SecretRequest`.

    This method is used by the SQL secrets store to store the secret values
    in the database.

    Args:
        secret_values: The new secret values.
        encryption_engine: The encryption engine to use to encrypt the
            secret values. If None, the values will be base64 encoded.
    """
    self.values = self._dump_secret_values(
        secret_values, encryption_engine
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Converts a secret schema to a secret model.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
SecretResponse

The secret model.

Source code in zenml/zen_stores/schemas/secret_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> SecretResponse:
    """Converts a secret schema to a secret model.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The secret model.
    """
    metadata = None
    if include_metadata:
        metadata = SecretResponseMetadata(
            workspace=self.workspace.to_model(),
        )

    # Don't load the secret values implicitly in the secret. The
    # SQL secret store will call `get_secret_values` to load the
    # values separately if SQL is used as the secrets store.
    body = SecretResponseBody(
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
        scope=self.scope,
    )
    return SecretResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, secret_update)

Update a SecretSchema from a SecretUpdate.

Parameters:

Name Type Description Default
secret_update SecretUpdate

The SecretUpdate from which to update the schema.

required

Returns:

Type Description
SecretSchema

The updated SecretSchema.

Source code in zenml/zen_stores/schemas/secret_schemas.py
def update(
    self,
    secret_update: SecretUpdate,
) -> "SecretSchema":
    """Update a `SecretSchema` from a `SecretUpdate`.

    Args:
        secret_update: The `SecretUpdate` from which to update the schema.

    Returns:
        The updated `SecretSchema`.
    """
    # Don't update the secret values implicitly in the secret. The
    # SQL secret store will call `set_secret_values` to update the
    # values separately if SQL is used as the secrets store.
    for field, value in secret_update.dict(
        exclude_unset=True, exclude={"workspace", "user", "values"}
    ).items():
        setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self

service_connector_schemas

SQL Model Implementations for Service Connectors.

ServiceConnectorSchema (NamedSchema) pydantic-model

SQL Model for service connectors.

Source code in zenml/zen_stores/schemas/service_connector_schemas.py
class ServiceConnectorSchema(NamedSchema, table=True):
    """SQL Model for service connectors."""

    __tablename__ = "service_connector"

    connector_type: str = Field(sa_column=Column(TEXT))
    description: str
    auth_method: str = Field(sa_column=Column(TEXT))
    resource_types: bytes
    resource_id: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
    supports_instances: bool
    configuration: Optional[bytes]
    secret_id: Optional[UUID]
    expires_at: Optional[datetime]
    expires_skew_tolerance: Optional[int]
    expiration_seconds: Optional[int]
    labels: Optional[bytes]

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(
        back_populates="service_connectors"
    )

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(
        back_populates="service_connectors"
    )
    components: List["StackComponentSchema"] = Relationship(
        back_populates="connector",
    )

    @property
    def resource_types_list(self) -> List[str]:
        """Returns the resource types as a list.

        Returns:
            The resource types as a list.
        """
        resource_types = json.loads(
            base64.b64decode(self.resource_types).decode()
        )
        assert isinstance(resource_types, list)
        return resource_types

    @property
    def labels_dict(self) -> Dict[str, str]:
        """Returns the labels as a dictionary.

        Returns:
            The labels as a dictionary.
        """
        if self.labels is None:
            return {}
        labels_dict = json.loads(base64.b64decode(self.labels).decode())
        return cast(Dict[str, str], labels_dict)

    def has_labels(self, labels: Dict[str, Optional[str]]) -> bool:
        """Checks if the connector has the given labels.

        Args:
            labels: The labels to check for.

        Returns:
            Whether the connector has the given labels.
        """
        return all(
            self.labels_dict.get(key, None) == value
            for key, value in labels.items()
            if value is not None
        ) and all(
            key in self.labels_dict
            for key, value in labels.items()
            if value is None
        )

    @classmethod
    def from_request(
        cls,
        connector_request: ServiceConnectorRequest,
        secret_id: Optional[UUID] = None,
    ) -> "ServiceConnectorSchema":
        """Create a `ServiceConnectorSchema` from a `ServiceConnectorRequest`.

        Args:
            connector_request: The `ServiceConnectorRequest` from which to
                create the schema.
            secret_id: The ID of the secret to use for this connector.

        Returns:
            The created `ServiceConnectorSchema`.
        """
        assert connector_request.user is not None, "User must be set."
        return cls(
            workspace_id=connector_request.workspace,
            user_id=connector_request.user,
            name=connector_request.name,
            description=connector_request.description,
            connector_type=connector_request.type,
            auth_method=connector_request.auth_method,
            resource_types=base64.b64encode(
                json.dumps(connector_request.resource_types).encode("utf-8")
            ),
            resource_id=connector_request.resource_id,
            supports_instances=connector_request.supports_instances,
            configuration=base64.b64encode(
                json.dumps(connector_request.configuration).encode("utf-8")
            )
            if connector_request.configuration
            else None,
            secret_id=secret_id,
            expires_at=connector_request.expires_at,
            expires_skew_tolerance=connector_request.expires_skew_tolerance,
            expiration_seconds=connector_request.expiration_seconds,
            labels=base64.b64encode(
                json.dumps(connector_request.labels).encode("utf-8")
            )
            if connector_request.labels
            else None,
        )

    def update(
        self,
        connector_update: ServiceConnectorUpdate,
        secret_id: Optional[UUID] = None,
    ) -> "ServiceConnectorSchema":
        """Updates a `ServiceConnectorSchema` from a `ServiceConnectorUpdate`.

        Args:
            connector_update: The `ServiceConnectorUpdate` to update from.
            secret_id: The ID of the secret to use for this connector.

        Returns:
            The updated `ServiceConnectorSchema`.
        """
        for field, value in connector_update.dict(
            exclude_unset=False,
            exclude={"workspace", "user", "secrets"},
        ).items():
            if value is None:
                if field == "resource_id":
                    # The resource ID field in the update is special: if set
                    # to None in the update, it triggers the existing resource
                    # ID to be cleared.
                    self.resource_id = None
                if field == "expiration_seconds":
                    # The expiration_seconds field in the update is special:
                    # if set to None in the update, it triggers the existing
                    # expiration_seconds to be cleared.
                    self.expiration_seconds = None
                continue
            if field == "configuration":
                self.configuration = (
                    base64.b64encode(
                        json.dumps(connector_update.configuration).encode(
                            "utf-8"
                        )
                    )
                    if connector_update.configuration
                    else None
                )
            elif field == "resource_types":
                self.resource_types = base64.b64encode(
                    json.dumps(connector_update.resource_types).encode("utf-8")
                )
            elif field == "labels":
                self.labels = (
                    base64.b64encode(
                        json.dumps(connector_update.labels).encode("utf-8")
                    )
                    if connector_update.labels
                    else None
                )
            else:
                setattr(self, field, value)
        self.secret_id = secret_id
        self.updated = datetime.utcnow()
        return self

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "ServiceConnectorResponse":
        """Creates a `ServiceConnector` from a `ServiceConnectorSchema`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            A `ServiceConnectorModel`
        """
        body = ServiceConnectorResponseBody(
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
            description=self.description,
            connector_type=self.connector_type,
            auth_method=self.auth_method,
            resource_types=self.resource_types_list,
            resource_id=self.resource_id,
            supports_instances=self.supports_instances,
            expires_at=self.expires_at,
            expires_skew_tolerance=self.expires_skew_tolerance,
        )
        metadata = None
        if include_metadata:
            metadata = ServiceConnectorResponseMetadata(
                workspace=self.workspace.to_model(),
                configuration=json.loads(
                    base64.b64decode(self.configuration).decode()
                )
                if self.configuration
                else {},
                secret_id=self.secret_id,
                expiration_seconds=self.expiration_seconds,
                labels=self.labels_dict,
            )
        return ServiceConnectorResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )
labels_dict: Dict[str, str] property readonly

Returns the labels as a dictionary.

Returns:

Type Description
Dict[str, str]

The labels as a dictionary.

resource_types_list: List[str] property readonly

Returns the resource types as a list.

Returns:

Type Description
List[str]

The resource types as a list.

from_request(connector_request, secret_id=None) classmethod

Create a ServiceConnectorSchema from a ServiceConnectorRequest.

Parameters:

Name Type Description Default
connector_request ServiceConnectorRequest

The ServiceConnectorRequest from which to create the schema.

required
secret_id Optional[uuid.UUID]

The ID of the secret to use for this connector.

None

Returns:

Type Description
ServiceConnectorSchema

The created ServiceConnectorSchema.

Source code in zenml/zen_stores/schemas/service_connector_schemas.py
@classmethod
def from_request(
    cls,
    connector_request: ServiceConnectorRequest,
    secret_id: Optional[UUID] = None,
) -> "ServiceConnectorSchema":
    """Create a `ServiceConnectorSchema` from a `ServiceConnectorRequest`.

    Args:
        connector_request: The `ServiceConnectorRequest` from which to
            create the schema.
        secret_id: The ID of the secret to use for this connector.

    Returns:
        The created `ServiceConnectorSchema`.
    """
    assert connector_request.user is not None, "User must be set."
    return cls(
        workspace_id=connector_request.workspace,
        user_id=connector_request.user,
        name=connector_request.name,
        description=connector_request.description,
        connector_type=connector_request.type,
        auth_method=connector_request.auth_method,
        resource_types=base64.b64encode(
            json.dumps(connector_request.resource_types).encode("utf-8")
        ),
        resource_id=connector_request.resource_id,
        supports_instances=connector_request.supports_instances,
        configuration=base64.b64encode(
            json.dumps(connector_request.configuration).encode("utf-8")
        )
        if connector_request.configuration
        else None,
        secret_id=secret_id,
        expires_at=connector_request.expires_at,
        expires_skew_tolerance=connector_request.expires_skew_tolerance,
        expiration_seconds=connector_request.expiration_seconds,
        labels=base64.b64encode(
            json.dumps(connector_request.labels).encode("utf-8")
        )
        if connector_request.labels
        else None,
    )
has_labels(self, labels)

Checks if the connector has the given labels.

Parameters:

Name Type Description Default
labels Dict[str, Optional[str]]

The labels to check for.

required

Returns:

Type Description
bool

Whether the connector has the given labels.

Source code in zenml/zen_stores/schemas/service_connector_schemas.py
def has_labels(self, labels: Dict[str, Optional[str]]) -> bool:
    """Checks if the connector has the given labels.

    Args:
        labels: The labels to check for.

    Returns:
        Whether the connector has the given labels.
    """
    return all(
        self.labels_dict.get(key, None) == value
        for key, value in labels.items()
        if value is not None
    ) and all(
        key in self.labels_dict
        for key, value in labels.items()
        if value is None
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Creates a ServiceConnector from a ServiceConnectorSchema.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
ServiceConnectorResponse

A ServiceConnectorModel

Source code in zenml/zen_stores/schemas/service_connector_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "ServiceConnectorResponse":
    """Creates a `ServiceConnector` from a `ServiceConnectorSchema`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        A `ServiceConnectorModel`
    """
    body = ServiceConnectorResponseBody(
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
        description=self.description,
        connector_type=self.connector_type,
        auth_method=self.auth_method,
        resource_types=self.resource_types_list,
        resource_id=self.resource_id,
        supports_instances=self.supports_instances,
        expires_at=self.expires_at,
        expires_skew_tolerance=self.expires_skew_tolerance,
    )
    metadata = None
    if include_metadata:
        metadata = ServiceConnectorResponseMetadata(
            workspace=self.workspace.to_model(),
            configuration=json.loads(
                base64.b64decode(self.configuration).decode()
            )
            if self.configuration
            else {},
            secret_id=self.secret_id,
            expiration_seconds=self.expiration_seconds,
            labels=self.labels_dict,
        )
    return ServiceConnectorResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, connector_update, secret_id=None)

Updates a ServiceConnectorSchema from a ServiceConnectorUpdate.

Parameters:

Name Type Description Default
connector_update ServiceConnectorUpdate

The ServiceConnectorUpdate to update from.

required
secret_id Optional[uuid.UUID]

The ID of the secret to use for this connector.

None

Returns:

Type Description
ServiceConnectorSchema

The updated ServiceConnectorSchema.

Source code in zenml/zen_stores/schemas/service_connector_schemas.py
def update(
    self,
    connector_update: ServiceConnectorUpdate,
    secret_id: Optional[UUID] = None,
) -> "ServiceConnectorSchema":
    """Updates a `ServiceConnectorSchema` from a `ServiceConnectorUpdate`.

    Args:
        connector_update: The `ServiceConnectorUpdate` to update from.
        secret_id: The ID of the secret to use for this connector.

    Returns:
        The updated `ServiceConnectorSchema`.
    """
    for field, value in connector_update.dict(
        exclude_unset=False,
        exclude={"workspace", "user", "secrets"},
    ).items():
        if value is None:
            if field == "resource_id":
                # The resource ID field in the update is special: if set
                # to None in the update, it triggers the existing resource
                # ID to be cleared.
                self.resource_id = None
            if field == "expiration_seconds":
                # The expiration_seconds field in the update is special:
                # if set to None in the update, it triggers the existing
                # expiration_seconds to be cleared.
                self.expiration_seconds = None
            continue
        if field == "configuration":
            self.configuration = (
                base64.b64encode(
                    json.dumps(connector_update.configuration).encode(
                        "utf-8"
                    )
                )
                if connector_update.configuration
                else None
            )
        elif field == "resource_types":
            self.resource_types = base64.b64encode(
                json.dumps(connector_update.resource_types).encode("utf-8")
            )
        elif field == "labels":
            self.labels = (
                base64.b64encode(
                    json.dumps(connector_update.labels).encode("utf-8")
                )
                if connector_update.labels
                else None
            )
        else:
            setattr(self, field, value)
    self.secret_id = secret_id
    self.updated = datetime.utcnow()
    return self

stack_schemas

SQL Model Implementations for Stacks.

StackCompositionSchema (SQLModel) pydantic-model

SQL Model for stack definitions.

Join table between Stacks and StackComponents.

Source code in zenml/zen_stores/schemas/stack_schemas.py
class StackCompositionSchema(SQLModel, table=True):
    """SQL Model for stack definitions.

    Join table between Stacks and StackComponents.
    """

    __tablename__ = "stack_composition"

    stack_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target="stack",
        source_column="stack_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
        primary_key=True,
    )
    component_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target="stack_component",
        source_column="component_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
        primary_key=True,
    )
StackSchema (NamedSchema) pydantic-model

SQL Model for stacks.

Source code in zenml/zen_stores/schemas/stack_schemas.py
class StackSchema(NamedSchema, table=True):
    """SQL Model for stacks."""

    __tablename__ = "stack"
    stack_spec_path: Optional[str]

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="stacks")

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(back_populates="stacks")

    components: List["StackComponentSchema"] = Relationship(
        back_populates="stacks",
        link_model=StackCompositionSchema,
    )
    builds: List["PipelineBuildSchema"] = Relationship(back_populates="stack")
    deployments: List["PipelineDeploymentSchema"] = Relationship(
        back_populates="stack",
    )

    def update(
        self,
        stack_update: "StackUpdate",
        components: List["StackComponentSchema"],
    ) -> "StackSchema":
        """Updates a stack schema with a stack update model.

        Args:
            stack_update: `StackUpdate` to update the stack with.
            components: List of `StackComponentSchema` to update the stack with.

        Returns:
            The updated StackSchema.
        """
        for field, value in stack_update.dict(
            exclude_unset=True, exclude={"workspace", "user"}
        ).items():
            if field == "components":
                self.components = components
            else:
                setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "StackResponse":
        """Converts the schema to a model.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The converted model.
        """
        body = StackResponseBody(
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = StackResponseMetadata(
                workspace=self.workspace.to_model(),
                components={c.type: [c.to_model()] for c in self.components},
                stack_spec_path=self.stack_spec_path,
            )

        return StackResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Converts the schema to a model.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
StackResponse

The converted model.

Source code in zenml/zen_stores/schemas/stack_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "StackResponse":
    """Converts the schema to a model.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The converted model.
    """
    body = StackResponseBody(
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = StackResponseMetadata(
            workspace=self.workspace.to_model(),
            components={c.type: [c.to_model()] for c in self.components},
            stack_spec_path=self.stack_spec_path,
        )

    return StackResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, stack_update, components)

Updates a stack schema with a stack update model.

Parameters:

Name Type Description Default
stack_update StackUpdate

StackUpdate to update the stack with.

required
components List[StackComponentSchema]

List of StackComponentSchema to update the stack with.

required

Returns:

Type Description
StackSchema

The updated StackSchema.

Source code in zenml/zen_stores/schemas/stack_schemas.py
def update(
    self,
    stack_update: "StackUpdate",
    components: List["StackComponentSchema"],
) -> "StackSchema":
    """Updates a stack schema with a stack update model.

    Args:
        stack_update: `StackUpdate` to update the stack with.
        components: List of `StackComponentSchema` to update the stack with.

    Returns:
        The updated StackSchema.
    """
    for field, value in stack_update.dict(
        exclude_unset=True, exclude={"workspace", "user"}
    ).items():
        if field == "components":
            self.components = components
        else:
            setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self

step_run_schemas

SQLModel implementation of step run tables.

StepRunInputArtifactSchema (SQLModel) pydantic-model

SQL Model that defines which artifacts are inputs to which step.

Source code in zenml/zen_stores/schemas/step_run_schemas.py
class StepRunInputArtifactSchema(SQLModel, table=True):
    """SQL Model that defines which artifacts are inputs to which step."""

    __tablename__ = "step_run_input_artifact"

    # Fields
    name: str = Field(nullable=False, primary_key=True)
    type: StepRunInputArtifactType

    # Foreign keys
    step_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=StepRunSchema.__tablename__,
        source_column="step_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
        primary_key=True,
    )
    # Note: We keep the name artifact_id instead of artifact_version_id here to
    # avoid having to drop and recreate the primary key constraint.
    artifact_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target="artifact_version",
        source_column="artifact_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
        primary_key=True,
    )

    # Relationships
    step_run: "StepRunSchema" = Relationship(back_populates="input_artifacts")
    artifact_version: "ArtifactVersionSchema" = Relationship()
StepRunOutputArtifactSchema (SQLModel) pydantic-model

SQL Model that defines which artifacts are outputs of which step.

Source code in zenml/zen_stores/schemas/step_run_schemas.py
class StepRunOutputArtifactSchema(SQLModel, table=True):
    """SQL Model that defines which artifacts are outputs of which step."""

    __tablename__ = "step_run_output_artifact"

    # Fields
    name: str
    type: StepRunOutputArtifactType

    # Foreign keys
    step_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=StepRunSchema.__tablename__,
        source_column="step_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
        primary_key=True,
    )
    # Note: we keep the name artifact_id instead of artifact_version_id here to
    # avoid having to drop and recreate the primary key constraint.
    artifact_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target="artifact_version",
        source_column="artifact_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
        primary_key=True,
    )

    # Relationship
    step_run: "StepRunSchema" = Relationship(back_populates="output_artifacts")
    artifact_version: "ArtifactVersionSchema" = Relationship(
        back_populates="output_of_step_runs"
    )
StepRunParentsSchema (SQLModel) pydantic-model

SQL Model that defines the order of steps.

Source code in zenml/zen_stores/schemas/step_run_schemas.py
class StepRunParentsSchema(SQLModel, table=True):
    """SQL Model that defines the order of steps."""

    __tablename__ = "step_run_parents"

    # Foreign Keys
    parent_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=StepRunSchema.__tablename__,
        source_column="parent_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
        primary_key=True,
    )
    child_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=StepRunSchema.__tablename__,
        source_column="child_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
        primary_key=True,
    )
StepRunSchema (NamedSchema) pydantic-model

SQL Model for steps of pipeline runs.

Source code in zenml/zen_stores/schemas/step_run_schemas.py
class StepRunSchema(NamedSchema, table=True):
    """SQL Model for steps of pipeline runs."""

    __tablename__ = "step_run"

    # Fields
    start_time: Optional[datetime] = Field(nullable=True)
    end_time: Optional[datetime] = Field(nullable=True)
    status: ExecutionStatus = Field(nullable=False)

    docstring: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
    cache_key: Optional[str] = Field(nullable=True)
    source_code: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
    code_hash: Optional[str] = Field(nullable=True)

    step_configuration: str = Field(
        sa_column=Column(
            String(length=MEDIUMTEXT_MAX_LENGTH).with_variant(
                MEDIUMTEXT, "mysql"
            ),
            nullable=True,
        )
    )

    # Foreign keys
    original_step_run_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=__tablename__,
        source_column="original_step_run_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    deployment_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=PipelineDeploymentSchema.__tablename__,
        source_column="deployment_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=True,
    )
    pipeline_run_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=PipelineRunSchema.__tablename__,
        source_column="pipeline_run_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )

    # Relationships
    workspace: "WorkspaceSchema" = Relationship(back_populates="step_runs")
    user: Optional["UserSchema"] = Relationship(back_populates="step_runs")
    deployment: Optional["PipelineDeploymentSchema"] = Relationship(
        back_populates="step_runs"
    )
    run_metadata: List["RunMetadataSchema"] = Relationship(
        back_populates="step_run",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(RunMetadataSchema.resource_type=='{MetadataResourceTypes.STEP_RUN.value}', foreign(RunMetadataSchema.resource_id)==StepRunSchema.id)",
            cascade="delete",
            overlaps="run_metadata",
        ),
    )
    input_artifacts: List["StepRunInputArtifactSchema"] = Relationship(
        sa_relationship_kwargs={"cascade": "delete"}
    )
    output_artifacts: List["StepRunOutputArtifactSchema"] = Relationship(
        sa_relationship_kwargs={"cascade": "delete"}
    )
    logs: Optional["LogsSchema"] = Relationship(
        back_populates="step_run",
        sa_relationship_kwargs={"cascade": "delete", "uselist": False},
    )
    parents: List["StepRunParentsSchema"] = Relationship(
        sa_relationship_kwargs={
            "cascade": "delete",
            "primaryjoin": "StepRunParentsSchema.child_id == StepRunSchema.id",
        },
    )

    @classmethod
    def from_request(cls, request: StepRunRequest) -> "StepRunSchema":
        """Create a step run schema from a step run request model.

        Args:
            request: The step run request model.

        Returns:
            The step run schema.
        """
        return cls(
            name=request.name,
            workspace_id=request.workspace,
            user_id=request.user,
            start_time=request.start_time,
            end_time=request.end_time,
            status=request.status,
            original_step_run_id=request.original_step_run_id,
            pipeline_run_id=request.pipeline_run_id,
            deployment_id=request.deployment,
            docstring=request.docstring,
            cache_key=request.cache_key,
            code_hash=request.code_hash,
            source_code=request.source_code,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> StepRunResponse:
        """Convert a `StepRunSchema` to a `StepRunResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created StepRunResponse.

        Raises:
            RuntimeError: If the step run schema does not have a deployment_id
                or a step_configuration.
        """
        run_metadata = {
            metadata_schema.key: metadata_schema.to_model()
            for metadata_schema in self.run_metadata
        }

        input_artifacts = {
            artifact.name: artifact.artifact_version.to_model()
            for artifact in self.input_artifacts
        }

        output_artifacts = {
            artifact.name: artifact.artifact_version.to_model()
            for artifact in self.output_artifacts
        }

        if self.deployment is not None:
            full_step_config = Step.parse_obj(
                json.loads(self.deployment.step_configurations)[self.name]
            )
        elif self.step_configuration is not None:
            full_step_config = Step.parse_raw(self.step_configuration)
        else:
            raise RuntimeError(
                "Step run model creation has failed. Each step run entry "
                "should either have a deployment_id or step_configuration."
            )

        body = StepRunResponseBody(
            user=self.user.to_model() if self.user else None,
            status=self.status,
            inputs=input_artifacts,
            outputs=output_artifacts,
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = StepRunResponseMetadata(
                workspace=self.workspace.to_model(),
                config=full_step_config.config,
                spec=full_step_config.spec,
                cache_key=self.cache_key,
                code_hash=self.code_hash,
                docstring=self.docstring,
                source_code=self.source_code,
                start_time=self.start_time,
                end_time=self.end_time,
                logs=self.logs.to_model() if self.logs else None,
                deployment_id=self.deployment_id,
                pipeline_run_id=self.pipeline_run_id,
                original_step_run_id=self.original_step_run_id,
                parent_step_ids=[p.parent_id for p in self.parents],
                run_metadata=run_metadata,
            )
        return StepRunResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )

    def update(self, step_update: "StepRunUpdate") -> "StepRunSchema":
        """Update a step run schema with a step run update model.

        Args:
            step_update: The step run update model.

        Returns:
            The updated step run schema.
        """
        for key, value in step_update.dict(
            exclude_unset=True, exclude_none=True
        ).items():
            if key == "status":
                self.status = value
            if key == "end_time":
                self.end_time = value

        self.updated = datetime.utcnow()

        return self
from_request(request) classmethod

Create a step run schema from a step run request model.

Parameters:

Name Type Description Default
request StepRunRequest

The step run request model.

required

Returns:

Type Description
StepRunSchema

The step run schema.

Source code in zenml/zen_stores/schemas/step_run_schemas.py
@classmethod
def from_request(cls, request: StepRunRequest) -> "StepRunSchema":
    """Create a step run schema from a step run request model.

    Args:
        request: The step run request model.

    Returns:
        The step run schema.
    """
    return cls(
        name=request.name,
        workspace_id=request.workspace,
        user_id=request.user,
        start_time=request.start_time,
        end_time=request.end_time,
        status=request.status,
        original_step_run_id=request.original_step_run_id,
        pipeline_run_id=request.pipeline_run_id,
        deployment_id=request.deployment,
        docstring=request.docstring,
        cache_key=request.cache_key,
        code_hash=request.code_hash,
        source_code=request.source_code,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a StepRunSchema to a StepRunResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
StepRunResponse

The created StepRunResponse.

Exceptions:

Type Description
RuntimeError

If the step run schema does not have a deployment_id or a step_configuration.

Source code in zenml/zen_stores/schemas/step_run_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> StepRunResponse:
    """Convert a `StepRunSchema` to a `StepRunResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created StepRunResponse.

    Raises:
        RuntimeError: If the step run schema does not have a deployment_id
            or a step_configuration.
    """
    run_metadata = {
        metadata_schema.key: metadata_schema.to_model()
        for metadata_schema in self.run_metadata
    }

    input_artifacts = {
        artifact.name: artifact.artifact_version.to_model()
        for artifact in self.input_artifacts
    }

    output_artifacts = {
        artifact.name: artifact.artifact_version.to_model()
        for artifact in self.output_artifacts
    }

    if self.deployment is not None:
        full_step_config = Step.parse_obj(
            json.loads(self.deployment.step_configurations)[self.name]
        )
    elif self.step_configuration is not None:
        full_step_config = Step.parse_raw(self.step_configuration)
    else:
        raise RuntimeError(
            "Step run model creation has failed. Each step run entry "
            "should either have a deployment_id or step_configuration."
        )

    body = StepRunResponseBody(
        user=self.user.to_model() if self.user else None,
        status=self.status,
        inputs=input_artifacts,
        outputs=output_artifacts,
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = StepRunResponseMetadata(
            workspace=self.workspace.to_model(),
            config=full_step_config.config,
            spec=full_step_config.spec,
            cache_key=self.cache_key,
            code_hash=self.code_hash,
            docstring=self.docstring,
            source_code=self.source_code,
            start_time=self.start_time,
            end_time=self.end_time,
            logs=self.logs.to_model() if self.logs else None,
            deployment_id=self.deployment_id,
            pipeline_run_id=self.pipeline_run_id,
            original_step_run_id=self.original_step_run_id,
            parent_step_ids=[p.parent_id for p in self.parents],
            run_metadata=run_metadata,
        )
    return StepRunResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update(self, step_update)

Update a step run schema with a step run update model.

Parameters:

Name Type Description Default
step_update StepRunUpdate

The step run update model.

required

Returns:

Type Description
StepRunSchema

The updated step run schema.

Source code in zenml/zen_stores/schemas/step_run_schemas.py
def update(self, step_update: "StepRunUpdate") -> "StepRunSchema":
    """Update a step run schema with a step run update model.

    Args:
        step_update: The step run update model.

    Returns:
        The updated step run schema.
    """
    for key, value in step_update.dict(
        exclude_unset=True, exclude_none=True
    ).items():
        if key == "status":
            self.status = value
        if key == "end_time":
            self.end_time = value

    self.updated = datetime.utcnow()

    return self

tag_schemas

SQLModel implementation of tag tables.

TagResourceSchema (BaseSchema) pydantic-model

SQL Model for tag resource relationship.

Source code in zenml/zen_stores/schemas/tag_schemas.py
class TagResourceSchema(BaseSchema, table=True):
    """SQL Model for tag resource relationship."""

    __tablename__ = "tag_resource"

    tag_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=TagSchema.__tablename__,
        source_column="tag_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    tag: "TagSchema" = Relationship(back_populates="links")
    resource_id: UUID
    resource_type: str = Field(sa_column=Column(VARCHAR(255), nullable=False))
    artifact: List["ArtifactSchema"] = Relationship(
        back_populates="tags",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(TagResourceSchema.resource_type=='{TaggableResourceTypes.ARTIFACT.value}', foreign(TagResourceSchema.resource_id)==ArtifactSchema.id)",
            overlaps="tags,model,artifact_version,model_version",
        ),
    )
    artifact_version: List["ArtifactVersionSchema"] = Relationship(
        back_populates="tags",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(TagResourceSchema.resource_type=='{TaggableResourceTypes.ARTIFACT_VERSION.value}', foreign(TagResourceSchema.resource_id)==ArtifactVersionSchema.id)",
            overlaps="tags,model,artifact,model_version",
        ),
    )
    model: List["ModelSchema"] = Relationship(
        back_populates="tags",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(TagResourceSchema.resource_type=='{TaggableResourceTypes.MODEL.value}', foreign(TagResourceSchema.resource_id)==ModelSchema.id)",
            overlaps="tags,artifact,artifact_version,model_version",
        ),
    )
    model_version: List["ModelVersionSchema"] = Relationship(
        back_populates="tags",
        sa_relationship_kwargs=dict(
            primaryjoin=f"and_(TagResourceSchema.resource_type=='{TaggableResourceTypes.MODEL_VERSION.value}', foreign(TagResourceSchema.resource_id)==ModelVersionSchema.id)",
            overlaps="tags,model,artifact,artifact_version",
        ),
    )

    @classmethod
    def from_request(cls, request: TagResourceRequest) -> "TagResourceSchema":
        """Convert an `TagResourceRequest` to an `TagResourceSchema`.

        Args:
            request: The request model version to convert.

        Returns:
            The converted schema.
        """
        return cls(
            tag_id=request.tag_id,
            resource_id=request.resource_id,
            resource_type=request.resource_type.value,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> TagResourceResponse:
        """Convert an `TagResourceSchema` to an `TagResourceResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `TagResourceResponse`.
        """
        return TagResourceResponse(
            id=self.id,
            body=TagResourceResponseBody(
                tag_id=self.tag_id,
                resource_id=self.resource_id,
                created=self.created,
                updated=self.updated,
                resource_type=TaggableResourceTypes(self.resource_type),
            ),
        )
from_request(request) classmethod

Convert an TagResourceRequest to an TagResourceSchema.

Parameters:

Name Type Description Default
request TagResourceRequest

The request model version to convert.

required

Returns:

Type Description
TagResourceSchema

The converted schema.

Source code in zenml/zen_stores/schemas/tag_schemas.py
@classmethod
def from_request(cls, request: TagResourceRequest) -> "TagResourceSchema":
    """Convert an `TagResourceRequest` to an `TagResourceSchema`.

    Args:
        request: The request model version to convert.

    Returns:
        The converted schema.
    """
    return cls(
        tag_id=request.tag_id,
        resource_id=request.resource_id,
        resource_type=request.resource_type.value,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an TagResourceSchema to an TagResourceResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
TagResourceResponse

The created TagResourceResponse.

Source code in zenml/zen_stores/schemas/tag_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> TagResourceResponse:
    """Convert an `TagResourceSchema` to an `TagResourceResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `TagResourceResponse`.
    """
    return TagResourceResponse(
        id=self.id,
        body=TagResourceResponseBody(
            tag_id=self.tag_id,
            resource_id=self.resource_id,
            created=self.created,
            updated=self.updated,
            resource_type=TaggableResourceTypes(self.resource_type),
        ),
    )
TagSchema (NamedSchema) pydantic-model

SQL Model for tag.

Source code in zenml/zen_stores/schemas/tag_schemas.py
class TagSchema(NamedSchema, table=True):
    """SQL Model for tag."""

    __tablename__ = "tag"

    color: str = Field(sa_column=Column(VARCHAR(255), nullable=False))
    links: List["TagResourceSchema"] = Relationship(
        back_populates="tag",
        sa_relationship_kwargs={"cascade": "delete"},
    )

    @classmethod
    def from_request(cls, request: TagRequest) -> "TagSchema":
        """Convert an `TagRequest` to an `TagSchema`.

        Args:
            request: The request model to convert.

        Returns:
            The converted schema.
        """
        return cls(
            name=request.name,
            color=request.color.value,
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> TagResponse:
        """Convert an `TagSchema` to an `TagResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The created `TagResponse`.
        """
        return TagResponse(
            id=self.id,
            name=self.name,
            body=TagResponseBody(
                created=self.created,
                updated=self.updated,
                color=ColorVariants(self.color),
                tagged_count=len(self.links),
            ),
        )

    def update(self, update: TagUpdate) -> "TagSchema":
        """Updates a `TagSchema` from a `TagUpdate`.

        Args:
            update: The `TagUpdate` to update from.

        Returns:
            The updated `TagSchema`.
        """
        for field, value in update.dict(exclude_unset=True).items():
            setattr(self, field, value)
        self.updated = datetime.utcnow()
        return self
from_request(request) classmethod

Convert an TagRequest to an TagSchema.

Parameters:

Name Type Description Default
request TagRequest

The request model to convert.

required

Returns:

Type Description
TagSchema

The converted schema.

Source code in zenml/zen_stores/schemas/tag_schemas.py
@classmethod
def from_request(cls, request: TagRequest) -> "TagSchema":
    """Convert an `TagRequest` to an `TagSchema`.

    Args:
        request: The request model to convert.

    Returns:
        The converted schema.
    """
    return cls(
        name=request.name,
        color=request.color.value,
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert an TagSchema to an TagResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
TagResponse

The created TagResponse.

Source code in zenml/zen_stores/schemas/tag_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> TagResponse:
    """Convert an `TagSchema` to an `TagResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The created `TagResponse`.
    """
    return TagResponse(
        id=self.id,
        name=self.name,
        body=TagResponseBody(
            created=self.created,
            updated=self.updated,
            color=ColorVariants(self.color),
            tagged_count=len(self.links),
        ),
    )
update(self, update)

Updates a TagSchema from a TagUpdate.

Parameters:

Name Type Description Default
update TagUpdate

The TagUpdate to update from.

required

Returns:

Type Description
TagSchema

The updated TagSchema.

Source code in zenml/zen_stores/schemas/tag_schemas.py
def update(self, update: TagUpdate) -> "TagSchema":
    """Updates a `TagSchema` from a `TagUpdate`.

    Args:
        update: The `TagUpdate` to update from.

    Returns:
        The updated `TagSchema`.
    """
    for field, value in update.dict(exclude_unset=True).items():
        setattr(self, field, value)
    self.updated = datetime.utcnow()
    return self

trigger_schemas

SQL Model Implementations for Triggers.

TriggerExecutionSchema (BaseSchema) pydantic-model

SQL Model for trigger executions.

Source code in zenml/zen_stores/schemas/trigger_schemas.py
class TriggerExecutionSchema(BaseSchema, table=True):
    """SQL Model for trigger executions."""

    __tablename__ = "trigger_execution"

    trigger_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=TriggerSchema.__tablename__,
        source_column="trigger_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    trigger: TriggerSchema = Relationship(back_populates="executions")

    event_metadata: Optional[bytes] = None

    @classmethod
    def from_request(
        cls, request: "TriggerExecutionRequest"
    ) -> "TriggerExecutionSchema":
        """Convert a `TriggerExecutionRequest` to a `TriggerExecutionSchema`.

        Args:
            request: The request model to convert.

        Returns:
            The converted schema.
        """
        return cls(
            trigger_id=request.trigger,
            event_metadata=base64.b64encode(
                json.dumps(request.event_metadata).encode("utf-8")
            ),
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "TriggerExecutionResponse":
        """Converts the schema to a model.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The converted model.
        """
        body = TriggerExecutionResponseBody(
            created=self.created,
            updated=self.updated,
        )
        metadata = None
        if include_metadata:
            metadata = TriggerExecutionResponseMetadata(
                event_metadata=json.loads(
                    base64.b64decode(self.event_metadata).decode()
                )
                if self.event_metadata
                else {},
            )
        resources = None
        if include_resources:
            resources = TriggerExecutionResponseResources(
                trigger=self.trigger.to_model(),
            )

        return TriggerExecutionResponse(
            id=self.id, body=body, metadata=metadata, resources=resources
        )
from_request(request) classmethod

Convert a TriggerExecutionRequest to a TriggerExecutionSchema.

Parameters:

Name Type Description Default
request TriggerExecutionRequest

The request model to convert.

required

Returns:

Type Description
TriggerExecutionSchema

The converted schema.

Source code in zenml/zen_stores/schemas/trigger_schemas.py
@classmethod
def from_request(
    cls, request: "TriggerExecutionRequest"
) -> "TriggerExecutionSchema":
    """Convert a `TriggerExecutionRequest` to a `TriggerExecutionSchema`.

    Args:
        request: The request model to convert.

    Returns:
        The converted schema.
    """
    return cls(
        trigger_id=request.trigger,
        event_metadata=base64.b64encode(
            json.dumps(request.event_metadata).encode("utf-8")
        ),
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Converts the schema to a model.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
TriggerExecutionResponse

The converted model.

Source code in zenml/zen_stores/schemas/trigger_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "TriggerExecutionResponse":
    """Converts the schema to a model.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The converted model.
    """
    body = TriggerExecutionResponseBody(
        created=self.created,
        updated=self.updated,
    )
    metadata = None
    if include_metadata:
        metadata = TriggerExecutionResponseMetadata(
            event_metadata=json.loads(
                base64.b64decode(self.event_metadata).decode()
            )
            if self.event_metadata
            else {},
        )
    resources = None
    if include_resources:
        resources = TriggerExecutionResponseResources(
            trigger=self.trigger.to_model(),
        )

    return TriggerExecutionResponse(
        id=self.id, body=body, metadata=metadata, resources=resources
    )
TriggerSchema (NamedSchema) pydantic-model

SQL Model for triggers.

Source code in zenml/zen_stores/schemas/trigger_schemas.py
class TriggerSchema(NamedSchema, table=True):
    """SQL Model for triggers."""

    __tablename__ = "trigger"

    workspace_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=WorkspaceSchema.__tablename__,
        source_column="workspace_id",
        target_column="id",
        ondelete="CASCADE",
        nullable=False,
    )
    workspace: "WorkspaceSchema" = Relationship(back_populates="triggers")

    user_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="user_id",
        target_column="id",
        ondelete="SET NULL",
        nullable=True,
    )
    user: Optional["UserSchema"] = Relationship(
        back_populates="triggers",
        sa_relationship_kwargs={"foreign_keys": "[TriggerSchema.user_id]"},
    )

    event_source_id: Optional[UUID] = build_foreign_key_field(
        source=__tablename__,
        target=EventSourceSchema.__tablename__,
        source_column="event_source_id",
        target_column="id",
        ondelete="CASCADE",  # TODO: this should be set null and the trigger should be deactivated
        nullable=False,
    )
    event_source: "EventSourceSchema" = Relationship(back_populates="triggers")

    executions: List["TriggerExecutionSchema"] = Relationship(
        back_populates="trigger"
    )

    service_account_id: UUID = build_foreign_key_field(
        source=__tablename__,
        target=UserSchema.__tablename__,
        source_column="service_account_id",
        target_column="id",
        ondelete="CASCADE",  # TODO: this should be set null and the trigger should be deactivated
        nullable=False,
    )
    service_account: UserSchema = Relationship(
        back_populates="auth_triggers",
        sa_relationship_kwargs={
            "foreign_keys": "[TriggerSchema.service_account_id]"
        },
    )
    auth_window: int

    event_filter: bytes

    action: bytes
    action_flavor: str  # <- "builtin"
    action_subtype: str  # <- "PipelineRun"

    description: str = Field(sa_column=Column(TEXT, nullable=True))
    is_active: bool = Field(nullable=False)

    def update(self, trigger_update: "TriggerUpdate") -> "TriggerSchema":
        """Updates a trigger schema with a trigger update model.

        Args:
            trigger_update: `TriggerUpdate` to update the trigger with.

        Returns:
            The updated TriggerSchema.
        """
        for field, value in trigger_update.dict(
            exclude_unset=True,
            exclude_none=True,
        ).items():
            if field == "event_filter":
                self.event_filter = base64.b64encode(
                    json.dumps(
                        trigger_update.event_filter, default=pydantic_encoder
                    ).encode("utf-8")
                )
            elif field == "action":
                self.action = base64.b64encode(
                    json.dumps(
                        trigger_update.action, default=pydantic_encoder
                    ).encode("utf-8")
                )
            else:
                setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    @classmethod
    def from_request(cls, request: "TriggerRequest") -> "TriggerSchema":
        """Convert a `TriggerRequest` to a `TriggerSchema`.

        Args:
            request: The request model to convert.

        Returns:
            The converted schema.

        Raises:
            ValueError: If `auth_window` is not set.
        """
        if request.auth_window is None:
            raise ValueError("auth_window must be set")
        return cls(
            name=request.name,
            workspace_id=request.workspace,
            user_id=request.user,
            action=base64.b64encode(
                json.dumps(request.action, default=pydantic_encoder).encode(
                    "utf-8"
                ),
            ),
            action_flavor=request.action_flavor,
            action_subtype=request.action_subtype,
            event_source_id=request.event_source_id,
            service_account_id=request.service_account_id,
            auth_window=request.auth_window,
            event_filter=base64.b64encode(
                json.dumps(
                    request.event_filter, default=pydantic_encoder
                ).encode("utf-8")
            ),
            description=request.description,
            is_active=True,  # Makes no sense for it to be created inactive
        )

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> "TriggerResponse":
        """Converts the schema to a model.

        Args:
            include_metadata: Flag deciding whether to include the output model(s)
                metadata fields in the response.
            include_resources: Flag deciding whether to include the output model(s)
                metadata fields in the response.
            **kwargs: Keyword arguments to allow schema specific logic

        Returns:
            The converted model.
        """
        from zenml.models import TriggerExecutionResponse

        body = TriggerResponseBody(
            user=self.user.to_model() if self.user else None,
            created=self.created,
            updated=self.updated,
            action_flavor=self.action_flavor,
            action_subtype=self.action_subtype,
            event_source_flavor=self.event_source.flavor,
            is_active=self.is_active,
        )
        metadata = None
        if include_metadata:
            metadata = TriggerResponseMetadata(
                workspace=self.workspace.to_model(),
                event_filter=json.loads(
                    base64.b64decode(self.event_filter).decode()
                ),
                action=json.loads(base64.b64decode(self.action).decode()),
                description=self.description,
                auth_window=self.auth_window,
            )
        resources = None
        if include_resources:
            executions = cast(
                Page[TriggerExecutionResponse],
                get_page_from_list(
                    items_list=self.executions,
                    response_model=TriggerExecutionResponse,
                    include_resources=include_resources,
                    include_metadata=include_metadata,
                ),
            )
            resources = TriggerResponseResources(
                event_source=self.event_source.to_model(),
                service_account=self.service_account.to_model(),
                executions=executions,
            )
        return TriggerResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
            resources=resources,
        )
from_request(request) classmethod

Convert a TriggerRequest to a TriggerSchema.

Parameters:

Name Type Description Default
request TriggerRequest

The request model to convert.

required

Returns:

Type Description
TriggerSchema

The converted schema.

Exceptions:

Type Description
ValueError

If auth_window is not set.

Source code in zenml/zen_stores/schemas/trigger_schemas.py
@classmethod
def from_request(cls, request: "TriggerRequest") -> "TriggerSchema":
    """Convert a `TriggerRequest` to a `TriggerSchema`.

    Args:
        request: The request model to convert.

    Returns:
        The converted schema.

    Raises:
        ValueError: If `auth_window` is not set.
    """
    if request.auth_window is None:
        raise ValueError("auth_window must be set")
    return cls(
        name=request.name,
        workspace_id=request.workspace,
        user_id=request.user,
        action=base64.b64encode(
            json.dumps(request.action, default=pydantic_encoder).encode(
                "utf-8"
            ),
        ),
        action_flavor=request.action_flavor,
        action_subtype=request.action_subtype,
        event_source_id=request.event_source_id,
        service_account_id=request.service_account_id,
        auth_window=request.auth_window,
        event_filter=base64.b64encode(
            json.dumps(
                request.event_filter, default=pydantic_encoder
            ).encode("utf-8")
        ),
        description=request.description,
        is_active=True,  # Makes no sense for it to be created inactive
    )
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Converts the schema to a model.

Parameters:

Name Type Description Default
include_metadata bool

Flag deciding whether to include the output model(s) metadata fields in the response.

False
include_resources bool

Flag deciding whether to include the output model(s) metadata fields in the response.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
TriggerResponse

The converted model.

Source code in zenml/zen_stores/schemas/trigger_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> "TriggerResponse":
    """Converts the schema to a model.

    Args:
        include_metadata: Flag deciding whether to include the output model(s)
            metadata fields in the response.
        include_resources: Flag deciding whether to include the output model(s)
            metadata fields in the response.
        **kwargs: Keyword arguments to allow schema specific logic

    Returns:
        The converted model.
    """
    from zenml.models import TriggerExecutionResponse

    body = TriggerResponseBody(
        user=self.user.to_model() if self.user else None,
        created=self.created,
        updated=self.updated,
        action_flavor=self.action_flavor,
        action_subtype=self.action_subtype,
        event_source_flavor=self.event_source.flavor,
        is_active=self.is_active,
    )
    metadata = None
    if include_metadata:
        metadata = TriggerResponseMetadata(
            workspace=self.workspace.to_model(),
            event_filter=json.loads(
                base64.b64decode(self.event_filter).decode()
            ),
            action=json.loads(base64.b64decode(self.action).decode()),
            description=self.description,
            auth_window=self.auth_window,
        )
    resources = None
    if include_resources:
        executions = cast(
            Page[TriggerExecutionResponse],
            get_page_from_list(
                items_list=self.executions,
                response_model=TriggerExecutionResponse,
                include_resources=include_resources,
                include_metadata=include_metadata,
            ),
        )
        resources = TriggerResponseResources(
            event_source=self.event_source.to_model(),
            service_account=self.service_account.to_model(),
            executions=executions,
        )
    return TriggerResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
        resources=resources,
    )
update(self, trigger_update)

Updates a trigger schema with a trigger update model.

Parameters:

Name Type Description Default
trigger_update TriggerUpdate

TriggerUpdate to update the trigger with.

required

Returns:

Type Description
TriggerSchema

The updated TriggerSchema.

Source code in zenml/zen_stores/schemas/trigger_schemas.py
def update(self, trigger_update: "TriggerUpdate") -> "TriggerSchema":
    """Updates a trigger schema with a trigger update model.

    Args:
        trigger_update: `TriggerUpdate` to update the trigger with.

    Returns:
        The updated TriggerSchema.
    """
    for field, value in trigger_update.dict(
        exclude_unset=True,
        exclude_none=True,
    ).items():
        if field == "event_filter":
            self.event_filter = base64.b64encode(
                json.dumps(
                    trigger_update.event_filter, default=pydantic_encoder
                ).encode("utf-8")
            )
        elif field == "action":
            self.action = base64.b64encode(
                json.dumps(
                    trigger_update.action, default=pydantic_encoder
                ).encode("utf-8")
            )
        else:
            setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self

user_schemas

SQLModel implementation of user tables.

UserSchema (NamedSchema) pydantic-model

SQL Model for users.

Source code in zenml/zen_stores/schemas/user_schemas.py
class UserSchema(NamedSchema, table=True):
    """SQL Model for users."""

    __tablename__ = "user"
    __table_args__ = (UniqueConstraint("name", "is_service_account"),)

    is_service_account: bool = Field(default=False)
    full_name: str
    description: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
    email: Optional[str] = Field(nullable=True)
    active: bool
    password: Optional[str] = Field(nullable=True)
    activation_token: Optional[str] = Field(nullable=True)
    hub_token: Optional[str] = Field(nullable=True)
    email_opted_in: Optional[bool] = Field(nullable=True)
    external_user_id: Optional[UUID] = Field(nullable=True)

    stacks: List["StackSchema"] = Relationship(back_populates="user")
    components: List["StackComponentSchema"] = Relationship(
        back_populates="user",
    )
    flavors: List["FlavorSchema"] = Relationship(back_populates="user")
    event_sources: List["EventSourceSchema"] = Relationship(
        back_populates="user"
    )
    pipelines: List["PipelineSchema"] = Relationship(back_populates="user")
    schedules: List["ScheduleSchema"] = Relationship(
        back_populates="user",
    )
    runs: List["PipelineRunSchema"] = Relationship(back_populates="user")
    step_runs: List["StepRunSchema"] = Relationship(back_populates="user")
    builds: List["PipelineBuildSchema"] = Relationship(back_populates="user")
    artifact_versions: List["ArtifactVersionSchema"] = Relationship(
        back_populates="user"
    )
    run_metadata: List["RunMetadataSchema"] = Relationship(
        back_populates="user"
    )
    secrets: List["SecretSchema"] = Relationship(
        back_populates="user",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    triggers: List["TriggerSchema"] = Relationship(
        back_populates="user",
        sa_relationship_kwargs={
            "cascade": "delete",
            "primaryjoin": "UserSchema.id==TriggerSchema.user_id",
        },
    )
    auth_triggers: List["TriggerSchema"] = Relationship(
        back_populates="service_account",
        sa_relationship_kwargs={
            "cascade": "delete",
            "primaryjoin": "UserSchema.id==TriggerSchema.service_account_id",
        },
    )
    deployments: List["PipelineDeploymentSchema"] = Relationship(
        back_populates="user",
    )
    code_repositories: List["CodeRepositorySchema"] = Relationship(
        back_populates="user",
    )
    service_connectors: List["ServiceConnectorSchema"] = Relationship(
        back_populates="user",
    )
    models: List["ModelSchema"] = Relationship(
        back_populates="user",
    )
    model_versions: List["ModelVersionSchema"] = Relationship(
        back_populates="user",
    )
    model_versions_artifacts_links: List["ModelVersionArtifactSchema"] = (
        Relationship(back_populates="user")
    )
    model_versions_pipeline_runs_links: List[
        "ModelVersionPipelineRunSchema"
    ] = Relationship(back_populates="user")
    auth_devices: List["OAuthDeviceSchema"] = Relationship(
        back_populates="user",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    api_keys: List["APIKeySchema"] = Relationship(
        back_populates="service_account",
        sa_relationship_kwargs={"cascade": "delete"},
    )

    @classmethod
    def from_user_request(cls, model: UserRequest) -> "UserSchema":
        """Create a `UserSchema` from a `UserRequest`.

        Args:
            model: The `UserRequest` from which to create the schema.

        Returns:
            The created `UserSchema`.
        """
        return cls(
            name=model.name,
            full_name=model.full_name,
            active=model.active,
            password=model.create_hashed_password(),
            activation_token=model.create_hashed_activation_token(),
            external_user_id=model.external_user_id,
            email_opted_in=model.email_opted_in,
            email=model.email,
            is_service_account=False,
        )

    @classmethod
    def from_service_account_request(
        cls, model: ServiceAccountRequest
    ) -> "UserSchema":
        """Create a `UserSchema` from a Service Account request.

        Args:
            model: The `ServiceAccountRequest` from which to create the
                schema.

        Returns:
            The created `UserSchema`.
        """
        return cls(
            name=model.name,
            description=model.description or "",
            active=model.active,
            is_service_account=True,
            email_opted_in=False,
            full_name="",
        )

    def update_user(self, user_update: UserUpdate) -> "UserSchema":
        """Update a `UserSchema` from a `UserUpdate`.

        Args:
            user_update: The `UserUpdate` from which to update the schema.

        Returns:
            The updated `UserSchema`.
        """
        for field, value in user_update.dict(exclude_unset=True).items():
            if field == "password":
                setattr(self, field, user_update.create_hashed_password())
            elif field == "activation_token":
                setattr(
                    self, field, user_update.create_hashed_activation_token()
                )
            else:
                setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    def update_service_account(
        self, service_account_update: ServiceAccountUpdate
    ) -> "UserSchema":
        """Update a `UserSchema` from a `ServiceAccountUpdate`.

        Args:
            service_account_update: The `ServiceAccountUpdate` from which
                to update the schema.

        Returns:
            The updated `UserSchema`.
        """
        for field, value in service_account_update.dict(
            exclude_none=True
        ).items():
            setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        include_private: bool = False,
        **kwargs: Any,
    ) -> UserResponse:
        """Convert a `UserSchema` to a `UserResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic
            include_private: Whether to include the user private information
                             this is to limit the amount of data one can get
                             about other users

        Returns:
            The converted `UserResponse`.
        """
        metadata = None
        if include_metadata:
            metadata = UserResponseMetadata(
                email=self.email if include_private else None,
                hub_token=self.hub_token if include_private else None,
                external_user_id=self.external_user_id,
            )

        return UserResponse(
            id=self.id,
            name=self.name,
            body=UserResponseBody(
                active=self.active,
                full_name=self.full_name,
                email_opted_in=self.email_opted_in,
                is_service_account=self.is_service_account,
                created=self.created,
                updated=self.updated,
            ),
            metadata=metadata,
        )

    def to_service_account_model(
        self, include_metadata: bool = False, include_resources: bool = False
    ) -> ServiceAccountResponse:
        """Convert a `UserSchema` to a `ServiceAccountResponse`.

        Args:
             include_metadata: Whether the metadata will be filled.
             include_resources: Whether the resources will be filled.

        Returns:
             The converted `ServiceAccountResponse`.
        """
        metadata = None
        if include_metadata:
            metadata = ServiceAccountResponseMetadata(
                description=self.description or "",
            )

        body = ServiceAccountResponseBody(
            created=self.created,
            updated=self.updated,
            active=self.active,
        )

        return ServiceAccountResponse(
            id=self.id,
            name=self.name,
            body=body,
            metadata=metadata,
        )
from_service_account_request(model) classmethod

Create a UserSchema from a Service Account request.

Parameters:

Name Type Description Default
model ServiceAccountRequest

The ServiceAccountRequest from which to create the schema.

required

Returns:

Type Description
UserSchema

The created UserSchema.

Source code in zenml/zen_stores/schemas/user_schemas.py
@classmethod
def from_service_account_request(
    cls, model: ServiceAccountRequest
) -> "UserSchema":
    """Create a `UserSchema` from a Service Account request.

    Args:
        model: The `ServiceAccountRequest` from which to create the
            schema.

    Returns:
        The created `UserSchema`.
    """
    return cls(
        name=model.name,
        description=model.description or "",
        active=model.active,
        is_service_account=True,
        email_opted_in=False,
        full_name="",
    )
from_user_request(model) classmethod

Create a UserSchema from a UserRequest.

Parameters:

Name Type Description Default
model UserRequest

The UserRequest from which to create the schema.

required

Returns:

Type Description
UserSchema

The created UserSchema.

Source code in zenml/zen_stores/schemas/user_schemas.py
@classmethod
def from_user_request(cls, model: UserRequest) -> "UserSchema":
    """Create a `UserSchema` from a `UserRequest`.

    Args:
        model: The `UserRequest` from which to create the schema.

    Returns:
        The created `UserSchema`.
    """
    return cls(
        name=model.name,
        full_name=model.full_name,
        active=model.active,
        password=model.create_hashed_password(),
        activation_token=model.create_hashed_activation_token(),
        external_user_id=model.external_user_id,
        email_opted_in=model.email_opted_in,
        email=model.email,
        is_service_account=False,
    )
to_model(self, include_metadata=False, include_resources=False, include_private=False, **kwargs)

Convert a UserSchema to a UserResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}
include_private bool

Whether to include the user private information this is to limit the amount of data one can get about other users

False

Returns:

Type Description
UserResponse

The converted UserResponse.

Source code in zenml/zen_stores/schemas/user_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    include_private: bool = False,
    **kwargs: Any,
) -> UserResponse:
    """Convert a `UserSchema` to a `UserResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic
        include_private: Whether to include the user private information
                         this is to limit the amount of data one can get
                         about other users

    Returns:
        The converted `UserResponse`.
    """
    metadata = None
    if include_metadata:
        metadata = UserResponseMetadata(
            email=self.email if include_private else None,
            hub_token=self.hub_token if include_private else None,
            external_user_id=self.external_user_id,
        )

    return UserResponse(
        id=self.id,
        name=self.name,
        body=UserResponseBody(
            active=self.active,
            full_name=self.full_name,
            email_opted_in=self.email_opted_in,
            is_service_account=self.is_service_account,
            created=self.created,
            updated=self.updated,
        ),
        metadata=metadata,
    )
to_service_account_model(self, include_metadata=False, include_resources=False)

Convert a UserSchema to a ServiceAccountResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False

Returns:

Type Description
ServiceAccountResponse

The converted ServiceAccountResponse.

Source code in zenml/zen_stores/schemas/user_schemas.py
def to_service_account_model(
    self, include_metadata: bool = False, include_resources: bool = False
) -> ServiceAccountResponse:
    """Convert a `UserSchema` to a `ServiceAccountResponse`.

    Args:
         include_metadata: Whether the metadata will be filled.
         include_resources: Whether the resources will be filled.

    Returns:
         The converted `ServiceAccountResponse`.
    """
    metadata = None
    if include_metadata:
        metadata = ServiceAccountResponseMetadata(
            description=self.description or "",
        )

    body = ServiceAccountResponseBody(
        created=self.created,
        updated=self.updated,
        active=self.active,
    )

    return ServiceAccountResponse(
        id=self.id,
        name=self.name,
        body=body,
        metadata=metadata,
    )
update_service_account(self, service_account_update)

Update a UserSchema from a ServiceAccountUpdate.

Parameters:

Name Type Description Default
service_account_update ServiceAccountUpdate

The ServiceAccountUpdate from which to update the schema.

required

Returns:

Type Description
UserSchema

The updated UserSchema.

Source code in zenml/zen_stores/schemas/user_schemas.py
def update_service_account(
    self, service_account_update: ServiceAccountUpdate
) -> "UserSchema":
    """Update a `UserSchema` from a `ServiceAccountUpdate`.

    Args:
        service_account_update: The `ServiceAccountUpdate` from which
            to update the schema.

    Returns:
        The updated `UserSchema`.
    """
    for field, value in service_account_update.dict(
        exclude_none=True
    ).items():
        setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self
update_user(self, user_update)

Update a UserSchema from a UserUpdate.

Parameters:

Name Type Description Default
user_update UserUpdate

The UserUpdate from which to update the schema.

required

Returns:

Type Description
UserSchema

The updated UserSchema.

Source code in zenml/zen_stores/schemas/user_schemas.py
def update_user(self, user_update: UserUpdate) -> "UserSchema":
    """Update a `UserSchema` from a `UserUpdate`.

    Args:
        user_update: The `UserUpdate` from which to update the schema.

    Returns:
        The updated `UserSchema`.
    """
    for field, value in user_update.dict(exclude_unset=True).items():
        if field == "password":
            setattr(self, field, user_update.create_hashed_password())
        elif field == "activation_token":
            setattr(
                self, field, user_update.create_hashed_activation_token()
            )
        else:
            setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self

utils

Utils for schemas.

Page[BaseResponse] (Page) pydantic-model
Config

Pydantic configuration class.

Source code in zenml/zen_stores/schemas/utils.py
class Config:
    """Pydantic configuration class."""

    # This is needed to allow the REST API server to unpack SecretStr
    # values correctly before sending them to the client.
    json_encoders = {
        SecretStr: lambda v: v.get_secret_value() if v else None
    }
__json_encoder__(obj) special staticmethod

partial(func, args, *keywords) - new function with partial application of the given arguments and keywords.

get_page_from_list(items_list, response_model, size=5, page=1, include_resources=False, include_metadata=False)

Converts list of schemas into page of response models.

Parameters:

Name Type Description Default
items_list List[~S]

List of schemas

required
response_model Type[zenml.models.v2.base.base.BaseResponse]

Response model

required
size int

Page size

5
page int

Page number

1
include_metadata bool

Whether metadata should be included in response models

False
include_resources bool

Whether resources should be included in response models

False

Returns:

Type Description
Page[BaseResponse]

A page of list items.

Source code in zenml/zen_stores/schemas/utils.py
def get_page_from_list(
    items_list: List[S],
    response_model: Type[BaseResponse],  # type: ignore[type-arg]
    size: int = 5,
    page: int = 1,
    include_resources: bool = False,
    include_metadata: bool = False,
) -> Page[BaseResponse]:  # type: ignore[type-arg]
    """Converts list of schemas into page of response models.

    Args:
        items_list: List of schemas
        response_model: Response model
        size: Page size
        page: Page number
        include_metadata: Whether metadata should be included in response models
        include_resources: Whether resources should be included in response models

    Returns:
        A page of list items.
    """
    total = len(items_list)
    if total == 0:
        total_pages = 1
    else:
        total_pages = math.ceil(total / size)

    start = (page - 1) * size
    end = start + size

    page_items = [
        item.to_model(
            include_metadata=include_metadata,
            include_resources=include_resources,
        )
        for item in items_list[start:end]
    ]
    return Page[response_model](  # type: ignore[valid-type]
        index=page,
        max_size=size,
        total_pages=total_pages,
        total=total,
        items=page_items,
    )

workspace_schemas

SQL Model Implementations for Workspaces.

WorkspaceSchema (NamedSchema) pydantic-model

SQL Model for workspaces.

Source code in zenml/zen_stores/schemas/workspace_schemas.py
class WorkspaceSchema(NamedSchema, table=True):
    """SQL Model for workspaces."""

    __tablename__ = "workspace"

    description: str

    stacks: List["StackSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    components: List["StackComponentSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    flavors: List["FlavorSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    pipelines: List["PipelineSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    schedules: List["ScheduleSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    runs: List["PipelineRunSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    step_runs: List["StepRunSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    builds: List["PipelineBuildSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    artifact_versions: List["ArtifactVersionSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    run_metadata: List["RunMetadataSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    secrets: List["SecretSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    triggers: List["TriggerSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    event_sources: List["EventSourceSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )

    deployments: List["PipelineDeploymentSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    code_repositories: List["CodeRepositorySchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    service_connectors: List["ServiceConnectorSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    models: List["ModelSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    model_versions: List["ModelVersionSchema"] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )
    model_versions_artifacts_links: List["ModelVersionArtifactSchema"] = (
        Relationship(
            back_populates="workspace",
            sa_relationship_kwargs={"cascade": "delete"},
        )
    )
    model_versions_pipeline_runs_links: List[
        "ModelVersionPipelineRunSchema"
    ] = Relationship(
        back_populates="workspace",
        sa_relationship_kwargs={"cascade": "delete"},
    )

    @classmethod
    def from_request(cls, workspace: WorkspaceRequest) -> "WorkspaceSchema":
        """Create a `WorkspaceSchema` from a `WorkspaceResponse`.

        Args:
            workspace: The `WorkspaceResponse` from which to create the schema.

        Returns:
            The created `WorkspaceSchema`.
        """
        return cls(name=workspace.name, description=workspace.description)

    def update(self, workspace_update: WorkspaceUpdate) -> "WorkspaceSchema":
        """Update a `WorkspaceSchema` from a `WorkspaceUpdate`.

        Args:
            workspace_update: The `WorkspaceUpdate` from which to update the
                schema.

        Returns:
            The updated `WorkspaceSchema`.
        """
        for field, value in workspace_update.dict(exclude_unset=True).items():
            setattr(self, field, value)

        self.updated = datetime.utcnow()
        return self

    def to_model(
        self,
        include_metadata: bool = False,
        include_resources: bool = False,
        **kwargs: Any,
    ) -> WorkspaceResponse:
        """Convert a `WorkspaceSchema` to a `WorkspaceResponse`.

        Args:
            include_metadata: Whether the metadata will be filled.
            include_resources: Whether the resources will be filled.
            **kwargs: Keyword arguments to allow schema specific logic


        Returns:
            The converted `WorkspaceResponseModel`.
        """
        metadata = None
        if include_metadata:
            metadata = WorkspaceResponseMetadata(
                description=self.description,
            )
        return WorkspaceResponse(
            id=self.id,
            name=self.name,
            body=WorkspaceResponseBody(
                created=self.created,
                updated=self.updated,
            ),
            metadata=metadata,
        )
from_request(workspace) classmethod

Create a WorkspaceSchema from a WorkspaceResponse.

Parameters:

Name Type Description Default
workspace WorkspaceRequest

The WorkspaceResponse from which to create the schema.

required

Returns:

Type Description
WorkspaceSchema

The created WorkspaceSchema.

Source code in zenml/zen_stores/schemas/workspace_schemas.py
@classmethod
def from_request(cls, workspace: WorkspaceRequest) -> "WorkspaceSchema":
    """Create a `WorkspaceSchema` from a `WorkspaceResponse`.

    Args:
        workspace: The `WorkspaceResponse` from which to create the schema.

    Returns:
        The created `WorkspaceSchema`.
    """
    return cls(name=workspace.name, description=workspace.description)
to_model(self, include_metadata=False, include_resources=False, **kwargs)

Convert a WorkspaceSchema to a WorkspaceResponse.

Parameters:

Name Type Description Default
include_metadata bool

Whether the metadata will be filled.

False
include_resources bool

Whether the resources will be filled.

False
**kwargs Any

Keyword arguments to allow schema specific logic

{}

Returns:

Type Description
WorkspaceResponse

The converted WorkspaceResponseModel.

Source code in zenml/zen_stores/schemas/workspace_schemas.py
def to_model(
    self,
    include_metadata: bool = False,
    include_resources: bool = False,
    **kwargs: Any,
) -> WorkspaceResponse:
    """Convert a `WorkspaceSchema` to a `WorkspaceResponse`.

    Args:
        include_metadata: Whether the metadata will be filled.
        include_resources: Whether the resources will be filled.
        **kwargs: Keyword arguments to allow schema specific logic


    Returns:
        The converted `WorkspaceResponseModel`.
    """
    metadata = None
    if include_metadata:
        metadata = WorkspaceResponseMetadata(
            description=self.description,
        )
    return WorkspaceResponse(
        id=self.id,
        name=self.name,
        body=WorkspaceResponseBody(
            created=self.created,
            updated=self.updated,
        ),
        metadata=metadata,
    )
update(self, workspace_update)

Update a WorkspaceSchema from a WorkspaceUpdate.

Parameters:

Name Type Description Default
workspace_update WorkspaceUpdate

The WorkspaceUpdate from which to update the schema.

required

Returns:

Type Description
WorkspaceSchema

The updated WorkspaceSchema.

Source code in zenml/zen_stores/schemas/workspace_schemas.py
def update(self, workspace_update: WorkspaceUpdate) -> "WorkspaceSchema":
    """Update a `WorkspaceSchema` from a `WorkspaceUpdate`.

    Args:
        workspace_update: The `WorkspaceUpdate` from which to update the
            schema.

    Returns:
        The updated `WorkspaceSchema`.
    """
    for field, value in workspace_update.dict(exclude_unset=True).items():
        setattr(self, field, value)

    self.updated = datetime.utcnow()
    return self

secrets_stores special

Centralized secrets management.

aws_secrets_store

AWS Secrets Store implementation.

AWSSecretsStore (ServiceConnectorSecretsStore) pydantic-model

Secrets store implementation that uses the AWS Secrets Manager API.

This secrets store implementation uses the AWS Secrets Manager API to store secrets. It allows a single AWS Secrets Manager region "instance" to be shared with other ZenML deployments as well as other third party users and applications.

Here are some implementation highlights:

  • the name/ID of an AWS secret is derived from the ZenML secret UUID and a zenml prefix in the form zenml/{zenml_secret_uuid}. This clearly identifies a secret as being managed by ZenML in the AWS console.

  • the Secrets Store also uses AWS secret tags to store additional metadata associated with a ZenML secret. The zenml tag in particular is used to identify and group all secrets that belong to the same ZenML deployment.

  • all secret key-values configured in a ZenML secret are stored as a single JSON string value in the AWS secret value.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
class AWSSecretsStore(ServiceConnectorSecretsStore):
    """Secrets store implementation that uses the AWS Secrets Manager API.

    This secrets store implementation uses the AWS Secrets Manager API to
    store secrets. It allows a single AWS Secrets Manager region "instance" to
    be shared with other ZenML deployments as well as other third party users
    and applications.

    Here are some implementation highlights:

    * the name/ID of an AWS secret is derived from the ZenML secret UUID and a
    `zenml` prefix in the form `zenml/{zenml_secret_uuid}`. This clearly
    identifies a secret as being managed by ZenML in the AWS console.

    * the Secrets Store also uses AWS secret tags to store additional
    metadata associated with a ZenML secret. The `zenml` tag in particular is
    used to identify and group all secrets that belong to the same ZenML
    deployment.

    * all secret key-values configured in a ZenML secret are stored as a single
    JSON string value in the AWS secret value.
    """

    config: AWSSecretsStoreConfiguration
    TYPE: ClassVar[SecretsStoreType] = SecretsStoreType.AWS
    CONFIG_TYPE: ClassVar[Type[ServiceConnectorSecretsStoreConfiguration]] = (
        AWSSecretsStoreConfiguration
    )
    SERVICE_CONNECTOR_TYPE: ClassVar[str] = AWS_CONNECTOR_TYPE
    SERVICE_CONNECTOR_RESOURCE_TYPE: ClassVar[str] = AWS_RESOURCE_TYPE

    # ====================================
    # Secrets Store interface implementation
    # ====================================

    # --------------------------------
    # Initialization and configuration
    # --------------------------------

    def _initialize_client_from_connector(self, client: Any) -> Any:
        """Initialize the AWS Secrets Manager client from the service connector client.

        Args:
            client: The authenticated client object returned by the service
                connector.

        Returns:
            The AWS Secrets Manager client.
        """
        assert isinstance(client, boto3.Session)
        return client.client(
            "secretsmanager",
            region_name=self.config.region,
        )

    # ------
    # Secrets
    # ------

    @staticmethod
    def _get_aws_secret_id(
        secret_id: UUID,
    ) -> str:
        """Get the AWS secret ID corresponding to a ZenML secret ID.

        The convention used for AWS secret names is to use the ZenML
        secret UUID prefixed with `zenml` as the AWS secret name,
        i.e. `zenml/<secret_uuid>`.

        Args:
            secret_id: The ZenML secret ID.

        Returns:
            The AWS secret name.
        """
        return f"{AWS_ZENML_SECRET_NAME_PREFIX}/{str(secret_id)}"

    @staticmethod
    def _get_aws_secret_tags(
        metadata: Dict[str, str],
    ) -> List[Dict[str, str]]:
        """Convert ZenML secret metadata to AWS secret tags.

        Args:
            metadata: The ZenML secret metadata.

        Returns:
            The AWS secret tags.
        """
        aws_tags: List[Dict[str, str]] = []
        for k, v in metadata.items():
            aws_tags.append(
                {
                    "Key": k,
                    "Value": str(v),
                }
            )

        return aws_tags

    def store_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Store secret values for a new secret.

        Args:
            secret_id: ID of the secret.
            secret_values: Values for the secret.

        Raises:
            RuntimeError: If the AWS Secrets Manager API returns an unexpected
                error.
        """
        aws_secret_id = self._get_aws_secret_id(secret_id)
        secret_value = json.dumps(secret_values)

        # Convert the ZenML secret metadata to AWS tags
        metadata = self._get_secret_metadata(secret_id=secret_id)
        tags = self._get_aws_secret_tags(metadata)

        try:
            self.client.create_secret(
                Name=aws_secret_id,
                SecretString=secret_value,
                Tags=tags,
            )
        except ClientError as e:
            raise RuntimeError(f"Error creating secret: {e}")

        logger.debug(f"Created AWS secret: {aws_secret_id}")

    def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
        """Get the secret values for an existing secret.

        Args:
            secret_id: ID of the secret.

        Returns:
            The secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: If the AWS Secrets Manager API returns an unexpected
                error.
        """
        aws_secret_id = self._get_aws_secret_id(secret_id)

        try:
            get_secret_value_response = self.client.get_secret_value(
                SecretId=aws_secret_id
            )
            # We need a separate AWS API call to get the AWS secret tags which
            # contain the ZenML secret metadata, since the get_secret_ value API
            # does not return them.
            describe_secret_response = self.client.describe_secret(
                SecretId=aws_secret_id
            )
        except ClientError as e:
            if e.response["Error"]["Code"] == "ResourceNotFoundException" or (
                e.response["Error"]["Code"] == "InvalidRequestException"
                and "marked for deletion" in e.response["Error"]["Message"]
            ):
                raise KeyError(
                    f"Can't find the secret values for secret ID '{secret_id}' "
                    f"in the secrets store back-end: {str(e)}"
                ) from e

            raise RuntimeError(
                f"Error fetching secret with ID {secret_id} {e}"
            )

        # Convert the AWS secret tags to a metadata dictionary.
        metadata: Dict[str, str] = {
            tag["Key"]: tag["Value"]
            for tag in describe_secret_response["Tags"]
        }

        # The _verify_secret_metadata method raises a KeyError if the
        # secret is not valid or does not belong to this server. Here we
        # simply pass the exception up the stack, as if the secret was not found
        # in the first place.
        self._verify_secret_metadata(
            secret_id=secret_id,
            metadata=metadata,
        )

        values = get_secret_value_response["SecretString"]

        logger.debug(f"Fetched AWS secret: {aws_secret_id}")

        secret_values = json.loads(values)

        if not isinstance(secret_values, dict):
            raise RuntimeError(
                f"AWS secret values for secret ID {aws_secret_id} could not be "
                "decoded: expected a dictionary."
            )

        return secret_values

    def update_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Updates secret values for an existing secret.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_values: The new secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: If the AWS Secrets Manager API returns an unexpected
                error.
        """
        aws_secret_id = self._get_aws_secret_id(secret_id)
        secret_value = json.dumps(secret_values)

        # Convert the ZenML secret metadata to AWS tags
        metadata = self._get_secret_metadata(secret_id)
        tags = self._get_aws_secret_tags(metadata)

        try:
            # One call to update the secret values
            self.client.put_secret_value(
                SecretId=aws_secret_id,
                SecretString=secret_value,
            )
            # Another call to update the tags
            self.client.tag_resource(
                SecretId=aws_secret_id,
                Tags=tags,
            )
        except ClientError as e:
            if e.response["Error"]["Code"] == "ResourceNotFoundException":
                raise KeyError(f"Secret with ID {secret_id} not found")
            raise RuntimeError(f"Error updating secret: {e}")

        logger.debug(f"Updated AWS secret: {aws_secret_id}")

    def delete_secret_values(self, secret_id: UUID) -> None:
        """Deletes secret values for an existing secret.

        Args:
            secret_id: The ID of the secret.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: If the AWS Secrets Manager API returns an unexpected
                error.
        """
        aws_secret_id = self._get_aws_secret_id(secret_id)

        try:
            self.client.delete_secret(
                SecretId=aws_secret_id,
                # We set this to force immediate deletion of the AWS secret
                # instead of waiting for the recovery window to expire.
                ForceDeleteWithoutRecovery=True,
            )
        except ClientError as e:
            if e.response["Error"]["Code"] == "ResourceNotFoundException":
                raise KeyError(f"Secret with ID {secret_id} not found")

            if (
                e.response["Error"]["Code"] == "InvalidRequestException"
                and "marked for deletion" in e.response["Error"]["Message"]
            ):
                raise KeyError(f"Secret with ID {secret_id} not found")

            raise RuntimeError(
                f"Error deleting secret with ID {secret_id}: {e}"
            )

        logger.debug(f"Deleted AWS secret: {aws_secret_id}")
CONFIG_TYPE (ServiceConnectorSecretsStoreConfiguration) pydantic-model

AWS secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
class AWSSecretsStoreConfiguration(ServiceConnectorSecretsStoreConfiguration):
    """AWS secrets store configuration.

    Attributes:
        type: The type of the store.
    """

    type: SecretsStoreType = SecretsStoreType.AWS

    @property
    def region(self) -> str:
        """The AWS region to use.

        Returns:
            The AWS region to use.

        Raises:
            ValueError: If the region is not configured.
        """
        region = self.auth_config.get("region")
        if region:
            return str(region)

        raise ValueError("AWS `region` must be specified in the auth_config.")

    @root_validator(pre=True)
    def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Populate the connector configuration from legacy attributes.

        Args:
            values: Dict representing user-specified runtime settings.

        Returns:
            Validated settings.
        """
        # Search for legacy attributes and populate the connector configuration
        # from them, if they exist.
        if values.get("region_name"):
            if not values.get("aws_access_key_id") or not values.get(
                "aws_secret_access_key"
            ):
                logger.warning(
                    "The `region_name` AWS secrets store attribute is deprecated "
                    "and will be removed in a future version of ZenML. Please use "
                    "the `auth_method` and `auth_config` attributes instead. "
                    "Using an implicit authentication method for AWS Secrets."
                )
                values["auth_method"] = AWSAuthenticationMethods.IMPLICIT
                values["auth_config"] = dict(
                    region=values.get("region_name"),
                )
            else:
                logger.warning(
                    "The `aws_access_key_id`, `aws_secret_access_key` and "
                    "`region_name` AWS secrets store attributes are deprecated and "
                    "will be removed in a future version of ZenML. Please use the "
                    "`auth_method` and `auth_config` attributes instead."
                )
                values["auth_method"] = AWSAuthenticationMethods.SECRET_KEY
                values["auth_config"] = dict(
                    aws_access_key_id=values.get("aws_access_key_id"),
                    aws_secret_access_key=values.get("aws_secret_access_key"),
                    region=values.get("region_name"),
                )

        return values

    class Config:
        """Pydantic configuration class."""

        # Allow extra attributes set in the class.
        extra = "allow"
region: str property readonly

The AWS region to use.

Returns:

Type Description
str

The AWS region to use.

Exceptions:

Type Description
ValueError

If the region is not configured.

Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Allow extra attributes set in the class.
    extra = "allow"
populate_config(values) classmethod

Populate the connector configuration from legacy attributes.

Parameters:

Name Type Description Default
values Dict[str, Any]

Dict representing user-specified runtime settings.

required

Returns:

Type Description
Dict[str, Any]

Validated settings.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
@root_validator(pre=True)
def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Populate the connector configuration from legacy attributes.

    Args:
        values: Dict representing user-specified runtime settings.

    Returns:
        Validated settings.
    """
    # Search for legacy attributes and populate the connector configuration
    # from them, if they exist.
    if values.get("region_name"):
        if not values.get("aws_access_key_id") or not values.get(
            "aws_secret_access_key"
        ):
            logger.warning(
                "The `region_name` AWS secrets store attribute is deprecated "
                "and will be removed in a future version of ZenML. Please use "
                "the `auth_method` and `auth_config` attributes instead. "
                "Using an implicit authentication method for AWS Secrets."
            )
            values["auth_method"] = AWSAuthenticationMethods.IMPLICIT
            values["auth_config"] = dict(
                region=values.get("region_name"),
            )
        else:
            logger.warning(
                "The `aws_access_key_id`, `aws_secret_access_key` and "
                "`region_name` AWS secrets store attributes are deprecated and "
                "will be removed in a future version of ZenML. Please use the "
                "`auth_method` and `auth_config` attributes instead."
            )
            values["auth_method"] = AWSAuthenticationMethods.SECRET_KEY
            values["auth_config"] = dict(
                aws_access_key_id=values.get("aws_access_key_id"),
                aws_secret_access_key=values.get("aws_secret_access_key"),
                region=values.get("region_name"),
            )

    return values
delete_secret_values(self, secret_id)

Deletes secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

If the AWS Secrets Manager API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
def delete_secret_values(self, secret_id: UUID) -> None:
    """Deletes secret values for an existing secret.

    Args:
        secret_id: The ID of the secret.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: If the AWS Secrets Manager API returns an unexpected
            error.
    """
    aws_secret_id = self._get_aws_secret_id(secret_id)

    try:
        self.client.delete_secret(
            SecretId=aws_secret_id,
            # We set this to force immediate deletion of the AWS secret
            # instead of waiting for the recovery window to expire.
            ForceDeleteWithoutRecovery=True,
        )
    except ClientError as e:
        if e.response["Error"]["Code"] == "ResourceNotFoundException":
            raise KeyError(f"Secret with ID {secret_id} not found")

        if (
            e.response["Error"]["Code"] == "InvalidRequestException"
            and "marked for deletion" in e.response["Error"]["Message"]
        ):
            raise KeyError(f"Secret with ID {secret_id} not found")

        raise RuntimeError(
            f"Error deleting secret with ID {secret_id}: {e}"
        )

    logger.debug(f"Deleted AWS secret: {aws_secret_id}")
get_secret_values(self, secret_id)

Get the secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required

Returns:

Type Description
Dict[str, str]

The secret values.

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

If the AWS Secrets Manager API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
    """Get the secret values for an existing secret.

    Args:
        secret_id: ID of the secret.

    Returns:
        The secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: If the AWS Secrets Manager API returns an unexpected
            error.
    """
    aws_secret_id = self._get_aws_secret_id(secret_id)

    try:
        get_secret_value_response = self.client.get_secret_value(
            SecretId=aws_secret_id
        )
        # We need a separate AWS API call to get the AWS secret tags which
        # contain the ZenML secret metadata, since the get_secret_ value API
        # does not return them.
        describe_secret_response = self.client.describe_secret(
            SecretId=aws_secret_id
        )
    except ClientError as e:
        if e.response["Error"]["Code"] == "ResourceNotFoundException" or (
            e.response["Error"]["Code"] == "InvalidRequestException"
            and "marked for deletion" in e.response["Error"]["Message"]
        ):
            raise KeyError(
                f"Can't find the secret values for secret ID '{secret_id}' "
                f"in the secrets store back-end: {str(e)}"
            ) from e

        raise RuntimeError(
            f"Error fetching secret with ID {secret_id} {e}"
        )

    # Convert the AWS secret tags to a metadata dictionary.
    metadata: Dict[str, str] = {
        tag["Key"]: tag["Value"]
        for tag in describe_secret_response["Tags"]
    }

    # The _verify_secret_metadata method raises a KeyError if the
    # secret is not valid or does not belong to this server. Here we
    # simply pass the exception up the stack, as if the secret was not found
    # in the first place.
    self._verify_secret_metadata(
        secret_id=secret_id,
        metadata=metadata,
    )

    values = get_secret_value_response["SecretString"]

    logger.debug(f"Fetched AWS secret: {aws_secret_id}")

    secret_values = json.loads(values)

    if not isinstance(secret_values, dict):
        raise RuntimeError(
            f"AWS secret values for secret ID {aws_secret_id} could not be "
            "decoded: expected a dictionary."
        )

    return secret_values
store_secret_values(self, secret_id, secret_values)

Store secret values for a new secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required
secret_values Dict[str, str]

Values for the secret.

required

Exceptions:

Type Description
RuntimeError

If the AWS Secrets Manager API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
def store_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Store secret values for a new secret.

    Args:
        secret_id: ID of the secret.
        secret_values: Values for the secret.

    Raises:
        RuntimeError: If the AWS Secrets Manager API returns an unexpected
            error.
    """
    aws_secret_id = self._get_aws_secret_id(secret_id)
    secret_value = json.dumps(secret_values)

    # Convert the ZenML secret metadata to AWS tags
    metadata = self._get_secret_metadata(secret_id=secret_id)
    tags = self._get_aws_secret_tags(metadata)

    try:
        self.client.create_secret(
            Name=aws_secret_id,
            SecretString=secret_value,
            Tags=tags,
        )
    except ClientError as e:
        raise RuntimeError(f"Error creating secret: {e}")

    logger.debug(f"Created AWS secret: {aws_secret_id}")
update_secret_values(self, secret_id, secret_values)

Updates secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_values Dict[str, str]

The new secret values.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

If the AWS Secrets Manager API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
def update_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Updates secret values for an existing secret.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_values: The new secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: If the AWS Secrets Manager API returns an unexpected
            error.
    """
    aws_secret_id = self._get_aws_secret_id(secret_id)
    secret_value = json.dumps(secret_values)

    # Convert the ZenML secret metadata to AWS tags
    metadata = self._get_secret_metadata(secret_id)
    tags = self._get_aws_secret_tags(metadata)

    try:
        # One call to update the secret values
        self.client.put_secret_value(
            SecretId=aws_secret_id,
            SecretString=secret_value,
        )
        # Another call to update the tags
        self.client.tag_resource(
            SecretId=aws_secret_id,
            Tags=tags,
        )
    except ClientError as e:
        if e.response["Error"]["Code"] == "ResourceNotFoundException":
            raise KeyError(f"Secret with ID {secret_id} not found")
        raise RuntimeError(f"Error updating secret: {e}")

    logger.debug(f"Updated AWS secret: {aws_secret_id}")
AWSSecretsStoreConfiguration (ServiceConnectorSecretsStoreConfiguration) pydantic-model

AWS secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
class AWSSecretsStoreConfiguration(ServiceConnectorSecretsStoreConfiguration):
    """AWS secrets store configuration.

    Attributes:
        type: The type of the store.
    """

    type: SecretsStoreType = SecretsStoreType.AWS

    @property
    def region(self) -> str:
        """The AWS region to use.

        Returns:
            The AWS region to use.

        Raises:
            ValueError: If the region is not configured.
        """
        region = self.auth_config.get("region")
        if region:
            return str(region)

        raise ValueError("AWS `region` must be specified in the auth_config.")

    @root_validator(pre=True)
    def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Populate the connector configuration from legacy attributes.

        Args:
            values: Dict representing user-specified runtime settings.

        Returns:
            Validated settings.
        """
        # Search for legacy attributes and populate the connector configuration
        # from them, if they exist.
        if values.get("region_name"):
            if not values.get("aws_access_key_id") or not values.get(
                "aws_secret_access_key"
            ):
                logger.warning(
                    "The `region_name` AWS secrets store attribute is deprecated "
                    "and will be removed in a future version of ZenML. Please use "
                    "the `auth_method` and `auth_config` attributes instead. "
                    "Using an implicit authentication method for AWS Secrets."
                )
                values["auth_method"] = AWSAuthenticationMethods.IMPLICIT
                values["auth_config"] = dict(
                    region=values.get("region_name"),
                )
            else:
                logger.warning(
                    "The `aws_access_key_id`, `aws_secret_access_key` and "
                    "`region_name` AWS secrets store attributes are deprecated and "
                    "will be removed in a future version of ZenML. Please use the "
                    "`auth_method` and `auth_config` attributes instead."
                )
                values["auth_method"] = AWSAuthenticationMethods.SECRET_KEY
                values["auth_config"] = dict(
                    aws_access_key_id=values.get("aws_access_key_id"),
                    aws_secret_access_key=values.get("aws_secret_access_key"),
                    region=values.get("region_name"),
                )

        return values

    class Config:
        """Pydantic configuration class."""

        # Allow extra attributes set in the class.
        extra = "allow"
region: str property readonly

The AWS region to use.

Returns:

Type Description
str

The AWS region to use.

Exceptions:

Type Description
ValueError

If the region is not configured.

Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Allow extra attributes set in the class.
    extra = "allow"
populate_config(values) classmethod

Populate the connector configuration from legacy attributes.

Parameters:

Name Type Description Default
values Dict[str, Any]

Dict representing user-specified runtime settings.

required

Returns:

Type Description
Dict[str, Any]

Validated settings.

Source code in zenml/zen_stores/secrets_stores/aws_secrets_store.py
@root_validator(pre=True)
def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Populate the connector configuration from legacy attributes.

    Args:
        values: Dict representing user-specified runtime settings.

    Returns:
        Validated settings.
    """
    # Search for legacy attributes and populate the connector configuration
    # from them, if they exist.
    if values.get("region_name"):
        if not values.get("aws_access_key_id") or not values.get(
            "aws_secret_access_key"
        ):
            logger.warning(
                "The `region_name` AWS secrets store attribute is deprecated "
                "and will be removed in a future version of ZenML. Please use "
                "the `auth_method` and `auth_config` attributes instead. "
                "Using an implicit authentication method for AWS Secrets."
            )
            values["auth_method"] = AWSAuthenticationMethods.IMPLICIT
            values["auth_config"] = dict(
                region=values.get("region_name"),
            )
        else:
            logger.warning(
                "The `aws_access_key_id`, `aws_secret_access_key` and "
                "`region_name` AWS secrets store attributes are deprecated and "
                "will be removed in a future version of ZenML. Please use the "
                "`auth_method` and `auth_config` attributes instead."
            )
            values["auth_method"] = AWSAuthenticationMethods.SECRET_KEY
            values["auth_config"] = dict(
                aws_access_key_id=values.get("aws_access_key_id"),
                aws_secret_access_key=values.get("aws_secret_access_key"),
                region=values.get("region_name"),
            )

    return values

azure_secrets_store

Azure Secrets Store implementation.

AzureSecretsStore (ServiceConnectorSecretsStore) pydantic-model

Secrets store implementation that uses the Azure Key Vault API.

This secrets store implementation uses the Azure Key Vault API to store secrets. It allows a single Azure Key Vault to be shared with other ZenML deployments as well as other third party users and applications.

Here are some implementation highlights:

  • the name/ID of an Azure secret is derived from the ZenML secret UUID and a zenml prefix in the form zenml-{zenml_secret_uuid}. This clearly identifies a secret as being managed by ZenML in the Azure console.

  • the Secrets Store also uses Azure Key Vault secret tags to store metadata associated with a ZenML secret. The zenml tag in particular is used to identify and group all secrets that belong to the same ZenML deployment.

  • all secret key-values configured in a ZenML secret are stored as a single JSON string value in the Azure Key Vault secret value.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
class AzureSecretsStore(ServiceConnectorSecretsStore):
    """Secrets store implementation that uses the Azure Key Vault API.

    This secrets store implementation uses the Azure Key Vault API to
    store secrets. It allows a single Azure Key Vault to be shared with other
    ZenML deployments as well as other third party users and applications.

    Here are some implementation highlights:

    * the name/ID of an Azure secret is derived from the ZenML secret UUID and a
    `zenml` prefix in the form `zenml-{zenml_secret_uuid}`. This clearly
    identifies a secret as being managed by ZenML in the Azure console.

    * the Secrets Store also uses Azure Key Vault secret tags to store metadata
    associated with a ZenML secret. The `zenml` tag in particular is used to
    identify and group all secrets that belong to the same ZenML deployment.

    * all secret key-values configured in a ZenML secret are stored as a single
    JSON string value in the Azure Key Vault secret value.
    """

    config: AzureSecretsStoreConfiguration
    TYPE: ClassVar[SecretsStoreType] = SecretsStoreType.AZURE
    CONFIG_TYPE: ClassVar[Type[ServiceConnectorSecretsStoreConfiguration]] = (
        AzureSecretsStoreConfiguration
    )
    SERVICE_CONNECTOR_TYPE: ClassVar[str] = AZURE_CONNECTOR_TYPE
    SERVICE_CONNECTOR_RESOURCE_TYPE: ClassVar[str] = AZURE_RESOURCE_TYPE

    @property
    def client(self) -> SecretClient:
        """Initialize and return the Azure Key Vault client.

        Returns:
            The Azure Key Vault client.
        """
        return cast(SecretClient, super().client)

    # ====================================
    # Secrets Store interface implementation
    # ====================================

    # --------------------------------
    # Initialization and configuration
    # --------------------------------

    def _initialize_client_from_connector(self, client: Any) -> Any:
        """Initialize the Azure Key Vault client from the service connector client.

        Args:
            client: The authenticated client object returned by the service
                connector.

        Returns:
            The Azure Key Vault client.
        """
        assert isinstance(client, TokenCredential)
        azure_logger = logging.getLogger("azure")

        # Suppress the INFO logging level of the Azure SDK if the
        # ZenML logging level is WARNING or lower.
        if logger.level <= logging.WARNING:
            azure_logger.setLevel(logging.WARNING)
        else:
            azure_logger.setLevel(logging.INFO)

        vault_url = f"https://{self.config.key_vault_name}.vault.azure.net"
        return SecretClient(vault_url=vault_url, credential=client)

    # ------
    # Secrets
    # ------

    @staticmethod
    def _get_azure_secret_id(
        secret_id: UUID,
    ) -> str:
        """Get the Azure secret ID corresponding to a ZenML secret ID.

        The convention used for Azure secret names is to use the ZenML
        secret UUID prefixed with `zenml` as the Azure secret name,
        i.e. `zenml-<secret_uuid>`.

        Args:
            secret_id: The ZenML secret ID.

        Returns:
            The Azure secret name.
        """
        return f"{AZURE_ZENML_SECRET_NAME_PREFIX}-{str(secret_id)}"

    def store_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Store secret values for a new secret.

        Args:
            secret_id: ID of the secret.
            secret_values: Values for the secret.

        Raises:
            RuntimeError: if the Azure Key Vault API returns an unexpected
                error.
        """
        azure_secret_id = self._get_azure_secret_id(secret_id)
        secret_value = json.dumps(secret_values)

        # Use the ZenML secret metadata as Azure tags
        metadata = self._get_secret_metadata(secret_id=secret_id)

        try:
            self.client.set_secret(
                azure_secret_id,
                secret_value,
                tags=metadata,
                content_type="application/json",
            )
        except HttpResponseError as e:
            raise RuntimeError(f"Error creating secret: {e}")

        logger.debug(f"Created Azure secret: {azure_secret_id}")

    def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
        """Get the secret values for an existing secret.

        Args:
            secret_id: ID of the secret.

        Returns:
            The secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: if the Azure Key Vault API returns an unexpected
                error.
        """
        azure_secret_id = self._get_azure_secret_id(secret_id)

        try:
            azure_secret = self.client.get_secret(
                azure_secret_id,
            )
        except ResourceNotFoundError as e:
            raise KeyError(
                f"Can't find the secret values for secret ID '{secret_id}' "
                f"in the secrets store back-end: {str(e)}"
            ) from e
        except HttpResponseError as e:
            raise RuntimeError(
                f"Error fetching secret with ID {secret_id} {e}"
            )

        # The _verify_secret_metadata method raises a KeyError if the
        # secret is not valid or does not belong to this server. Here we
        # simply pass the exception up the stack, as if the secret was not found
        # in the first place.
        assert azure_secret.properties.tags is not None
        self._verify_secret_metadata(
            secret_id=secret_id,
            metadata=azure_secret.properties.tags,
        )

        values = json.loads(azure_secret.value) if azure_secret.value else {}

        if not isinstance(values, dict):
            raise RuntimeError(
                f"Azure Key Vault secret values for secret {azure_secret_id} "
                "could not be retrieved: invalid type for values"
            )

        logger.debug(f"Retrieved Azure secret: {azure_secret_id}")

        return values

    def update_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Updates secret values for an existing secret.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_values: The new secret values.

        Raises:
            RuntimeError: if the Azure Key Vault API returns an unexpected
                error.
        """
        azure_secret_id = self._get_azure_secret_id(secret_id)
        secret_value = json.dumps(secret_values)

        # Convert the ZenML secret metadata to Azure tags
        metadata = self._get_secret_metadata(secret_id=secret_id)

        try:
            self.client.set_secret(
                azure_secret_id,
                secret_value,
                tags=metadata,
                content_type="application/json",
            )
        except HttpResponseError as e:
            raise RuntimeError(f"Error updating secret {secret_id}: {e}")

        logger.debug(f"Updated Azure secret: {azure_secret_id}")

    def delete_secret_values(self, secret_id: UUID) -> None:
        """Deletes secret values for an existing secret.

        Args:
            secret_id: The ID of the secret.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: if the Azure Key Vault API returns an unexpected
                error.
        """
        azure_secret_id = self._get_azure_secret_id(secret_id)

        try:
            self.client.begin_delete_secret(
                azure_secret_id,
            ).wait()
        except ResourceNotFoundError:
            raise KeyError(f"Secret with ID {secret_id} not found")
        except HttpResponseError as e:
            raise RuntimeError(
                f"Error deleting secret with ID {secret_id}: {e}"
            )

        logger.debug(f"Deleted Azure secret: {azure_secret_id}")
client: azure.keyvault.secrets.SecretClient property readonly

Initialize and return the Azure Key Vault client.

Returns:

Type Description
azure.keyvault.secrets.SecretClient

The Azure Key Vault client.

CONFIG_TYPE (ServiceConnectorSecretsStoreConfiguration) pydantic-model

Azure secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

key_vault_name str

Name of the Azure Key Vault that this secrets store will use to store secrets.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
class AzureSecretsStoreConfiguration(
    ServiceConnectorSecretsStoreConfiguration
):
    """Azure secrets store configuration.

    Attributes:
        type: The type of the store.
        key_vault_name: Name of the Azure Key Vault that this secrets store
            will use to store secrets.
    """

    type: SecretsStoreType = SecretsStoreType.AZURE
    key_vault_name: str

    @root_validator(pre=True)
    def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Populate the connector configuration from legacy attributes.

        Args:
            values: Dict representing user-specified runtime settings.

        Returns:
            Validated settings.
        """
        # Search for legacy attributes and populate the connector configuration
        # from them, if they exist.
        if (
            values.get("azure_client_id")
            and values.get("azure_client_secret")
            and values.get("azure_tenant_id")
        ):
            logger.warning(
                "The `azure_client_id`, `azure_client_secret` and "
                "`azure_tenant_id` attributes are deprecated and will be "
                "removed in a future version or ZenML. Please use the "
                "`auth_method` and `auth_config` attributes instead."
            )
            values["auth_method"] = (
                AzureAuthenticationMethods.SERVICE_PRINCIPAL
            )
            values["auth_config"] = dict(
                client_id=values.get("azure_client_id"),
                client_secret=values.get("azure_client_secret"),
                tenant_id=values.get("azure_tenant_id"),
            )

        return values

    class Config:
        """Pydantic configuration class."""

        # Forbid extra attributes set in the class.
        extra = "allow"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Forbid extra attributes set in the class.
    extra = "allow"
populate_config(values) classmethod

Populate the connector configuration from legacy attributes.

Parameters:

Name Type Description Default
values Dict[str, Any]

Dict representing user-specified runtime settings.

required

Returns:

Type Description
Dict[str, Any]

Validated settings.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
@root_validator(pre=True)
def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Populate the connector configuration from legacy attributes.

    Args:
        values: Dict representing user-specified runtime settings.

    Returns:
        Validated settings.
    """
    # Search for legacy attributes and populate the connector configuration
    # from them, if they exist.
    if (
        values.get("azure_client_id")
        and values.get("azure_client_secret")
        and values.get("azure_tenant_id")
    ):
        logger.warning(
            "The `azure_client_id`, `azure_client_secret` and "
            "`azure_tenant_id` attributes are deprecated and will be "
            "removed in a future version or ZenML. Please use the "
            "`auth_method` and `auth_config` attributes instead."
        )
        values["auth_method"] = (
            AzureAuthenticationMethods.SERVICE_PRINCIPAL
        )
        values["auth_config"] = dict(
            client_id=values.get("azure_client_id"),
            client_secret=values.get("azure_client_secret"),
            tenant_id=values.get("azure_tenant_id"),
        )

    return values
delete_secret_values(self, secret_id)

Deletes secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

if the Azure Key Vault API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
def delete_secret_values(self, secret_id: UUID) -> None:
    """Deletes secret values for an existing secret.

    Args:
        secret_id: The ID of the secret.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: if the Azure Key Vault API returns an unexpected
            error.
    """
    azure_secret_id = self._get_azure_secret_id(secret_id)

    try:
        self.client.begin_delete_secret(
            azure_secret_id,
        ).wait()
    except ResourceNotFoundError:
        raise KeyError(f"Secret with ID {secret_id} not found")
    except HttpResponseError as e:
        raise RuntimeError(
            f"Error deleting secret with ID {secret_id}: {e}"
        )

    logger.debug(f"Deleted Azure secret: {azure_secret_id}")
get_secret_values(self, secret_id)

Get the secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required

Returns:

Type Description
Dict[str, str]

The secret values.

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

if the Azure Key Vault API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
    """Get the secret values for an existing secret.

    Args:
        secret_id: ID of the secret.

    Returns:
        The secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: if the Azure Key Vault API returns an unexpected
            error.
    """
    azure_secret_id = self._get_azure_secret_id(secret_id)

    try:
        azure_secret = self.client.get_secret(
            azure_secret_id,
        )
    except ResourceNotFoundError as e:
        raise KeyError(
            f"Can't find the secret values for secret ID '{secret_id}' "
            f"in the secrets store back-end: {str(e)}"
        ) from e
    except HttpResponseError as e:
        raise RuntimeError(
            f"Error fetching secret with ID {secret_id} {e}"
        )

    # The _verify_secret_metadata method raises a KeyError if the
    # secret is not valid or does not belong to this server. Here we
    # simply pass the exception up the stack, as if the secret was not found
    # in the first place.
    assert azure_secret.properties.tags is not None
    self._verify_secret_metadata(
        secret_id=secret_id,
        metadata=azure_secret.properties.tags,
    )

    values = json.loads(azure_secret.value) if azure_secret.value else {}

    if not isinstance(values, dict):
        raise RuntimeError(
            f"Azure Key Vault secret values for secret {azure_secret_id} "
            "could not be retrieved: invalid type for values"
        )

    logger.debug(f"Retrieved Azure secret: {azure_secret_id}")

    return values
store_secret_values(self, secret_id, secret_values)

Store secret values for a new secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required
secret_values Dict[str, str]

Values for the secret.

required

Exceptions:

Type Description
RuntimeError

if the Azure Key Vault API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
def store_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Store secret values for a new secret.

    Args:
        secret_id: ID of the secret.
        secret_values: Values for the secret.

    Raises:
        RuntimeError: if the Azure Key Vault API returns an unexpected
            error.
    """
    azure_secret_id = self._get_azure_secret_id(secret_id)
    secret_value = json.dumps(secret_values)

    # Use the ZenML secret metadata as Azure tags
    metadata = self._get_secret_metadata(secret_id=secret_id)

    try:
        self.client.set_secret(
            azure_secret_id,
            secret_value,
            tags=metadata,
            content_type="application/json",
        )
    except HttpResponseError as e:
        raise RuntimeError(f"Error creating secret: {e}")

    logger.debug(f"Created Azure secret: {azure_secret_id}")
update_secret_values(self, secret_id, secret_values)

Updates secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_values Dict[str, str]

The new secret values.

required

Exceptions:

Type Description
RuntimeError

if the Azure Key Vault API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
def update_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Updates secret values for an existing secret.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_values: The new secret values.

    Raises:
        RuntimeError: if the Azure Key Vault API returns an unexpected
            error.
    """
    azure_secret_id = self._get_azure_secret_id(secret_id)
    secret_value = json.dumps(secret_values)

    # Convert the ZenML secret metadata to Azure tags
    metadata = self._get_secret_metadata(secret_id=secret_id)

    try:
        self.client.set_secret(
            azure_secret_id,
            secret_value,
            tags=metadata,
            content_type="application/json",
        )
    except HttpResponseError as e:
        raise RuntimeError(f"Error updating secret {secret_id}: {e}")

    logger.debug(f"Updated Azure secret: {azure_secret_id}")
AzureSecretsStoreConfiguration (ServiceConnectorSecretsStoreConfiguration) pydantic-model

Azure secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

key_vault_name str

Name of the Azure Key Vault that this secrets store will use to store secrets.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
class AzureSecretsStoreConfiguration(
    ServiceConnectorSecretsStoreConfiguration
):
    """Azure secrets store configuration.

    Attributes:
        type: The type of the store.
        key_vault_name: Name of the Azure Key Vault that this secrets store
            will use to store secrets.
    """

    type: SecretsStoreType = SecretsStoreType.AZURE
    key_vault_name: str

    @root_validator(pre=True)
    def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Populate the connector configuration from legacy attributes.

        Args:
            values: Dict representing user-specified runtime settings.

        Returns:
            Validated settings.
        """
        # Search for legacy attributes and populate the connector configuration
        # from them, if they exist.
        if (
            values.get("azure_client_id")
            and values.get("azure_client_secret")
            and values.get("azure_tenant_id")
        ):
            logger.warning(
                "The `azure_client_id`, `azure_client_secret` and "
                "`azure_tenant_id` attributes are deprecated and will be "
                "removed in a future version or ZenML. Please use the "
                "`auth_method` and `auth_config` attributes instead."
            )
            values["auth_method"] = (
                AzureAuthenticationMethods.SERVICE_PRINCIPAL
            )
            values["auth_config"] = dict(
                client_id=values.get("azure_client_id"),
                client_secret=values.get("azure_client_secret"),
                tenant_id=values.get("azure_tenant_id"),
            )

        return values

    class Config:
        """Pydantic configuration class."""

        # Forbid extra attributes set in the class.
        extra = "allow"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Forbid extra attributes set in the class.
    extra = "allow"
populate_config(values) classmethod

Populate the connector configuration from legacy attributes.

Parameters:

Name Type Description Default
values Dict[str, Any]

Dict representing user-specified runtime settings.

required

Returns:

Type Description
Dict[str, Any]

Validated settings.

Source code in zenml/zen_stores/secrets_stores/azure_secrets_store.py
@root_validator(pre=True)
def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Populate the connector configuration from legacy attributes.

    Args:
        values: Dict representing user-specified runtime settings.

    Returns:
        Validated settings.
    """
    # Search for legacy attributes and populate the connector configuration
    # from them, if they exist.
    if (
        values.get("azure_client_id")
        and values.get("azure_client_secret")
        and values.get("azure_tenant_id")
    ):
        logger.warning(
            "The `azure_client_id`, `azure_client_secret` and "
            "`azure_tenant_id` attributes are deprecated and will be "
            "removed in a future version or ZenML. Please use the "
            "`auth_method` and `auth_config` attributes instead."
        )
        values["auth_method"] = (
            AzureAuthenticationMethods.SERVICE_PRINCIPAL
        )
        values["auth_config"] = dict(
            client_id=values.get("azure_client_id"),
            client_secret=values.get("azure_client_secret"),
            tenant_id=values.get("azure_tenant_id"),
        )

    return values

base_secrets_store

Base Secrets Store implementation.

BaseSecretsStore (BaseModel, SecretsStoreInterface, ABC) pydantic-model

Base class for accessing and persisting ZenML secret values.

Attributes:

Name Type Description
config

The configuration of the secret store.

_zen_store

The ZenML store that owns this secrets store.

Source code in zenml/zen_stores/secrets_stores/base_secrets_store.py
class BaseSecretsStore(BaseModel, SecretsStoreInterface, ABC):
    """Base class for accessing and persisting ZenML secret values.

    Attributes:
        config: The configuration of the secret store.
        _zen_store: The ZenML store that owns this secrets store.
    """

    config: SecretsStoreConfiguration
    _zen_store: Optional["BaseZenStore"] = None

    TYPE: ClassVar[SecretsStoreType]
    CONFIG_TYPE: ClassVar[Type[SecretsStoreConfiguration]]

    # ---------------------------------
    # Initialization and configuration
    # ---------------------------------

    def __init__(
        self,
        zen_store: "BaseZenStore",
        **kwargs: Any,
    ) -> None:
        """Create and initialize a secrets store.

        Args:
            zen_store: The ZenML store that owns this secrets store.
            **kwargs: Additional keyword arguments to pass to the Pydantic
                constructor.

        Raises:
            RuntimeError: If the store cannot be initialized.
        """
        super().__init__(**kwargs)
        self._zen_store = zen_store

        try:
            self._initialize()
        except Exception as e:
            raise RuntimeError(
                f"Error initializing {self.type.value} secrets store: {str(e)}"
            ) from e

    @staticmethod
    def _load_custom_store_class(
        store_config: SecretsStoreConfiguration,
    ) -> Type["BaseSecretsStore"]:
        """Loads the custom secrets store class from the given config.

        Args:
            store_config: The configuration of the secrets store.

        Returns:
            The secrets store class corresponding to the configured custom
            secrets store.

        Raises:
            ValueError: If the configured class path cannot be imported or is
                not a subclass of `BaseSecretsStore`.
        """
        # Ensured through Pydantic root validation
        assert store_config.class_path is not None

        # Import the class dynamically
        try:
            store_class = source_utils.load_and_validate_class(
                store_config.class_path, expected_class=BaseSecretsStore
            )
        except (ImportError, AttributeError) as e:
            raise ValueError(
                f"Could not import class `{store_config.class_path}`: {str(e)}"
            ) from e

        return store_class

    @staticmethod
    def get_store_class(
        store_config: SecretsStoreConfiguration,
    ) -> Type["BaseSecretsStore"]:
        """Returns the class of the given secrets store type.

        Args:
            store_config: The configuration of the secrets store.

        Returns:
            The class corresponding to the configured secrets store or None if
            the type is unknown.

        Raises:
            TypeError: If the secrets store type is unsupported.
        """
        if store_config.type == SecretsStoreType.SQL:
            from zenml.zen_stores.secrets_stores.sql_secrets_store import (
                SqlSecretsStore,
            )

            return SqlSecretsStore

        if store_config.type == SecretsStoreType.AWS:
            from zenml.zen_stores.secrets_stores.aws_secrets_store import (
                AWSSecretsStore,
            )

            return AWSSecretsStore
        elif store_config.type == SecretsStoreType.GCP:
            from zenml.zen_stores.secrets_stores.gcp_secrets_store import (
                GCPSecretsStore,
            )

            return GCPSecretsStore
        elif store_config.type == SecretsStoreType.AZURE:
            from zenml.zen_stores.secrets_stores.azure_secrets_store import (
                AzureSecretsStore,
            )

            return AzureSecretsStore
        elif store_config.type == SecretsStoreType.HASHICORP:
            from zenml.zen_stores.secrets_stores.hashicorp_secrets_store import (
                HashiCorpVaultSecretsStore,
            )

            return HashiCorpVaultSecretsStore
        elif store_config.type != SecretsStoreType.CUSTOM:
            raise TypeError(
                f"No store implementation found for secrets store type "
                f"`{store_config.type.value}`."
            )

        return BaseSecretsStore._load_custom_store_class(store_config)

    @staticmethod
    def create_store(
        config: SecretsStoreConfiguration,
        **kwargs: Any,
    ) -> "BaseSecretsStore":
        """Create and initialize a secrets store from a secrets store configuration.

        Args:
            config: The secrets store configuration to use.
            **kwargs: Additional keyword arguments to pass to the store class

        Returns:
            The initialized secrets store.
        """
        logger.debug(
            f"Creating secrets store with type '{config.type.value}'..."
        )
        store_class = BaseSecretsStore.get_store_class(config)
        store = store_class(
            config=config,
            **kwargs,
        )
        return store

    @property
    def type(self) -> SecretsStoreType:
        """The type of the secrets store.

        Returns:
            The type of the secrets store.
        """
        return self.TYPE

    @property
    def zen_store(self) -> "BaseZenStore":
        """The ZenML store that owns this secrets store.

        Returns:
            The ZenML store that owns this secrets store.

        Raises:
            ValueError: If the store is not initialized.
        """
        if not self._zen_store:
            raise ValueError("Store not initialized")
        return self._zen_store

    # --------------------------------------------------------
    # Helpers for Secrets Store back-ends that use tags/labels
    # --------------------------------------------------------

    def _get_secret_metadata(
        self,
        secret_id: Optional[UUID] = None,
    ) -> Dict[str, str]:
        """Get a dictionary with metadata that can be used as tags/labels.

        This utility method can be used with Secrets Managers that can
        associate metadata (e.g. tags, labels) with a secret. The metadata can
        be configured alongside each secret.

        NOTE: the ZENML_SECRET_LABEL is always included in the metadata to
        distinguish ZenML secrets from other secrets that might be stored in
        the same backend, as well as to distinguish between different ZenML
        deployments using the same backend. Its value is set to the ZenML
        deployment ID.

        Args:
            secret_id: Optional secret ID to include in the metadata.

        Returns:
            Dictionary with secret metadata information.
        """
        # Always include the main ZenML label to distinguish ZenML secrets
        # from other secrets that might be stored in the same backend and
        # to distinguish between different ZenML deployments using the same
        # backend.
        metadata: Dict[str, str] = {
            ZENML_SECRET_LABEL: str(self.zen_store.get_store_info().id)
        }

        # Include the secret ID if provided.
        if secret_id is not None:
            metadata[ZENML_SECRET_ID_LABEL] = str(secret_id)

        return metadata

    def _verify_secret_metadata(
        self,
        secret_id: UUID,
        metadata: Dict[str, str],
    ) -> None:
        """Verify that the given metadata corresponds to a valid ZenML secret.

        Args:
            secret_id: The ID of the secret.
            metadata: ZenML secret metadata collected from the backend secret
                (e.g. from secret tags/labels).

        Raises:
            KeyError: If the secret does not have the required metadata or if it
                is not managed by this ZenML instance.
        """
        # Double-check that the secret is managed by this ZenML instance.
        if metadata.get(ZENML_SECRET_LABEL) != str(
            self.zen_store.get_store_info().id
        ):
            raise KeyError("Secret is not managed by this ZenML instance")

        # Recover the ZenML secret fields from the input secret metadata.
        try:
            stored_secret_id = UUID(metadata[ZENML_SECRET_ID_LABEL])
        except KeyError as e:
            raise KeyError(
                f"Secret could not be retrieved: missing required metadata: {e}"
            )

        if secret_id != stored_secret_id:
            raise KeyError(
                f"Secret could not be retrieved: secret ID mismatch: "
                f"expected {secret_id}, got {stored_secret_id}"
            )

    class Config:
        """Pydantic configuration class."""

        # Validate attributes when assigning them. We need to set this in order
        # to have a mix of mutable and immutable attributes
        validate_assignment = True
        # Ignore extra attributes from configs of previous ZenML versions
        extra = "ignore"
        # all attributes with leading underscore are private and therefore
        # are mutable and not included in serialization
        underscore_attrs_are_private = True
type: SecretsStoreType property readonly

The type of the secrets store.

Returns:

Type Description
SecretsStoreType

The type of the secrets store.

zen_store: BaseZenStore property readonly

The ZenML store that owns this secrets store.

Returns:

Type Description
BaseZenStore

The ZenML store that owns this secrets store.

Exceptions:

Type Description
ValueError

If the store is not initialized.

Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/base_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Validate attributes when assigning them. We need to set this in order
    # to have a mix of mutable and immutable attributes
    validate_assignment = True
    # Ignore extra attributes from configs of previous ZenML versions
    extra = "ignore"
    # all attributes with leading underscore are private and therefore
    # are mutable and not included in serialization
    underscore_attrs_are_private = True
__init__(self, zen_store, **kwargs) special

Create and initialize a secrets store.

Parameters:

Name Type Description Default
zen_store BaseZenStore

The ZenML store that owns this secrets store.

required
**kwargs Any

Additional keyword arguments to pass to the Pydantic constructor.

{}

Exceptions:

Type Description
RuntimeError

If the store cannot be initialized.

Source code in zenml/zen_stores/secrets_stores/base_secrets_store.py
def __init__(
    self,
    zen_store: "BaseZenStore",
    **kwargs: Any,
) -> None:
    """Create and initialize a secrets store.

    Args:
        zen_store: The ZenML store that owns this secrets store.
        **kwargs: Additional keyword arguments to pass to the Pydantic
            constructor.

    Raises:
        RuntimeError: If the store cannot be initialized.
    """
    super().__init__(**kwargs)
    self._zen_store = zen_store

    try:
        self._initialize()
    except Exception as e:
        raise RuntimeError(
            f"Error initializing {self.type.value} secrets store: {str(e)}"
        ) from e
create_store(config, **kwargs) staticmethod

Create and initialize a secrets store from a secrets store configuration.

Parameters:

Name Type Description Default
config SecretsStoreConfiguration

The secrets store configuration to use.

required
**kwargs Any

Additional keyword arguments to pass to the store class

{}

Returns:

Type Description
BaseSecretsStore

The initialized secrets store.

Source code in zenml/zen_stores/secrets_stores/base_secrets_store.py
@staticmethod
def create_store(
    config: SecretsStoreConfiguration,
    **kwargs: Any,
) -> "BaseSecretsStore":
    """Create and initialize a secrets store from a secrets store configuration.

    Args:
        config: The secrets store configuration to use.
        **kwargs: Additional keyword arguments to pass to the store class

    Returns:
        The initialized secrets store.
    """
    logger.debug(
        f"Creating secrets store with type '{config.type.value}'..."
    )
    store_class = BaseSecretsStore.get_store_class(config)
    store = store_class(
        config=config,
        **kwargs,
    )
    return store
get_store_class(store_config) staticmethod

Returns the class of the given secrets store type.

Parameters:

Name Type Description Default
store_config SecretsStoreConfiguration

The configuration of the secrets store.

required

Returns:

Type Description
Type[BaseSecretsStore]

The class corresponding to the configured secrets store or None if the type is unknown.

Exceptions:

Type Description
TypeError

If the secrets store type is unsupported.

Source code in zenml/zen_stores/secrets_stores/base_secrets_store.py
@staticmethod
def get_store_class(
    store_config: SecretsStoreConfiguration,
) -> Type["BaseSecretsStore"]:
    """Returns the class of the given secrets store type.

    Args:
        store_config: The configuration of the secrets store.

    Returns:
        The class corresponding to the configured secrets store or None if
        the type is unknown.

    Raises:
        TypeError: If the secrets store type is unsupported.
    """
    if store_config.type == SecretsStoreType.SQL:
        from zenml.zen_stores.secrets_stores.sql_secrets_store import (
            SqlSecretsStore,
        )

        return SqlSecretsStore

    if store_config.type == SecretsStoreType.AWS:
        from zenml.zen_stores.secrets_stores.aws_secrets_store import (
            AWSSecretsStore,
        )

        return AWSSecretsStore
    elif store_config.type == SecretsStoreType.GCP:
        from zenml.zen_stores.secrets_stores.gcp_secrets_store import (
            GCPSecretsStore,
        )

        return GCPSecretsStore
    elif store_config.type == SecretsStoreType.AZURE:
        from zenml.zen_stores.secrets_stores.azure_secrets_store import (
            AzureSecretsStore,
        )

        return AzureSecretsStore
    elif store_config.type == SecretsStoreType.HASHICORP:
        from zenml.zen_stores.secrets_stores.hashicorp_secrets_store import (
            HashiCorpVaultSecretsStore,
        )

        return HashiCorpVaultSecretsStore
    elif store_config.type != SecretsStoreType.CUSTOM:
        raise TypeError(
            f"No store implementation found for secrets store type "
            f"`{store_config.type.value}`."
        )

    return BaseSecretsStore._load_custom_store_class(store_config)

gcp_secrets_store

Implementation of the GCP Secrets Store.

GCPSecretsStore (ServiceConnectorSecretsStore) pydantic-model

Secrets store implementation that uses the GCP Secrets Manager API.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
class GCPSecretsStore(ServiceConnectorSecretsStore):
    """Secrets store implementation that uses the GCP Secrets Manager API."""

    config: GCPSecretsStoreConfiguration
    TYPE: ClassVar[SecretsStoreType] = SecretsStoreType.GCP
    CONFIG_TYPE: ClassVar[Type[ServiceConnectorSecretsStoreConfiguration]] = (
        GCPSecretsStoreConfiguration
    )
    SERVICE_CONNECTOR_TYPE: ClassVar[str] = GCP_CONNECTOR_TYPE
    SERVICE_CONNECTOR_RESOURCE_TYPE: ClassVar[str] = GCP_RESOURCE_TYPE

    _client: Optional[SecretManagerServiceClient] = None

    @property
    def client(self) -> SecretManagerServiceClient:
        """Initialize and return the GCP Secrets Manager client.

        Returns:
            The GCP Secrets Manager client instance.
        """
        return cast(SecretManagerServiceClient, super().client)

    # ====================================
    # Secrets Store interface implementation
    # ====================================

    # --------------------------------
    # Initialization and configuration
    # --------------------------------

    def _initialize_client_from_connector(self, client: Any) -> Any:
        """Initialize the GCP Secrets Manager client from the service connector client.

        Args:
            client: The authenticated client object returned by the service
                connector.

        Returns:
            The GCP Secrets Manager client.
        """
        return SecretManagerServiceClient(credentials=client)

    # ------
    # Secrets
    # ------

    @property
    def parent_name(self) -> str:
        """Construct the GCP parent path to the secret manager.

        Returns:
            The parent path to the secret manager
        """
        return f"projects/{self.config.project_id}"

    def _get_gcp_secret_name(
        self,
        secret_id: UUID,
    ) -> str:
        """Get the GCP secret name for the given secret.

        The convention used for GCP secret names is to use the ZenML
        secret UUID prefixed with `zenml` as the AWS secret name,
        i.e. `zenml/<secret_uuid>`.

        Args:
            secret_id: The ZenML secret ID.

        Returns:
            The GCP secret name.
        """
        return f"{GCP_ZENML_SECRET_NAME_PREFIX}-{str(secret_id)}"

    def store_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Store secret values for a new secret.

        Args:
            secret_id: ID of the secret.
            secret_values: Values for the secret.

        Raises:
            RuntimeError: if the GCP Secrets Manager API returns an unexpected
                error.
        """
        secret_value = json.dumps(secret_values)

        labels = self._get_secret_metadata(secret_id=secret_id)

        try:
            gcp_secret = self.client.create_secret(
                request={
                    "parent": self.parent_name,
                    "secret_id": self._get_gcp_secret_name(secret_id),
                    "secret": {
                        "replication": {"automatic": {}},
                        "labels": labels,
                    },
                }
            )

            logger.debug(f"Created empty GCP parent secret: {gcp_secret.name}")

            self.client.add_secret_version(
                request={
                    "parent": gcp_secret.name,
                    "payload": {"data": secret_value.encode()},
                }
            )

            logger.debug(f"Added value to GCP secret {gcp_secret.name}")
        except Exception as e:
            raise RuntimeError(f"Failed to create secret.: {str(e)}") from e

        logger.debug(f"Created GCP secret {gcp_secret.name}")

    def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
        """Get the secret values for an existing secret.

        Args:
            secret_id: ID of the secret.

        Returns:
            The secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: if the GCP Secrets Manager API returns an unexpected
                error.
        """
        gcp_secret_name = self.client.secret_path(
            self.config.project_id,
            self._get_gcp_secret_name(secret_id=secret_id),
        )

        try:
            secret = self.client.get_secret(name=gcp_secret_name)
            secret_version_values = self.client.access_secret_version(
                name=f"{gcp_secret_name}/versions/latest"
            )
        except google_exceptions.NotFound as e:
            raise KeyError(
                f"Can't find the secret values for secret ID '{secret_id}' "
                f"in the secrets store back-end: {str(e)}"
            ) from e
        except Exception as e:
            raise RuntimeError(
                f"Error fetching secret with ID {secret_id} {e}"
            )

        # The GCP secret labels do not really behave like a dictionary: when
        # a key is not found, it does not raise a KeyError, but instead
        # returns an empty string. That's why we make this conversion.
        metadata = dict(secret.labels)

        # The _verify_secret_metadata method raises a KeyError if the
        # secret is not valid or does not belong to this server. Here we
        # simply pass the exception up the stack, as if the secret was not found
        # in the first place.
        self._verify_secret_metadata(
            secret_id=secret_id,
            metadata=metadata,
        )

        secret_values = json.loads(
            secret_version_values.payload.data.decode("UTF-8")
        )

        if not isinstance(secret_values, dict):
            raise RuntimeError(
                f"Google secret values for secret ID {gcp_secret_name} could "
                "not be decoded: expected a dictionary."
            )

        logger.debug(f"Fetched GCP secret: {gcp_secret_name}")

        return secret_values

    def update_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Updates secret values for an existing secret.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_values: The new secret values.

        Raises:
            RuntimeError: if the GCP Secrets Manager API returns an unexpected
                error.
        """
        gcp_secret_name = self.client.secret_path(
            self.config.project_id,
            self._get_gcp_secret_name(secret_id=secret_id),
        )

        # Convert the ZenML secret metadata to GCP labels
        metadata = self._get_secret_metadata(secret_id)

        try:
            # Update the secret metadata
            update_secret = {
                "name": gcp_secret_name,
                "labels": metadata,
            }
            update_mask = {"paths": ["labels"]}
            gcp_updated_secret = self.client.update_secret(
                request={
                    "secret": update_secret,
                    "update_mask": update_mask,
                }
            )
            # Add a new secret version
            secret_value = json.dumps(secret_values)
            self.client.add_secret_version(
                request={
                    "parent": gcp_updated_secret.name,
                    "payload": {"data": secret_value.encode()},
                }
            )
        except Exception as e:
            raise RuntimeError(f"Error updating secret: {e}") from e

        logger.debug(f"Updated GCP secret: {gcp_secret_name}")

    def delete_secret_values(self, secret_id: UUID) -> None:
        """Deletes secret values for an existing secret.

        Args:
            secret_id: The ID of the secret.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: if the GCP Secrets Manager API returns an unexpected
                error.
        """
        gcp_secret_name = self.client.secret_path(
            self.config.project_id,
            self._get_gcp_secret_name(secret_id=secret_id),
        )

        try:
            self.client.delete_secret(request={"name": gcp_secret_name})
        except google_exceptions.NotFound:
            raise KeyError(f"Secret with ID {secret_id} not found")
        except Exception as e:
            raise RuntimeError(f"Failed to delete secret: {str(e)}") from e

        logger.debug(f"Deleted GCP secret: {gcp_secret_name}")
client: google.cloud.secretmanager.SecretManagerServiceClient property readonly

Initialize and return the GCP Secrets Manager client.

Returns:

Type Description
google.cloud.secretmanager.SecretManagerServiceClient

The GCP Secrets Manager client instance.

parent_name: str property readonly

Construct the GCP parent path to the secret manager.

Returns:

Type Description
str

The parent path to the secret manager

CONFIG_TYPE (ServiceConnectorSecretsStoreConfiguration) pydantic-model

GCP secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
class GCPSecretsStoreConfiguration(ServiceConnectorSecretsStoreConfiguration):
    """GCP secrets store configuration.

    Attributes:
        type: The type of the store.
    """

    type: SecretsStoreType = SecretsStoreType.GCP

    @property
    def project_id(self) -> str:
        """Get the GCP project ID.

        Returns:
            The GCP project ID.

        Raises:
            ValueError: If the project ID is not set.
        """
        project_id = self.auth_config.get("project_id")
        if project_id:
            return str(project_id)

        raise ValueError("GCP `project_id` must be specified in auth_config.")

    @root_validator(pre=True)
    def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Populate the connector configuration from legacy attributes.

        Args:
            values: Dict representing user-specified runtime settings.

        Returns:
            Validated settings.
        """
        # Search for legacy attributes and populate the connector configuration
        # from them, if they exist.
        if values.get("project_id"):
            if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
                logger.warning(
                    "The `project_id` GCP secrets store attribute is "
                    "deprecated and will be removed in a future version of ZenML. "
                    "Please use the `auth_method` and `auth_config` attributes "
                    "instead. Using an implicit GCP authentication to access "
                    "the GCP Secrets Manager API."
                )
                values["auth_method"] = GCPAuthenticationMethods.IMPLICIT
                values["auth_config"] = dict(
                    project_id=values.get("project_id"),
                )
            else:
                logger.warning(
                    "The `project_id` GCP secrets store attribute and the "
                    "`GOOGLE_APPLICATION_CREDENTIALS` environment variable are "
                    "deprecated and will be removed in a future version of ZenML. "
                    "Please use the `auth_method` and `auth_config` attributes "
                    "instead."
                )
                values["auth_method"] = (
                    GCPAuthenticationMethods.SERVICE_ACCOUNT
                )
                values["auth_config"] = dict(
                    project_id=values.get("project_id"),
                )
                # Load the service account credentials from the file
                with open(os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) as f:
                    values["auth_config"]["service_account_json"] = f.read()

        return values

    class Config:
        """Pydantic configuration class."""

        # Forbid extra attributes set in the class.
        extra = "allow"
project_id: str property readonly

Get the GCP project ID.

Returns:

Type Description
str

The GCP project ID.

Exceptions:

Type Description
ValueError

If the project ID is not set.

Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Forbid extra attributes set in the class.
    extra = "allow"
populate_config(values) classmethod

Populate the connector configuration from legacy attributes.

Parameters:

Name Type Description Default
values Dict[str, Any]

Dict representing user-specified runtime settings.

required

Returns:

Type Description
Dict[str, Any]

Validated settings.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
@root_validator(pre=True)
def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Populate the connector configuration from legacy attributes.

    Args:
        values: Dict representing user-specified runtime settings.

    Returns:
        Validated settings.
    """
    # Search for legacy attributes and populate the connector configuration
    # from them, if they exist.
    if values.get("project_id"):
        if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
            logger.warning(
                "The `project_id` GCP secrets store attribute is "
                "deprecated and will be removed in a future version of ZenML. "
                "Please use the `auth_method` and `auth_config` attributes "
                "instead. Using an implicit GCP authentication to access "
                "the GCP Secrets Manager API."
            )
            values["auth_method"] = GCPAuthenticationMethods.IMPLICIT
            values["auth_config"] = dict(
                project_id=values.get("project_id"),
            )
        else:
            logger.warning(
                "The `project_id` GCP secrets store attribute and the "
                "`GOOGLE_APPLICATION_CREDENTIALS` environment variable are "
                "deprecated and will be removed in a future version of ZenML. "
                "Please use the `auth_method` and `auth_config` attributes "
                "instead."
            )
            values["auth_method"] = (
                GCPAuthenticationMethods.SERVICE_ACCOUNT
            )
            values["auth_config"] = dict(
                project_id=values.get("project_id"),
            )
            # Load the service account credentials from the file
            with open(os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) as f:
                values["auth_config"]["service_account_json"] = f.read()

    return values
delete_secret_values(self, secret_id)

Deletes secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

if the GCP Secrets Manager API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
def delete_secret_values(self, secret_id: UUID) -> None:
    """Deletes secret values for an existing secret.

    Args:
        secret_id: The ID of the secret.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: if the GCP Secrets Manager API returns an unexpected
            error.
    """
    gcp_secret_name = self.client.secret_path(
        self.config.project_id,
        self._get_gcp_secret_name(secret_id=secret_id),
    )

    try:
        self.client.delete_secret(request={"name": gcp_secret_name})
    except google_exceptions.NotFound:
        raise KeyError(f"Secret with ID {secret_id} not found")
    except Exception as e:
        raise RuntimeError(f"Failed to delete secret: {str(e)}") from e

    logger.debug(f"Deleted GCP secret: {gcp_secret_name}")
get_secret_values(self, secret_id)

Get the secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required

Returns:

Type Description
Dict[str, str]

The secret values.

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

if the GCP Secrets Manager API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
    """Get the secret values for an existing secret.

    Args:
        secret_id: ID of the secret.

    Returns:
        The secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: if the GCP Secrets Manager API returns an unexpected
            error.
    """
    gcp_secret_name = self.client.secret_path(
        self.config.project_id,
        self._get_gcp_secret_name(secret_id=secret_id),
    )

    try:
        secret = self.client.get_secret(name=gcp_secret_name)
        secret_version_values = self.client.access_secret_version(
            name=f"{gcp_secret_name}/versions/latest"
        )
    except google_exceptions.NotFound as e:
        raise KeyError(
            f"Can't find the secret values for secret ID '{secret_id}' "
            f"in the secrets store back-end: {str(e)}"
        ) from e
    except Exception as e:
        raise RuntimeError(
            f"Error fetching secret with ID {secret_id} {e}"
        )

    # The GCP secret labels do not really behave like a dictionary: when
    # a key is not found, it does not raise a KeyError, but instead
    # returns an empty string. That's why we make this conversion.
    metadata = dict(secret.labels)

    # The _verify_secret_metadata method raises a KeyError if the
    # secret is not valid or does not belong to this server. Here we
    # simply pass the exception up the stack, as if the secret was not found
    # in the first place.
    self._verify_secret_metadata(
        secret_id=secret_id,
        metadata=metadata,
    )

    secret_values = json.loads(
        secret_version_values.payload.data.decode("UTF-8")
    )

    if not isinstance(secret_values, dict):
        raise RuntimeError(
            f"Google secret values for secret ID {gcp_secret_name} could "
            "not be decoded: expected a dictionary."
        )

    logger.debug(f"Fetched GCP secret: {gcp_secret_name}")

    return secret_values
store_secret_values(self, secret_id, secret_values)

Store secret values for a new secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required
secret_values Dict[str, str]

Values for the secret.

required

Exceptions:

Type Description
RuntimeError

if the GCP Secrets Manager API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
def store_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Store secret values for a new secret.

    Args:
        secret_id: ID of the secret.
        secret_values: Values for the secret.

    Raises:
        RuntimeError: if the GCP Secrets Manager API returns an unexpected
            error.
    """
    secret_value = json.dumps(secret_values)

    labels = self._get_secret_metadata(secret_id=secret_id)

    try:
        gcp_secret = self.client.create_secret(
            request={
                "parent": self.parent_name,
                "secret_id": self._get_gcp_secret_name(secret_id),
                "secret": {
                    "replication": {"automatic": {}},
                    "labels": labels,
                },
            }
        )

        logger.debug(f"Created empty GCP parent secret: {gcp_secret.name}")

        self.client.add_secret_version(
            request={
                "parent": gcp_secret.name,
                "payload": {"data": secret_value.encode()},
            }
        )

        logger.debug(f"Added value to GCP secret {gcp_secret.name}")
    except Exception as e:
        raise RuntimeError(f"Failed to create secret.: {str(e)}") from e

    logger.debug(f"Created GCP secret {gcp_secret.name}")
update_secret_values(self, secret_id, secret_values)

Updates secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_values Dict[str, str]

The new secret values.

required

Exceptions:

Type Description
RuntimeError

if the GCP Secrets Manager API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
def update_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Updates secret values for an existing secret.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_values: The new secret values.

    Raises:
        RuntimeError: if the GCP Secrets Manager API returns an unexpected
            error.
    """
    gcp_secret_name = self.client.secret_path(
        self.config.project_id,
        self._get_gcp_secret_name(secret_id=secret_id),
    )

    # Convert the ZenML secret metadata to GCP labels
    metadata = self._get_secret_metadata(secret_id)

    try:
        # Update the secret metadata
        update_secret = {
            "name": gcp_secret_name,
            "labels": metadata,
        }
        update_mask = {"paths": ["labels"]}
        gcp_updated_secret = self.client.update_secret(
            request={
                "secret": update_secret,
                "update_mask": update_mask,
            }
        )
        # Add a new secret version
        secret_value = json.dumps(secret_values)
        self.client.add_secret_version(
            request={
                "parent": gcp_updated_secret.name,
                "payload": {"data": secret_value.encode()},
            }
        )
    except Exception as e:
        raise RuntimeError(f"Error updating secret: {e}") from e

    logger.debug(f"Updated GCP secret: {gcp_secret_name}")
GCPSecretsStoreConfiguration (ServiceConnectorSecretsStoreConfiguration) pydantic-model

GCP secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
class GCPSecretsStoreConfiguration(ServiceConnectorSecretsStoreConfiguration):
    """GCP secrets store configuration.

    Attributes:
        type: The type of the store.
    """

    type: SecretsStoreType = SecretsStoreType.GCP

    @property
    def project_id(self) -> str:
        """Get the GCP project ID.

        Returns:
            The GCP project ID.

        Raises:
            ValueError: If the project ID is not set.
        """
        project_id = self.auth_config.get("project_id")
        if project_id:
            return str(project_id)

        raise ValueError("GCP `project_id` must be specified in auth_config.")

    @root_validator(pre=True)
    def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Populate the connector configuration from legacy attributes.

        Args:
            values: Dict representing user-specified runtime settings.

        Returns:
            Validated settings.
        """
        # Search for legacy attributes and populate the connector configuration
        # from them, if they exist.
        if values.get("project_id"):
            if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
                logger.warning(
                    "The `project_id` GCP secrets store attribute is "
                    "deprecated and will be removed in a future version of ZenML. "
                    "Please use the `auth_method` and `auth_config` attributes "
                    "instead. Using an implicit GCP authentication to access "
                    "the GCP Secrets Manager API."
                )
                values["auth_method"] = GCPAuthenticationMethods.IMPLICIT
                values["auth_config"] = dict(
                    project_id=values.get("project_id"),
                )
            else:
                logger.warning(
                    "The `project_id` GCP secrets store attribute and the "
                    "`GOOGLE_APPLICATION_CREDENTIALS` environment variable are "
                    "deprecated and will be removed in a future version of ZenML. "
                    "Please use the `auth_method` and `auth_config` attributes "
                    "instead."
                )
                values["auth_method"] = (
                    GCPAuthenticationMethods.SERVICE_ACCOUNT
                )
                values["auth_config"] = dict(
                    project_id=values.get("project_id"),
                )
                # Load the service account credentials from the file
                with open(os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) as f:
                    values["auth_config"]["service_account_json"] = f.read()

        return values

    class Config:
        """Pydantic configuration class."""

        # Forbid extra attributes set in the class.
        extra = "allow"
project_id: str property readonly

Get the GCP project ID.

Returns:

Type Description
str

The GCP project ID.

Exceptions:

Type Description
ValueError

If the project ID is not set.

Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Forbid extra attributes set in the class.
    extra = "allow"
populate_config(values) classmethod

Populate the connector configuration from legacy attributes.

Parameters:

Name Type Description Default
values Dict[str, Any]

Dict representing user-specified runtime settings.

required

Returns:

Type Description
Dict[str, Any]

Validated settings.

Source code in zenml/zen_stores/secrets_stores/gcp_secrets_store.py
@root_validator(pre=True)
def populate_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Populate the connector configuration from legacy attributes.

    Args:
        values: Dict representing user-specified runtime settings.

    Returns:
        Validated settings.
    """
    # Search for legacy attributes and populate the connector configuration
    # from them, if they exist.
    if values.get("project_id"):
        if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"):
            logger.warning(
                "The `project_id` GCP secrets store attribute is "
                "deprecated and will be removed in a future version of ZenML. "
                "Please use the `auth_method` and `auth_config` attributes "
                "instead. Using an implicit GCP authentication to access "
                "the GCP Secrets Manager API."
            )
            values["auth_method"] = GCPAuthenticationMethods.IMPLICIT
            values["auth_config"] = dict(
                project_id=values.get("project_id"),
            )
        else:
            logger.warning(
                "The `project_id` GCP secrets store attribute and the "
                "`GOOGLE_APPLICATION_CREDENTIALS` environment variable are "
                "deprecated and will be removed in a future version of ZenML. "
                "Please use the `auth_method` and `auth_config` attributes "
                "instead."
            )
            values["auth_method"] = (
                GCPAuthenticationMethods.SERVICE_ACCOUNT
            )
            values["auth_config"] = dict(
                project_id=values.get("project_id"),
            )
            # Load the service account credentials from the file
            with open(os.environ["GOOGLE_APPLICATION_CREDENTIALS"]) as f:
                values["auth_config"]["service_account_json"] = f.read()

    return values

hashicorp_secrets_store

HashiCorp Vault Secrets Store implementation.

HashiCorpVaultSecretsStore (BaseSecretsStore) pydantic-model

Secrets store implementation that uses the HashiCorp Vault API.

This secrets store implementation uses the HashiCorp Vault API to store secrets. It allows a single HashiCorp Vault server to be shared with other ZenML deployments as well as other third party users and applications.

Here are some implementation highlights:

  • the name/ID of an HashiCorp Vault secret is derived from the ZenML secret UUID and a zenml prefix in the form zenml/{zenml_secret_uuid}. This clearly identifies a secret as being managed by ZenML in the HashiCorp Vault server.

  • given that HashiCorp Vault secrets do not support attaching arbitrary metadata in the form of label or tags, we store the entire ZenML secret metadata alongside the secret values in the HashiCorp Vault secret value.

Attributes:

Name Type Description
config

The configuration of the HashiCorp Vault secrets store.

TYPE

The type of the store.

CONFIG_TYPE

The type of the store configuration.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
class HashiCorpVaultSecretsStore(BaseSecretsStore):
    """Secrets store implementation that uses the HashiCorp Vault API.

    This secrets store implementation uses the HashiCorp Vault API to
    store secrets. It allows a single HashiCorp Vault server to be shared with
    other ZenML deployments as well as other third party users and applications.

    Here are some implementation highlights:

    * the name/ID of an HashiCorp Vault secret is derived from the ZenML secret
    UUID and a `zenml` prefix in the form `zenml/{zenml_secret_uuid}`. This
    clearly identifies a secret as being managed by ZenML in the HashiCorp Vault
    server.

    * given that HashiCorp Vault secrets do not support attaching arbitrary
    metadata in the form of label or tags, we store the entire ZenML secret
    metadata alongside the secret values in the HashiCorp Vault secret value.

    Attributes:
        config: The configuration of the HashiCorp Vault secrets store.
        TYPE: The type of the store.
        CONFIG_TYPE: The type of the store configuration.
    """

    config: HashiCorpVaultSecretsStoreConfiguration
    TYPE: ClassVar[SecretsStoreType] = SecretsStoreType.HASHICORP
    CONFIG_TYPE: ClassVar[Type[SecretsStoreConfiguration]] = (
        HashiCorpVaultSecretsStoreConfiguration
    )

    _client: Optional[hvac.Client] = None

    @property
    def client(self) -> hvac.Client:
        """Initialize and return the HashiCorp Vault client.

        Returns:
            The HashiCorp Vault client.
        """
        if self._client is None:
            # Initialize the HashiCorp Vault client with the
            # credentials from the configuration.
            self._client = hvac.Client(
                url=self.config.vault_addr,
                token=self.config.vault_token.get_secret_value()
                if self.config.vault_token
                else None,
                namespace=self.config.vault_namespace,
            )
            self._client.secrets.kv.v2.configure(
                max_versions=self.config.max_versions,
            )
            if self.config.mount_point:
                self._client.secrets.kv.v2.configure(
                    mount_point=self.config.mount_point,
                )
        return self._client

    # ====================================
    # Secrets Store interface implementation
    # ====================================

    # --------------------------------
    # Initialization and configuration
    # --------------------------------

    def _initialize(self) -> None:
        """Initialize the HashiCorp Vault secrets store."""
        logger.debug("Initializing HashiCorpVaultSecretsStore")

        # Initialize the HashiCorp Vault client early, just to catch any
        # configuration or authentication errors early, before the Secrets
        # Store is used.
        _ = self.client

    # ------
    # Secrets
    # ------

    @staticmethod
    def _get_vault_secret_id(
        secret_id: UUID,
    ) -> str:
        """Get the HashiCorp Vault secret ID corresponding to a ZenML secret ID.

        The convention used for HashiCorp Vault secret names is to use the ZenML
        secret UUID prefixed with `zenml` as the HashiCorp Vault secret name,
        i.e. `zenml/<secret_uuid>`.

        Args:
            secret_id: The ZenML secret ID.

        Returns:
            The HashiCorp Vault secret name.
        """
        return f"{HVAC_ZENML_SECRET_NAME_PREFIX}/{str(secret_id)}"

    def store_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Store secret values for a new secret.

        Args:
            secret_id: ID of the secret.
            secret_values: Values for the secret.

        Raises:
            RuntimeError: If the HashiCorp Vault API returns an unexpected
                error.
        """
        vault_secret_id = self._get_vault_secret_id(secret_id)

        metadata = self._get_secret_metadata(secret_id=secret_id)

        try:
            self.client.secrets.kv.v2.create_or_update_secret(
                path=vault_secret_id,
                # Store the ZenML secret metadata alongside the secret values
                secret={
                    ZENML_VAULT_SECRET_VALUES_KEY: secret_values,
                    ZENML_VAULT_SECRET_METADATA_KEY: metadata,
                },
                # Do not allow overwriting an existing secret
                cas=0,
            )
        except VaultError as e:
            raise RuntimeError(f"Error creating secret: {e}")

        logger.debug(f"Created HashiCorp Vault secret: {vault_secret_id}")

    def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
        """Get the secret values for an existing secret.

        Args:
            secret_id: ID of the secret.

        Returns:
            The secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: If the HashiCorp Vault API returns an unexpected
                error.
        """
        vault_secret_id = self._get_vault_secret_id(secret_id)

        try:
            vault_secret = (
                self.client.secrets.kv.v2.read_secret(
                    path=vault_secret_id,
                )
                .get("data", {})
                .get("data", {})
            )
        except InvalidPath as e:
            raise KeyError(
                f"Can't find the secret values for secret ID '{secret_id}' "
                f"in the secrets store back-end: {str(e)}"
            ) from e
        except VaultError as e:
            raise RuntimeError(
                f"Error fetching secret with ID {secret_id} {e}"
            )

        try:
            metadata = vault_secret[ZENML_VAULT_SECRET_METADATA_KEY]
            values = vault_secret[ZENML_VAULT_SECRET_VALUES_KEY]
        except (KeyError, ValueError) as e:
            raise KeyError(
                f"Secret could not be retrieved: missing required metadata: {e}"
            )

        if not isinstance(values, dict) or not isinstance(metadata, dict):
            raise RuntimeError(
                f"HashiCorp Vault secret values for secret {vault_secret_id} "
                "could not be retrieved: invalid type for metadata or values"
            )

        # The _verify_secret_metadata method raises a KeyError if the
        # secret is not valid or does not belong to this server. Here we
        # simply pass the exception up the stack, as if the secret was not found
        # in the first place.
        self._verify_secret_metadata(
            secret_id=secret_id,
            metadata=metadata,
        )

        logger.debug(f"Fetched HashiCorp Vault secret: {vault_secret_id}")

        return values

    def update_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Updates secret values for an existing secret.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_values: The new secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: If the HashiCorp Vault API returns an unexpected
                error.
        """
        vault_secret_id = self._get_vault_secret_id(secret_id)

        # Convert the ZenML secret metadata to HashiCorp Vault tags
        metadata = self._get_secret_metadata(secret_id=secret_id)

        try:
            self.client.secrets.kv.v2.create_or_update_secret(
                path=vault_secret_id,
                # Store the ZenML secret metadata alongside the secret values
                secret={
                    ZENML_VAULT_SECRET_VALUES_KEY: secret_values,
                    ZENML_VAULT_SECRET_METADATA_KEY: metadata,
                },
            )
        except InvalidPath:
            raise KeyError(f"Secret with ID {secret_id} does not exist.")
        except VaultError as e:
            raise RuntimeError(f"Error updating secret {secret_id}: {e}")

        logger.debug(f"Updated HashiCorp Vault secret: {vault_secret_id}")

    def delete_secret_values(self, secret_id: UUID) -> None:
        """Deletes secret values for an existing secret.

        Args:
            secret_id: The ID of the secret.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
            RuntimeError: If the HashiCorp Vault API returns an unexpected
                error.
        """
        vault_secret_id = self._get_vault_secret_id(secret_id)

        try:
            self.client.secrets.kv.v2.delete_metadata_and_all_versions(
                path=vault_secret_id,
            )
        except InvalidPath:
            raise KeyError(f"Secret with ID {secret_id} does not exist.")
        except VaultError as e:
            raise RuntimeError(
                f"Error deleting secret with ID {secret_id}: {e}"
            )

        logger.debug(f"Deleted HashiCorp Vault secret: {vault_secret_id}")
client: hvac.Client property readonly

Initialize and return the HashiCorp Vault client.

Returns:

Type Description
hvac.Client

The HashiCorp Vault client.

CONFIG_TYPE (SecretsStoreConfiguration) pydantic-model

HashiCorp Vault secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

vault_addr str

The url of the Vault server. If not set, the value will be loaded from the VAULT_ADDR environment variable, if configured.

vault_token Optional[pydantic.types.SecretStr]

The token used to authenticate with the Vault server. If not set, the token will be loaded from the VAULT_TOKEN environment variable or from the ~/.vault-token file, if configured.

vault_namespace Optional[str]

The Vault Enterprise namespace.

mount_point Optional[str]

The mount point to use for all secrets.

max_versions int

The maximum number of secret versions to keep.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
class HashiCorpVaultSecretsStoreConfiguration(SecretsStoreConfiguration):
    """HashiCorp Vault secrets store configuration.

    Attributes:
        type: The type of the store.
        vault_addr: The url of the Vault server. If not set, the value will be
            loaded from the VAULT_ADDR environment variable, if configured.
        vault_token: The token used to authenticate with the Vault server. If
            not set, the token will be loaded from the VAULT_TOKEN environment
            variable or from the ~/.vault-token file, if configured.
        vault_namespace: The Vault Enterprise namespace.
        mount_point: The mount point to use for all secrets.
        max_versions: The maximum number of secret versions to keep.
    """

    type: SecretsStoreType = SecretsStoreType.HASHICORP

    vault_addr: str
    vault_token: Optional[SecretStr] = None
    vault_namespace: Optional[str] = None
    mount_point: Optional[str] = None
    max_versions: int = 1

    class Config:
        """Pydantic configuration class."""

        # Forbid extra attributes set in the class.
        extra = "forbid"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Forbid extra attributes set in the class.
    extra = "forbid"
delete_secret_values(self, secret_id)

Deletes secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

If the HashiCorp Vault API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
def delete_secret_values(self, secret_id: UUID) -> None:
    """Deletes secret values for an existing secret.

    Args:
        secret_id: The ID of the secret.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: If the HashiCorp Vault API returns an unexpected
            error.
    """
    vault_secret_id = self._get_vault_secret_id(secret_id)

    try:
        self.client.secrets.kv.v2.delete_metadata_and_all_versions(
            path=vault_secret_id,
        )
    except InvalidPath:
        raise KeyError(f"Secret with ID {secret_id} does not exist.")
    except VaultError as e:
        raise RuntimeError(
            f"Error deleting secret with ID {secret_id}: {e}"
        )

    logger.debug(f"Deleted HashiCorp Vault secret: {vault_secret_id}")
get_secret_values(self, secret_id)

Get the secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required

Returns:

Type Description
Dict[str, str]

The secret values.

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

If the HashiCorp Vault API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
    """Get the secret values for an existing secret.

    Args:
        secret_id: ID of the secret.

    Returns:
        The secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: If the HashiCorp Vault API returns an unexpected
            error.
    """
    vault_secret_id = self._get_vault_secret_id(secret_id)

    try:
        vault_secret = (
            self.client.secrets.kv.v2.read_secret(
                path=vault_secret_id,
            )
            .get("data", {})
            .get("data", {})
        )
    except InvalidPath as e:
        raise KeyError(
            f"Can't find the secret values for secret ID '{secret_id}' "
            f"in the secrets store back-end: {str(e)}"
        ) from e
    except VaultError as e:
        raise RuntimeError(
            f"Error fetching secret with ID {secret_id} {e}"
        )

    try:
        metadata = vault_secret[ZENML_VAULT_SECRET_METADATA_KEY]
        values = vault_secret[ZENML_VAULT_SECRET_VALUES_KEY]
    except (KeyError, ValueError) as e:
        raise KeyError(
            f"Secret could not be retrieved: missing required metadata: {e}"
        )

    if not isinstance(values, dict) or not isinstance(metadata, dict):
        raise RuntimeError(
            f"HashiCorp Vault secret values for secret {vault_secret_id} "
            "could not be retrieved: invalid type for metadata or values"
        )

    # The _verify_secret_metadata method raises a KeyError if the
    # secret is not valid or does not belong to this server. Here we
    # simply pass the exception up the stack, as if the secret was not found
    # in the first place.
    self._verify_secret_metadata(
        secret_id=secret_id,
        metadata=metadata,
    )

    logger.debug(f"Fetched HashiCorp Vault secret: {vault_secret_id}")

    return values
store_secret_values(self, secret_id, secret_values)

Store secret values for a new secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required
secret_values Dict[str, str]

Values for the secret.

required

Exceptions:

Type Description
RuntimeError

If the HashiCorp Vault API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
def store_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Store secret values for a new secret.

    Args:
        secret_id: ID of the secret.
        secret_values: Values for the secret.

    Raises:
        RuntimeError: If the HashiCorp Vault API returns an unexpected
            error.
    """
    vault_secret_id = self._get_vault_secret_id(secret_id)

    metadata = self._get_secret_metadata(secret_id=secret_id)

    try:
        self.client.secrets.kv.v2.create_or_update_secret(
            path=vault_secret_id,
            # Store the ZenML secret metadata alongside the secret values
            secret={
                ZENML_VAULT_SECRET_VALUES_KEY: secret_values,
                ZENML_VAULT_SECRET_METADATA_KEY: metadata,
            },
            # Do not allow overwriting an existing secret
            cas=0,
        )
    except VaultError as e:
        raise RuntimeError(f"Error creating secret: {e}")

    logger.debug(f"Created HashiCorp Vault secret: {vault_secret_id}")
update_secret_values(self, secret_id, secret_values)

Updates secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_values Dict[str, str]

The new secret values.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

RuntimeError

If the HashiCorp Vault API returns an unexpected error.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
def update_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Updates secret values for an existing secret.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_values: The new secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
        RuntimeError: If the HashiCorp Vault API returns an unexpected
            error.
    """
    vault_secret_id = self._get_vault_secret_id(secret_id)

    # Convert the ZenML secret metadata to HashiCorp Vault tags
    metadata = self._get_secret_metadata(secret_id=secret_id)

    try:
        self.client.secrets.kv.v2.create_or_update_secret(
            path=vault_secret_id,
            # Store the ZenML secret metadata alongside the secret values
            secret={
                ZENML_VAULT_SECRET_VALUES_KEY: secret_values,
                ZENML_VAULT_SECRET_METADATA_KEY: metadata,
            },
        )
    except InvalidPath:
        raise KeyError(f"Secret with ID {secret_id} does not exist.")
    except VaultError as e:
        raise RuntimeError(f"Error updating secret {secret_id}: {e}")

    logger.debug(f"Updated HashiCorp Vault secret: {vault_secret_id}")
HashiCorpVaultSecretsStoreConfiguration (SecretsStoreConfiguration) pydantic-model

HashiCorp Vault secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

vault_addr str

The url of the Vault server. If not set, the value will be loaded from the VAULT_ADDR environment variable, if configured.

vault_token Optional[pydantic.types.SecretStr]

The token used to authenticate with the Vault server. If not set, the token will be loaded from the VAULT_TOKEN environment variable or from the ~/.vault-token file, if configured.

vault_namespace Optional[str]

The Vault Enterprise namespace.

mount_point Optional[str]

The mount point to use for all secrets.

max_versions int

The maximum number of secret versions to keep.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
class HashiCorpVaultSecretsStoreConfiguration(SecretsStoreConfiguration):
    """HashiCorp Vault secrets store configuration.

    Attributes:
        type: The type of the store.
        vault_addr: The url of the Vault server. If not set, the value will be
            loaded from the VAULT_ADDR environment variable, if configured.
        vault_token: The token used to authenticate with the Vault server. If
            not set, the token will be loaded from the VAULT_TOKEN environment
            variable or from the ~/.vault-token file, if configured.
        vault_namespace: The Vault Enterprise namespace.
        mount_point: The mount point to use for all secrets.
        max_versions: The maximum number of secret versions to keep.
    """

    type: SecretsStoreType = SecretsStoreType.HASHICORP

    vault_addr: str
    vault_token: Optional[SecretStr] = None
    vault_namespace: Optional[str] = None
    mount_point: Optional[str] = None
    max_versions: int = 1

    class Config:
        """Pydantic configuration class."""

        # Forbid extra attributes set in the class.
        extra = "forbid"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/hashicorp_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Forbid extra attributes set in the class.
    extra = "forbid"

secrets_store_interface

ZenML secrets store interface.

SecretsStoreInterface (ABC)

ZenML secrets store interface.

All ZenML secrets stores must implement the methods in this interface.

Source code in zenml/zen_stores/secrets_stores/secrets_store_interface.py
class SecretsStoreInterface(ABC):
    """ZenML secrets store interface.

    All ZenML secrets stores must implement the methods in this interface.
    """

    # ---------------------------------
    # Initialization and configuration
    # ---------------------------------

    @abstractmethod
    def _initialize(self) -> None:
        """Initialize the secrets store.

        This method is called immediately after the secrets store is created.
        It should be used to set up the backend (database, connection etc.).
        """

    # ---------
    # Secrets
    # ---------

    @abstractmethod
    def store_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Store secret values for a new secret.

        Args:
            secret_id: ID of the secret.
            secret_values: Values for the secret.
        """

    @abstractmethod
    def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
        """Get the secret values for an existing secret.

        Args:
            secret_id: ID of the secret.

        Returns:
            The secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
        """

    @abstractmethod
    def update_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Updates secret values for an existing secret.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_values: The new secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
        """

    @abstractmethod
    def delete_secret_values(self, secret_id: UUID) -> None:
        """Deletes secret values for an existing secret.

        Args:
            secret_id: The ID of the secret.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
        """
delete_secret_values(self, secret_id)

Deletes secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

Source code in zenml/zen_stores/secrets_stores/secrets_store_interface.py
@abstractmethod
def delete_secret_values(self, secret_id: UUID) -> None:
    """Deletes secret values for an existing secret.

    Args:
        secret_id: The ID of the secret.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
    """
get_secret_values(self, secret_id)

Get the secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required

Returns:

Type Description
Dict[str, str]

The secret values.

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

Source code in zenml/zen_stores/secrets_stores/secrets_store_interface.py
@abstractmethod
def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
    """Get the secret values for an existing secret.

    Args:
        secret_id: ID of the secret.

    Returns:
        The secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
    """
store_secret_values(self, secret_id, secret_values)

Store secret values for a new secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required
secret_values Dict[str, str]

Values for the secret.

required
Source code in zenml/zen_stores/secrets_stores/secrets_store_interface.py
@abstractmethod
def store_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Store secret values for a new secret.

    Args:
        secret_id: ID of the secret.
        secret_values: Values for the secret.
    """
update_secret_values(self, secret_id, secret_values)

Updates secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_values Dict[str, str]

The new secret values.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

Source code in zenml/zen_stores/secrets_stores/secrets_store_interface.py
@abstractmethod
def update_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Updates secret values for an existing secret.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_values: The new secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
    """

service_connector_secrets_store

Base secrets store class used for all secrets stores that use a service connector.

ServiceConnectorSecretsStore (BaseSecretsStore) pydantic-model

Base secrets store class for service connector-based secrets stores.

All secrets store implementations that use a Service Connector to authenticate and connect to the secrets store back-end should inherit from this class and:

  • implement the _initialize_client_from_connector method
  • use a configuration class that inherits from ServiceConnectorSecretsStoreConfiguration
  • set the SERVICE_CONNECTOR_TYPE to the service connector type used to connect to the secrets store back-end
  • set the SERVICE_CONNECTOR_RESOURCE_TYPE to the resource type used to connect to the secrets store back-end
Source code in zenml/zen_stores/secrets_stores/service_connector_secrets_store.py
class ServiceConnectorSecretsStore(BaseSecretsStore):
    """Base secrets store class for service connector-based secrets stores.

    All secrets store implementations that use a Service Connector to
    authenticate and connect to the secrets store back-end should inherit from
    this class and:

    * implement the `_initialize_client_from_connector` method
    * use a configuration class that inherits from
    `ServiceConnectorSecretsStoreConfiguration`
    * set the `SERVICE_CONNECTOR_TYPE` to the service connector type used
    to connect to the secrets store back-end
    * set the `SERVICE_CONNECTOR_RESOURCE_TYPE` to the resource type used
    to connect to the secrets store back-end
    """

    config: ServiceConnectorSecretsStoreConfiguration
    CONFIG_TYPE: ClassVar[Type[ServiceConnectorSecretsStoreConfiguration]]
    SERVICE_CONNECTOR_TYPE: ClassVar[str]
    SERVICE_CONNECTOR_RESOURCE_TYPE: ClassVar[str]

    _connector: Optional[ServiceConnector] = None
    _client: Optional[Any] = None
    _lock: Optional[Lock] = None

    def _initialize(self) -> None:
        """Initialize the secrets store."""
        self._lock = Lock()
        # Initialize the client early, just to catch any configuration or
        # authentication errors early, before the Secrets Store is used.
        _ = self.client

    def _get_client(self) -> Any:
        """Initialize and return the secrets store API client.

        Returns:
            The secrets store API client object.
        """
        if self._connector is not None:
            # If the client connector expires, we'll try to get a new one.
            if self._connector.has_expired():
                self._connector = None
                self._client = None

        if self._connector is None:
            # Initialize a base service connector with the credentials from
            # the configuration.
            request = ServiceConnectorRequest(
                name="secrets-store",
                connector_type=self.SERVICE_CONNECTOR_TYPE,
                resource_types=[self.SERVICE_CONNECTOR_RESOURCE_TYPE],
                user=uuid4(),  # Use a fake user ID
                workspace=uuid4(),  # Use a fake workspace ID
                auth_method=self.config.auth_method,
                configuration=self.config.auth_config,
            )
            base_connector = service_connector_registry.instantiate_connector(
                model=request
            )
            # Set the `allow_implicit_auth_methods` flag to `True` to allow
            # implicit authentication methods to be used even when not globally
            # enabled via the `ZENML_ENABLE_IMPLICIT_AUTH_METHODS` environment
            # variable.
            base_connector.allow_implicit_auth_methods = True
            self._connector = base_connector.get_connector_client()

        if self._client is None:
            # Use the connector to get a client object.
            client = self._connector.connect(
                # Don't verify again because we already did that when we
                # initialized the connector.
                verify=False
            )

            self._client = self._initialize_client_from_connector(client)
        return self._client

    @property
    def lock(self) -> Lock:
        """Get the lock used to treat the client initialization as a critical section.

        Returns:
            The lock instance.
        """
        assert self._lock is not None
        return self._lock

    @property
    def client(self) -> Any:
        """Get the secrets store API client.

        Returns:
            The secrets store API client instance.
        """
        # Multiple API calls can be made to this secrets store in the context
        # of different threads. We want to make sure that we only initialize
        # the client once, and then reuse it. We have to use a lock to treat
        # this method as a critical section.
        with self.lock:
            return self._get_client()

    @abstractmethod
    def _initialize_client_from_connector(self, client: Any) -> Any:
        """Initialize the client from the service connector.

        Args:
            client: The authenticated client object returned by the service
                connector.

        Returns:
            The initialized client instance.
        """
client: Any property readonly

Get the secrets store API client.

Returns:

Type Description
Any

The secrets store API client instance.

lock: <built-in function allocate_lock> property readonly

Get the lock used to treat the client initialization as a critical section.

Returns:

Type Description
<built-in function allocate_lock>

The lock instance.

ServiceConnectorSecretsStoreConfiguration (SecretsStoreConfiguration) pydantic-model

Base configuration for secrets stores that use a service connector.

Attributes:

Name Type Description
auth_method str

The service connector authentication method to use.

auth_config Dict[str, Any]

The service connector authentication configuration.

Source code in zenml/zen_stores/secrets_stores/service_connector_secrets_store.py
class ServiceConnectorSecretsStoreConfiguration(SecretsStoreConfiguration):
    """Base configuration for secrets stores that use a service connector.

    Attributes:
        auth_method: The service connector authentication method to use.
        auth_config: The service connector authentication configuration.
    """

    auth_method: str
    auth_config: Dict[str, Any] = Field(default_factory=dict)

    @root_validator(pre=True)
    def validate_auth_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Convert the authentication configuration if given in JSON format.

        Args:
            values: The configuration values.

        Returns:
            The validated configuration values.

        Raises:
            ValueError: If the authentication configuration is not a valid
                JSON object.
        """
        if isinstance(values.get("auth_config"), str):
            try:
                values["auth_config"] = json.loads(values["auth_config"])
            except json.JSONDecodeError as e:
                raise ValueError(
                    f"The authentication configuration is not a valid JSON "
                    f"object: {e}"
                )
        return values
validate_auth_config(values) classmethod

Convert the authentication configuration if given in JSON format.

Parameters:

Name Type Description Default
values Dict[str, Any]

The configuration values.

required

Returns:

Type Description
Dict[str, Any]

The validated configuration values.

Exceptions:

Type Description
ValueError

If the authentication configuration is not a valid JSON object.

Source code in zenml/zen_stores/secrets_stores/service_connector_secrets_store.py
@root_validator(pre=True)
def validate_auth_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
    """Convert the authentication configuration if given in JSON format.

    Args:
        values: The configuration values.

    Returns:
        The validated configuration values.

    Raises:
        ValueError: If the authentication configuration is not a valid
            JSON object.
    """
    if isinstance(values.get("auth_config"), str):
        try:
            values["auth_config"] = json.loads(values["auth_config"])
        except json.JSONDecodeError as e:
            raise ValueError(
                f"The authentication configuration is not a valid JSON "
                f"object: {e}"
            )
    return values

sql_secrets_store

SQL Secrets Store implementation.

SqlSecretsStore (BaseSecretsStore) pydantic-model

Secrets store implementation that uses the SQL ZenML store as a backend.

This secrets store piggybacks on the SQL ZenML store. It uses the same database and configuration as the SQL ZenML store.

Attributes:

Name Type Description
config

The configuration of the SQL secrets store.

TYPE

The type of the store.

CONFIG_TYPE

The type of the store configuration.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
class SqlSecretsStore(BaseSecretsStore):
    """Secrets store implementation that uses the SQL ZenML store as a backend.

    This secrets store piggybacks on the SQL ZenML store. It uses the same
    database and configuration as the SQL ZenML store.

    Attributes:
        config: The configuration of the SQL secrets store.
        TYPE: The type of the store.
        CONFIG_TYPE: The type of the store configuration.
    """

    config: SqlSecretsStoreConfiguration
    TYPE: ClassVar[SecretsStoreType] = SecretsStoreType.SQL
    CONFIG_TYPE: ClassVar[Type[SecretsStoreConfiguration]] = (
        SqlSecretsStoreConfiguration
    )

    _encryption_engine: Optional[AesGcmEngine] = None

    def __init__(
        self,
        zen_store: "BaseZenStore",
        **kwargs: Any,
    ) -> None:
        """Create and initialize the SQL secrets store.

        Args:
            zen_store: The ZenML store that owns this SQL secrets store.
            **kwargs: Additional keyword arguments to pass to the Pydantic
                constructor.

        Raises:
            IllegalOperationError: If the ZenML store to which this secrets
                store belongs is not a SQL ZenML store.
        """
        from zenml.zen_stores.sql_zen_store import SqlZenStore

        if not isinstance(zen_store, SqlZenStore):
            raise IllegalOperationError(
                "The SQL secrets store can only be used with the SQL ZenML "
                "store."
            )
        super().__init__(zen_store, **kwargs)

    @property
    def engine(self) -> Engine:
        """The SQLAlchemy engine.

        Returns:
            The SQLAlchemy engine.
        """
        return self.zen_store.engine

    @property
    def zen_store(self) -> "SqlZenStore":
        """The ZenML store that this SQL secrets store is using as a back-end.

        Returns:
            The ZenML store that this SQL secrets store is using as a back-end.

        Raises:
            ValueError: If the store is not initialized.
        """
        from zenml.zen_stores.sql_zen_store import SqlZenStore

        if not self._zen_store:
            raise ValueError("Store not initialized")
        assert isinstance(self._zen_store, SqlZenStore)
        return self._zen_store

    # ====================================
    # Secrets Store interface implementation
    # ====================================

    # --------------------------------
    # Initialization and configuration
    # --------------------------------

    def _initialize(self) -> None:
        """Initialize the secrets SQL store."""
        logger.debug("Initializing SqlSecretsStore")

        # Initialize the encryption engine
        if self.config.encryption_key:
            self._encryption_engine = AesGcmEngine()
            self._encryption_engine._update_key(self.config.encryption_key)

        # Nothing else to do here, the SQL ZenML store back-end is already
        # initialized

    # ------
    # Secrets
    # ------

    def store_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Store secret values for a new secret.

        The secret is already created in the database by the SQL Zen store, this
        method only stores the secret values.

        Args:
            secret_id: ID of the secret.
            secret_values: Values for the secret.

        Raises:
            KeyError: if a secret for the given ID is not found.
        """
        with Session(self.engine) as session:
            secret_in_db = session.exec(
                select(SecretSchema).where(SecretSchema.id == secret_id)
            ).first()
            if secret_in_db is None:
                raise KeyError(f"Secret with ID {secret_id} not found.")
            secret_in_db.set_secret_values(
                secret_values=secret_values,
                encryption_engine=self._encryption_engine,
            )
            session.add(secret_in_db)
            session.commit()

    def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
        """Get the secret values for an existing secret.

        Args:
            secret_id: ID of the secret.

        Returns:
            The secret values.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
        """
        with Session(self.engine) as session:
            secret_in_db = session.exec(
                select(SecretSchema).where(SecretSchema.id == secret_id)
            ).first()
            if secret_in_db is None:
                raise KeyError(f"Secret with ID {secret_id} not found.")
            try:
                return secret_in_db.get_secret_values(
                    encryption_engine=self._encryption_engine,
                )
            except SecretDecodeError:
                raise KeyError(
                    f"Secret values for secret {secret_id} could not be "
                    f"decoded. This can happen if encryption has "
                    f"been enabled/disabled or if the encryption key has been "
                    "reconfigured without proper secrets migration."
                )

    def update_secret_values(
        self,
        secret_id: UUID,
        secret_values: Dict[str, str],
    ) -> None:
        """Updates secret values for an existing secret.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_values: The new secret values.
        """
        self.store_secret_values(secret_id, secret_values)

    def delete_secret_values(self, secret_id: UUID) -> None:
        """Deletes secret values for an existing secret.

        Args:
            secret_id: The ID of the secret.

        Raises:
            KeyError: if no secret values for the given ID are stored in the
                secrets store.
        """
        with Session(self.engine) as session:
            try:
                secret_in_db = session.exec(
                    select(SecretSchema).where(SecretSchema.id == secret_id)
                ).one()
                secret_in_db.values = None
                session.commit()
            except NoResultFound:
                raise KeyError(f"Secret with ID {secret_id} not found.")
engine: Engine property readonly

The SQLAlchemy engine.

Returns:

Type Description
Engine

The SQLAlchemy engine.

zen_store: SqlZenStore property readonly

The ZenML store that this SQL secrets store is using as a back-end.

Returns:

Type Description
SqlZenStore

The ZenML store that this SQL secrets store is using as a back-end.

Exceptions:

Type Description
ValueError

If the store is not initialized.

CONFIG_TYPE (SecretsStoreConfiguration) pydantic-model

SQL secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

encryption_key Optional[str]

The encryption key to use for the SQL secrets store. If not set, the passwords will not be encrypted in the database.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
class SqlSecretsStoreConfiguration(SecretsStoreConfiguration):
    """SQL secrets store configuration.

    Attributes:
        type: The type of the store.
        encryption_key: The encryption key to use for the SQL secrets store.
            If not set, the passwords will not be encrypted in the database.
    """

    type: SecretsStoreType = SecretsStoreType.SQL
    encryption_key: Optional[str] = None

    class Config:
        """Pydantic configuration class."""

        # Don't validate attributes when assigning them. This is necessary
        # because the certificate attributes can be expanded to the contents
        # of the certificate files.
        validate_assignment = False
        # Forbid extra attributes set in the class.
        extra = "forbid"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Don't validate attributes when assigning them. This is necessary
    # because the certificate attributes can be expanded to the contents
    # of the certificate files.
    validate_assignment = False
    # Forbid extra attributes set in the class.
    extra = "forbid"
__init__(self, zen_store, **kwargs) special

Create and initialize the SQL secrets store.

Parameters:

Name Type Description Default
zen_store BaseZenStore

The ZenML store that owns this SQL secrets store.

required
**kwargs Any

Additional keyword arguments to pass to the Pydantic constructor.

{}

Exceptions:

Type Description
IllegalOperationError

If the ZenML store to which this secrets store belongs is not a SQL ZenML store.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
def __init__(
    self,
    zen_store: "BaseZenStore",
    **kwargs: Any,
) -> None:
    """Create and initialize the SQL secrets store.

    Args:
        zen_store: The ZenML store that owns this SQL secrets store.
        **kwargs: Additional keyword arguments to pass to the Pydantic
            constructor.

    Raises:
        IllegalOperationError: If the ZenML store to which this secrets
            store belongs is not a SQL ZenML store.
    """
    from zenml.zen_stores.sql_zen_store import SqlZenStore

    if not isinstance(zen_store, SqlZenStore):
        raise IllegalOperationError(
            "The SQL secrets store can only be used with the SQL ZenML "
            "store."
        )
    super().__init__(zen_store, **kwargs)
delete_secret_values(self, secret_id)

Deletes secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret.

required

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
def delete_secret_values(self, secret_id: UUID) -> None:
    """Deletes secret values for an existing secret.

    Args:
        secret_id: The ID of the secret.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
    """
    with Session(self.engine) as session:
        try:
            secret_in_db = session.exec(
                select(SecretSchema).where(SecretSchema.id == secret_id)
            ).one()
            secret_in_db.values = None
            session.commit()
        except NoResultFound:
            raise KeyError(f"Secret with ID {secret_id} not found.")
get_secret_values(self, secret_id)

Get the secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required

Returns:

Type Description
Dict[str, str]

The secret values.

Exceptions:

Type Description
KeyError

if no secret values for the given ID are stored in the secrets store.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
def get_secret_values(self, secret_id: UUID) -> Dict[str, str]:
    """Get the secret values for an existing secret.

    Args:
        secret_id: ID of the secret.

    Returns:
        The secret values.

    Raises:
        KeyError: if no secret values for the given ID are stored in the
            secrets store.
    """
    with Session(self.engine) as session:
        secret_in_db = session.exec(
            select(SecretSchema).where(SecretSchema.id == secret_id)
        ).first()
        if secret_in_db is None:
            raise KeyError(f"Secret with ID {secret_id} not found.")
        try:
            return secret_in_db.get_secret_values(
                encryption_engine=self._encryption_engine,
            )
        except SecretDecodeError:
            raise KeyError(
                f"Secret values for secret {secret_id} could not be "
                f"decoded. This can happen if encryption has "
                f"been enabled/disabled or if the encryption key has been "
                "reconfigured without proper secrets migration."
            )
store_secret_values(self, secret_id, secret_values)

Store secret values for a new secret.

The secret is already created in the database by the SQL Zen store, this method only stores the secret values.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required
secret_values Dict[str, str]

Values for the secret.

required

Exceptions:

Type Description
KeyError

if a secret for the given ID is not found.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
def store_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Store secret values for a new secret.

    The secret is already created in the database by the SQL Zen store, this
    method only stores the secret values.

    Args:
        secret_id: ID of the secret.
        secret_values: Values for the secret.

    Raises:
        KeyError: if a secret for the given ID is not found.
    """
    with Session(self.engine) as session:
        secret_in_db = session.exec(
            select(SecretSchema).where(SecretSchema.id == secret_id)
        ).first()
        if secret_in_db is None:
            raise KeyError(f"Secret with ID {secret_id} not found.")
        secret_in_db.set_secret_values(
            secret_values=secret_values,
            encryption_engine=self._encryption_engine,
        )
        session.add(secret_in_db)
        session.commit()
update_secret_values(self, secret_id, secret_values)

Updates secret values for an existing secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_values Dict[str, str]

The new secret values.

required
Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
def update_secret_values(
    self,
    secret_id: UUID,
    secret_values: Dict[str, str],
) -> None:
    """Updates secret values for an existing secret.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_values: The new secret values.
    """
    self.store_secret_values(secret_id, secret_values)
SqlSecretsStoreConfiguration (SecretsStoreConfiguration) pydantic-model

SQL secrets store configuration.

Attributes:

Name Type Description
type SecretsStoreType

The type of the store.

encryption_key Optional[str]

The encryption key to use for the SQL secrets store. If not set, the passwords will not be encrypted in the database.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
class SqlSecretsStoreConfiguration(SecretsStoreConfiguration):
    """SQL secrets store configuration.

    Attributes:
        type: The type of the store.
        encryption_key: The encryption key to use for the SQL secrets store.
            If not set, the passwords will not be encrypted in the database.
    """

    type: SecretsStoreType = SecretsStoreType.SQL
    encryption_key: Optional[str] = None

    class Config:
        """Pydantic configuration class."""

        # Don't validate attributes when assigning them. This is necessary
        # because the certificate attributes can be expanded to the contents
        # of the certificate files.
        validate_assignment = False
        # Forbid extra attributes set in the class.
        extra = "forbid"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/secrets_stores/sql_secrets_store.py
class Config:
    """Pydantic configuration class."""

    # Don't validate attributes when assigning them. This is necessary
    # because the certificate attributes can be expanded to the contents
    # of the certificate files.
    validate_assignment = False
    # Forbid extra attributes set in the class.
    extra = "forbid"

sql_zen_store

SQL Zen Store implementation.

SQLDatabaseDriver (StrEnum)

SQL database drivers supported by the SQL ZenML store.

Source code in zenml/zen_stores/sql_zen_store.py
class SQLDatabaseDriver(StrEnum):
    """SQL database drivers supported by the SQL ZenML store."""

    MYSQL = "mysql"
    SQLITE = "sqlite"

SqlZenStore (BaseZenStore) pydantic-model

Store Implementation that uses SQL database backend.

Attributes:

Name Type Description
config SqlZenStoreConfiguration

The configuration of the SQL ZenML store.

skip_migrations bool

Whether to skip migrations when initializing the store.

TYPE ClassVar[zenml.enums.StoreType]

The type of the store.

CONFIG_TYPE ClassVar[Type[zenml.config.store_config.StoreConfiguration]]

The type of the store configuration.

_engine Optional[sqlalchemy.engine.base.Engine]

The SQLAlchemy engine.

Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStore(BaseZenStore):
    """Store Implementation that uses SQL database backend.

    Attributes:
        config: The configuration of the SQL ZenML store.
        skip_migrations: Whether to skip migrations when initializing the store.
        TYPE: The type of the store.
        CONFIG_TYPE: The type of the store configuration.
        _engine: The SQLAlchemy engine.
    """

    config: SqlZenStoreConfiguration
    skip_migrations: bool = False
    TYPE: ClassVar[StoreType] = StoreType.SQL
    CONFIG_TYPE: ClassVar[Type[StoreConfiguration]] = SqlZenStoreConfiguration

    _engine: Optional[Engine] = None
    _migration_utils: Optional[MigrationUtils] = None
    _alembic: Optional[Alembic] = None
    _secrets_store: Optional[BaseSecretsStore] = None
    _backup_secrets_store: Optional[BaseSecretsStore] = None

    @property
    def secrets_store(self) -> "BaseSecretsStore":
        """The secrets store associated with this store.

        Returns:
            The secrets store associated with this store.

        Raises:
            SecretsStoreNotConfiguredError: If no secrets store is configured.
        """
        if self._secrets_store is None:
            raise SecretsStoreNotConfiguredError(
                "No secrets store is configured. Please configure a secrets "
                "store to create and manage ZenML secrets."
            )

        return self._secrets_store

    @property
    def backup_secrets_store(self) -> Optional["BaseSecretsStore"]:
        """The backup secrets store associated with this store.

        Returns:
            The backup secrets store associated with this store.
        """
        return self._backup_secrets_store

    @property
    def engine(self) -> Engine:
        """The SQLAlchemy engine.

        Returns:
            The SQLAlchemy engine.

        Raises:
            ValueError: If the store is not initialized.
        """
        if not self._engine:
            raise ValueError("Store not initialized")
        return self._engine

    @property
    def migration_utils(self) -> MigrationUtils:
        """The migration utils.

        Returns:
            The migration utils.

        Raises:
            ValueError: If the store is not initialized.
        """
        if not self._migration_utils:
            raise ValueError("Store not initialized")
        return self._migration_utils

    @property
    def alembic(self) -> Alembic:
        """The Alembic wrapper.

        Returns:
            The Alembic wrapper.

        Raises:
            ValueError: If the store is not initialized.
        """
        if not self._alembic:
            raise ValueError("Store not initialized")
        return self._alembic

    @classmethod
    def filter_and_paginate(
        cls,
        session: Session,
        query: Union[Select[Any], SelectOfScalar[Any]],
        table: Type[AnySchema],
        filter_model: BaseFilter,
        custom_schema_to_model_conversion: Optional[
            Callable[..., AnyResponse]
        ] = None,
        custom_fetch: Optional[
            Callable[
                [
                    Session,
                    Union[Select[Any], SelectOfScalar[Any]],
                    BaseFilter,
                ],
                List[Any],
            ]
        ] = None,
        hydrate: bool = False,
    ) -> Page[AnyResponse]:
        """Given a query, return a Page instance with a list of filtered Models.

        Args:
            session: The SQLModel Session
            query: The query to execute
            table: The table to select from
            filter_model: The filter to use, including pagination and sorting
            custom_schema_to_model_conversion: Callable to convert the schema
                into a model. This is used if the Model contains additional
                data that is not explicitly stored as a field or relationship
                on the model.
            custom_fetch: Custom callable to use to fetch items from the
                database for a given query. This is used if the items fetched
                from the database need to be processed differently (e.g. to
                perform additional filtering). The callable should take a
                `Session`, a `Select` query and a `BaseFilterModel` filter as
                arguments and return a `List` of items.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The Domain Model representation of the DB resource

        Raises:
            ValueError: if the filtered page number is out of bounds.
            RuntimeError: if the schema does not have a `to_model` method.
        """
        query = filter_model.apply_filter(query=query, table=table)

        # Get the total amount of items in the database for a given query
        custom_fetch_result: Optional[List[Any]] = None
        if custom_fetch:
            custom_fetch_result = custom_fetch(session, query, filter_model)
            total = len(custom_fetch_result)
        else:
            total = session.scalar(
                select([func.count("*")]).select_from(
                    query.options(noload("*")).subquery()
                )
            )

        # Sorting
        column, operand = filter_model.sorting_params
        if operand == SorterOps.DESCENDING:
            query = query.order_by(desc(getattr(table, column)))
        else:
            query = query.order_by(asc(getattr(table, column)))

        # Get the total amount of pages in the database for a given query
        if total == 0:
            total_pages = 1
        else:
            total_pages = math.ceil(total / filter_model.size)

        if filter_model.page > total_pages:
            raise ValueError(
                f"Invalid page {filter_model.page}. The requested page size is "
                f"{filter_model.size} and there are a total of {total} items "
                f"for this query. The maximum page value therefore is "
                f"{total_pages}."
            )

        # Get a page of the actual data
        item_schemas: List[AnySchema]
        if custom_fetch:
            assert custom_fetch_result is not None
            item_schemas = custom_fetch_result
            # select the items in the current page
            item_schemas = item_schemas[
                filter_model.offset : filter_model.offset + filter_model.size
            ]
        else:
            item_schemas = (
                session.exec(
                    query.limit(filter_model.size).offset(filter_model.offset)
                )
                .unique()
                .all()
            )

        # Convert this page of items from schemas to models.
        items: List[AnyResponse] = []
        for schema in item_schemas:
            # If a custom conversion function is provided, use it.
            if custom_schema_to_model_conversion:
                items.append(custom_schema_to_model_conversion(schema))
                continue
            # Otherwise, try to use the `to_model` method of the schema.
            to_model = getattr(schema, "to_model", None)
            if callable(to_model):
                items.append(to_model(include_metadata=hydrate))
                continue
            # If neither of the above work, raise an error.
            raise RuntimeError(
                f"Cannot convert schema `{schema.__class__.__name__}` to model "
                "since it does not have a `to_model` method."
            )

        return Page[Any](
            total=total,
            total_pages=total_pages,
            items=items,
            index=filter_model.page,
            max_size=filter_model.size,
        )

    # ====================================
    # ZenML Store interface implementation
    # ====================================

    # --------------------------------
    # Initialization and configuration
    # --------------------------------

    def _initialize(self) -> None:
        """Initialize the SQL store."""
        logger.debug("Initializing SqlZenStore at %s", self.config.url)

        url, connect_args, engine_args = self.config.get_sqlalchemy_config()
        self._engine = create_engine(
            url=url, connect_args=connect_args, **engine_args
        )
        self._migration_utils = MigrationUtils(
            url=url,
            connect_args=connect_args,
            engine_args=engine_args,
        )

        # SQLite: As long as the parent directory exists, SQLAlchemy will
        # automatically create the database.
        if (
            self.config.driver == SQLDatabaseDriver.SQLITE
            and self.config.database
            and not fileio.exists(self.config.database)
        ):
            fileio.makedirs(os.path.dirname(self.config.database))

        # MySQL: We might need to create the database manually.
        # To do so, we create a new engine that connects to the `mysql` database
        # and then create the desired database.
        # See https://stackoverflow.com/a/8977109
        if (
            self.config.driver == SQLDatabaseDriver.MYSQL
            and self.config.database
        ):
            if not self.migration_utils.database_exists():
                self.migration_utils.create_database()

        self._alembic = Alembic(self.engine)

        if (
            not self.skip_migrations
            and ENV_ZENML_DISABLE_DATABASE_MIGRATION not in os.environ
        ):
            self.migrate_database()

        secrets_store_config = self.config.secrets_store

        # Initialize the secrets store
        if (
            secrets_store_config
            and secrets_store_config.type != SecretsStoreType.NONE
        ):
            secrets_store_class = BaseSecretsStore.get_store_class(
                secrets_store_config
            )
            self._secrets_store = secrets_store_class(
                zen_store=self,
                config=secrets_store_config,
            )
            # Update the config with the actual secrets store config
            # to reflect the default values in the saved configuration
            self.config.secrets_store = self._secrets_store.config

        backup_secrets_store_config = self.config.backup_secrets_store

        # Initialize the backup secrets store, if configured
        if (
            backup_secrets_store_config
            and backup_secrets_store_config.type != SecretsStoreType.NONE
        ):
            secrets_store_class = BaseSecretsStore.get_store_class(
                backup_secrets_store_config
            )
            self._backup_secrets_store = secrets_store_class(
                zen_store=self,
                config=backup_secrets_store_config,
            )
            # Update the config with the actual secrets store config
            # to reflect the default values in the saved configuration
            self.config.backup_secrets_store = (
                self._backup_secrets_store.config
            )

    def _initialize_database(self) -> None:
        """Initialize the database on first use."""
        self._get_or_create_default_workspace()

        config = ServerConfiguration.get_server_config()
        # If the auth scheme is external, don't create the default user
        if config.auth_scheme != AuthScheme.EXTERNAL:
            self._get_or_create_default_user()

    def _get_db_backup_file_path(self) -> str:
        """Get the path to the database backup file.

        Returns:
            The path to the configured database backup file.
        """
        if self.config.driver == SQLDatabaseDriver.SQLITE:
            return os.path.join(
                self.config.backup_directory,
                # Add the -backup suffix to the database filename
                ZENML_SQLITE_DB_FILENAME[:-3] + "-backup.db",
            )

        # For a MySQL database, we need to dump the database to a JSON
        # file
        return os.path.join(
            self.config.backup_directory,
            f"{self.engine.url.database}-backup.json",
        )

    def backup_database(
        self,
        strategy: Optional[DatabaseBackupStrategy] = None,
        location: Optional[str] = None,
        overwrite: bool = False,
    ) -> Tuple[str, Any]:
        """Backup the database.

        Args:
            strategy: Custom backup strategy to use. If not set, the backup
                strategy from the store configuration will be used.
            location: Custom target location to backup the database to. If not
                set, the configured backup location will be used. Depending on
                the backup strategy, this can be a file path or a database name.
            overwrite: Whether to overwrite an existing backup if it exists.
                If set to False, the existing backup will be reused.

        Returns:
            The location where the database was backed up to and an accompanying
            user-friendly message that describes the backup location, or None
            if no backup was created (i.e. because the backup already exists).

        Raises:
            ValueError: If the backup database name is not set when the backup
                database is requested or if the backup strategy is invalid.
        """
        strategy = strategy or self.config.backup_strategy

        if (
            strategy == DatabaseBackupStrategy.DUMP_FILE
            or self.config.driver == SQLDatabaseDriver.SQLITE
        ):
            dump_file = location or self._get_db_backup_file_path()

            if not overwrite and os.path.isfile(dump_file):
                logger.warning(
                    f"A previous backup file already exists at '{dump_file}'. "
                    "Reusing the existing backup."
                )
            else:
                self.migration_utils.backup_database_to_file(
                    dump_file=dump_file
                )
            return f"the '{dump_file}' backup file", dump_file
        elif strategy == DatabaseBackupStrategy.DATABASE:
            backup_db_name = location or self.config.backup_database
            if not backup_db_name:
                raise ValueError(
                    "The backup database name must be set in the store "
                    "configuration to use the backup database strategy."
                )

            if not overwrite and self.migration_utils.database_exists(
                backup_db_name
            ):
                logger.warning(
                    "A previous backup database already exists at "
                    f"'{backup_db_name}'. Reusing the existing backup."
                )
            else:
                self.migration_utils.backup_database_to_db(
                    backup_db_name=backup_db_name
                )
            return f"the '{backup_db_name}' backup database", backup_db_name
        elif strategy == DatabaseBackupStrategy.IN_MEMORY:
            return (
                "memory",
                self.migration_utils.backup_database_to_memory(),
            )

        else:
            raise ValueError(f"Invalid backup strategy: {strategy}.")

    def restore_database(
        self,
        strategy: Optional[DatabaseBackupStrategy] = None,
        location: Optional[Any] = None,
        cleanup: bool = False,
    ) -> None:
        """Restore the database.

        Args:
            strategy: Custom backup strategy to use. If not set, the backup
                strategy from the store configuration will be used.
            location: Custom target location to restore the database from. If
                not set, the configured backup location will be used. Depending
                on the backup strategy, this can be a file path, a database
                name or an in-memory database representation.
            cleanup: Whether to cleanup the backup after restoring the database.

        Raises:
            ValueError: If the backup database name is not set when the backup
                database is requested or if the backup strategy is invalid.
        """
        strategy = strategy or self.config.backup_strategy

        if (
            strategy == DatabaseBackupStrategy.DUMP_FILE
            or self.config.driver == SQLDatabaseDriver.SQLITE
        ):
            dump_file = location or self._get_db_backup_file_path()
            self.migration_utils.restore_database_from_file(
                dump_file=dump_file
            )
        elif strategy == DatabaseBackupStrategy.DATABASE:
            backup_db_name = location or self.config.backup_database
            if not backup_db_name:
                raise ValueError(
                    "The backup database name must be set in the store "
                    "configuration to use the backup database strategy."
                )

            self.migration_utils.restore_database_from_db(
                backup_db_name=backup_db_name
            )
        elif strategy == DatabaseBackupStrategy.IN_MEMORY:
            if location is None or not isinstance(location, list):
                raise ValueError(
                    "The in-memory database representation must be provided "
                    "to restore the database from an in-memory backup."
                )
            self.migration_utils.restore_database_from_memory(db_dump=location)

        else:
            raise ValueError(f"Invalid backup strategy: {strategy}.")

        if cleanup:
            self.cleanup_database_backup()

    def cleanup_database_backup(
        self,
        strategy: Optional[DatabaseBackupStrategy] = None,
        location: Optional[Any] = None,
    ) -> None:
        """Delete the database backup.

        Args:
            strategy: Custom backup strategy to use. If not set, the backup
                strategy from the store configuration will be used.
            location: Custom target location to delete the database backup
                from. If not set, the configured backup location will be used.
                Depending on the backup strategy, this can be a file path or a
                database name.

        Raises:
            ValueError: If the backup database name is not set when the backup
                database is requested.
        """
        strategy = strategy or self.config.backup_strategy

        if (
            strategy == DatabaseBackupStrategy.DUMP_FILE
            or self.config.driver == SQLDatabaseDriver.SQLITE
        ):
            dump_file = location or self._get_db_backup_file_path()
            if dump_file is not None and os.path.isfile(dump_file):
                try:
                    os.remove(dump_file)
                except OSError:
                    logger.warning(
                        f"Failed to cleanup database dump file "
                        f"{dump_file}."
                    )
                else:
                    logger.info(
                        f"Successfully cleaned up database dump file "
                        f"{dump_file}."
                    )
        elif strategy == DatabaseBackupStrategy.DATABASE:
            backup_db_name = location or self.config.backup_database

            if not backup_db_name:
                raise ValueError(
                    "The backup database name must be set in the store "
                    "configuration to use the backup database strategy."
                )
            if self.migration_utils.database_exists(backup_db_name):
                # Drop the backup database
                self.migration_utils.drop_database(
                    database=backup_db_name,
                )
                logger.info(
                    f"Successfully cleaned up backup database "
                    f"{backup_db_name}."
                )

    def migrate_database(self) -> None:
        """Migrate the database to the head as defined by the python package.

        Raises:
            RuntimeError: If the database exists and is not empty but has never
                been migrated with alembic before.
        """
        alembic_logger = logging.getLogger("alembic")

        # remove all existing handlers
        while len(alembic_logger.handlers):
            alembic_logger.removeHandler(alembic_logger.handlers[0])

        logging_level = get_logging_level()

        # suppress alembic info logging if the zenml logging level is not debug
        if logging_level == LoggingLevels.DEBUG:
            alembic_logger.setLevel(logging.DEBUG)
        else:
            alembic_logger.setLevel(logging.WARNING)

        alembic_logger.addHandler(get_console_handler())

        # We need to account for 3 distinct cases here:
        # 1. the database is completely empty (not initialized)
        # 2. the database is not empty and has been migrated with alembic before
        # 3. the database is not empty, but has never been migrated with alembic
        #   before (i.e. was created with SQLModel back when alembic wasn't
        #   used). We don't support this direct upgrade case anymore.
        current_revisions = self.alembic.current_revisions()
        head_revisions = self.alembic.head_revisions()
        if len(current_revisions) >= 1:
            # Case 2: the database has been migrated with alembic before. Just
            # upgrade to the latest revision.
            if len(current_revisions) > 1:
                logger.warning(
                    "The ZenML database has more than one migration head "
                    "revision. This is not expected and might indicate a "
                    "database migration problem. Please raise an issue on "
                    "GitHub if you encounter this."
                )

            logger.debug("Current revisions: %s", current_revisions)
            logger.debug("Head revisions: %s", head_revisions)

            # If the current revision and head revision don't match, a database
            # migration that changes the database structure or contents may
            # actually be performed, in which case we enable the backup
            # functionality. We only enable the backup functionality if the
            # database will actually be changed, to avoid the overhead for
            # unnecessary backups.
            backup_enabled = (
                self.config.backup_strategy != DatabaseBackupStrategy.DISABLED
                and set(current_revisions) != set(head_revisions)
            )
            backup_location: Optional[Any] = None
            backup_location_msg: Optional[str] = None

            if backup_enabled:
                try:
                    logger.info("Backing up the database before migration.")
                    (
                        backup_location_msg,
                        backup_location,
                    ) = self.backup_database(overwrite=True)
                except Exception as e:
                    raise RuntimeError(
                        f"Failed to backup the database: {str(e)}. "
                        "Please check the logs for more details."
                        "If you would like to disable the database backup "
                        "functionality, set the `backup_strategy` attribute "
                        "of the store configuration to `disabled`."
                    ) from e
                else:
                    if backup_location is not None:
                        logger.info(
                            "Database successfully backed up to "
                            f"{backup_location_msg}. If something goes wrong "
                            "with the upgrade, ZenML will attempt to restore "
                            "the database from this backup automatically."
                        )

            try:
                self.alembic.upgrade()
            except Exception as e:
                if backup_enabled and backup_location:
                    logger.exception(
                        "Failed to migrate the database. Attempting to restore "
                        f"the database from {backup_location_msg}."
                    )
                    try:
                        self.restore_database(location=backup_location)
                    except Exception:
                        logger.exception(
                            "Failed to restore the database from "
                            f"{backup_location_msg}. Please "
                            "check the logs for more details. You might need "
                            "to restore the database manually."
                        )
                    else:
                        raise RuntimeError(
                            "The database migration failed, but the database "
                            "was successfully restored from the backup. "
                            "You can safely retry the upgrade or revert to "
                            "the previous version of ZenML. Please check the "
                            "logs for more details."
                        ) from e
                raise RuntimeError(
                    f"The database migration failed: {str(e)}"
                ) from e

            else:
                # We always remove the backup after a successful upgrade,
                # not just to avoid cluttering the disk, but also to avoid
                # reusing an outdated database from the backup in case of
                # future upgrade failures.
                try:
                    self.cleanup_database_backup()
                except Exception:
                    logger.exception("Failed to cleanup the database backup.")

        elif self.alembic.db_is_empty():
            # Case 1: the database is empty. We can just create the
            # tables from scratch with from SQLModel. After tables are
            # created we put an alembic revision to latest and populate
            # identity table with needed info.
            logger.info("Creating database tables")
            with self.engine.begin() as conn:
                conn.run_callable(
                    SQLModel.metadata.create_all  # type: ignore[arg-type]
                )
            with Session(self.engine) as session:
                session.add(
                    IdentitySchema(
                        id=str(GlobalConfiguration().user_id).replace("-", "")
                    )
                )
                session.commit()
            self.alembic.stamp("head")
        else:
            # Case 3: the database is not empty, but has never been
            # migrated with alembic before. We don't support this direct
            # upgrade case anymore. The user needs to run a two-step
            # upgrade.
            raise RuntimeError(
                "The ZenML database has never been migrated with alembic "
                "before. This can happen if you are performing a direct "
                "upgrade from a really old version of ZenML. This direct "
                "upgrade path is not supported anymore. Please upgrade "
                "your ZenML installation first to 0.54.0 or an earlier "
                "version and then to the latest version."
            )

        # If an alembic migration took place, all non-custom flavors are purged
        #  and the FlavorRegistry recreates all in-built and integration
        #  flavors in the db.
        revisions_afterwards = self.alembic.current_revisions()

        if current_revisions != revisions_afterwards:
            self._sync_flavors()

    def _sync_flavors(self) -> None:
        """Purge all in-built and integration flavors from the DB and sync."""
        FlavorRegistry().register_flavors(store=self)

    def get_store_info(self) -> ServerModel:
        """Get information about the store.

        Returns:
            Information about the store.
        """
        model = super().get_store_info()
        sql_url = make_url(self.config.url)
        model.database_type = ServerDatabaseType(sql_url.drivername)
        # Fetch the deployment ID from the database and use it to replace
        # the one fetched from the global configuration
        model.id = self.get_deployment_id()
        return model

    def get_deployment_id(self) -> UUID:
        """Get the ID of the deployment.

        Returns:
            The ID of the deployment.

        Raises:
            KeyError: If the deployment ID could not be loaded from the
                database.
        """
        # Fetch the deployment ID from the database
        with Session(self.engine) as session:
            identity = session.exec(select(IdentitySchema)).first()

            if identity is None:
                raise KeyError(
                    "The deployment ID could not be loaded from the database."
                )
            return identity.id

    # ------------------------- API Keys -------------------------

    def _get_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        session: Session,
    ) -> APIKeySchema:
        """Helper method to fetch an API key by name or ID.

        Args:
            service_account_id: The ID of the service account for which to
                fetch the API key.
            api_key_name_or_id: The name or ID of the API key to get.
            session: The database session to use for the query.

        Returns:
            The requested API key.

        Raises:
            KeyError: if the name or ID does not identify an API key that is
                configured for the given service account.
        """
        # Fetch the service account, to make sure it exists
        service_account = self._get_account_schema(
            service_account_id, session=session, service_account=True
        )

        if uuid_utils.is_valid_uuid(api_key_name_or_id):
            filter_params = APIKeySchema.id == api_key_name_or_id
        else:
            filter_params = APIKeySchema.name == api_key_name_or_id

        api_key = session.exec(
            select(APIKeySchema)
            .where(filter_params)
            .where(APIKeySchema.service_account_id == service_account.id)
        ).first()

        if api_key is None:
            raise KeyError(
                f"An API key with ID or name '{api_key_name_or_id}' is not "
                f"configured for service account with ID "
                f"'{service_account_id}'."
            )
        return api_key

    def create_api_key(
        self, service_account_id: UUID, api_key: APIKeyRequest
    ) -> APIKeyResponse:
        """Create a new API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                create the API key.
            api_key: The API key to create.

        Returns:
            The created API key.

        Raises:
            EntityExistsError: If an API key with the same name is already
                configured for the same service account.
        """
        with Session(self.engine) as session:
            # Fetch the service account
            service_account = self._get_account_schema(
                service_account_id, session=session, service_account=True
            )

            # Check if a key with the same name already exists for the same
            # service account
            try:
                self._get_api_key(
                    service_account_id=service_account.id,
                    api_key_name_or_id=api_key.name,
                    session=session,
                )
                raise EntityExistsError(
                    f"Unable to register API key with name '{api_key.name}': "
                    "Found an existing API key with the same name configured "
                    f"for the same '{service_account.name}' service account."
                )
            except KeyError:
                pass

            new_api_key, key_value = APIKeySchema.from_request(
                service_account_id=service_account.id,
                request=api_key,
            )
            session.add(new_api_key)
            session.commit()

            api_key_model = new_api_key.to_model(include_metadata=True)
            api_key_model.set_key(key_value)
            return api_key_model

    def get_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        hydrate: bool = True,
    ) -> APIKeyResponse:
        """Get an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to fetch
                the API key.
            api_key_name_or_id: The name or ID of the API key to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The API key with the given ID.
        """
        with Session(self.engine) as session:
            api_key = self._get_api_key(
                service_account_id=service_account_id,
                api_key_name_or_id=api_key_name_or_id,
                session=session,
            )
            return api_key.to_model(include_metadata=hydrate)

    def get_internal_api_key(
        self, api_key_id: UUID, hydrate: bool = True
    ) -> APIKeyInternalResponse:
        """Get internal details for an API key by its unique ID.

        Args:
            api_key_id: The ID of the API key to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The internal details for the API key with the given ID.

        Raises:
            KeyError: if the API key doesn't exist.
        """
        with Session(self.engine) as session:
            api_key = session.exec(
                select(APIKeySchema).where(APIKeySchema.id == api_key_id)
            ).first()
            if api_key is None:
                raise KeyError(f"API key with ID {api_key_id} not found.")
            return api_key.to_internal_model(hydrate=hydrate)

    def list_api_keys(
        self,
        service_account_id: UUID,
        filter_model: APIKeyFilter,
        hydrate: bool = False,
    ) -> Page[APIKeyResponse]:
        """List all API keys for a service account matching the given filter criteria.

        Args:
            service_account_id: The ID of the service account for which to list
                the API keys.
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all API keys matching the filter criteria.
        """
        with Session(self.engine) as session:
            # Fetch the service account
            service_account = self._get_account_schema(
                service_account_id, session=session, service_account=True
            )

            filter_model.set_service_account(service_account.id)
            query = select(APIKeySchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=APIKeySchema,
                filter_model=filter_model,
                hydrate=hydrate,
            )

    def update_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        api_key_update: APIKeyUpdate,
    ) -> APIKeyResponse:
        """Update an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to update
                the API key.
            api_key_name_or_id: The name or ID of the API key to update.
            api_key_update: The update request on the API key.

        Returns:
            The updated API key.

        Raises:
            EntityExistsError: if the API key update would result in a name
                conflict with an existing API key for the same service account.
        """
        with Session(self.engine) as session:
            api_key = self._get_api_key(
                service_account_id=service_account_id,
                api_key_name_or_id=api_key_name_or_id,
                session=session,
            )

            if api_key_update.name and api_key.name != api_key_update.name:
                # Check if a key with the new name already exists for the same
                # service account
                try:
                    self._get_api_key(
                        service_account_id=service_account_id,
                        api_key_name_or_id=api_key_update.name,
                        session=session,
                    )

                    raise EntityExistsError(
                        f"Unable to update API key with name "
                        f"'{api_key_update.name}': Found an existing API key "
                        "with the same name configured for the same "
                        f"'{api_key.service_account.name}' service account."
                    )
                except KeyError:
                    pass

            api_key.update(update=api_key_update)
            session.add(api_key)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(api_key)
            return api_key.to_model(include_metadata=True)

    def update_internal_api_key(
        self, api_key_id: UUID, api_key_update: APIKeyInternalUpdate
    ) -> APIKeyResponse:
        """Update an API key with internal details.

        Args:
            api_key_id: The ID of the API key.
            api_key_update: The update request on the API key.

        Returns:
            The updated API key.

        Raises:
            KeyError: if the API key doesn't exist.
        """
        with Session(self.engine) as session:
            api_key = session.exec(
                select(APIKeySchema).where(APIKeySchema.id == api_key_id)
            ).first()

            if not api_key:
                raise KeyError(f"API key with ID {api_key_id} not found.")

            api_key.internal_update(update=api_key_update)
            session.add(api_key)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(api_key)
            return api_key.to_model(include_metadata=True)

    def rotate_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        rotate_request: APIKeyRotateRequest,
    ) -> APIKeyResponse:
        """Rotate an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                rotate the API key.
            api_key_name_or_id: The name or ID of the API key to rotate.
            rotate_request: The rotate request on the API key.

        Returns:
            The updated API key.
        """
        with Session(self.engine) as session:
            api_key = self._get_api_key(
                service_account_id=service_account_id,
                api_key_name_or_id=api_key_name_or_id,
                session=session,
            )

            _, new_key = api_key.rotate(rotate_request)
            session.add(api_key)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(api_key)
            api_key_model = api_key.to_model()
            api_key_model.set_key(new_key)

            return api_key_model

    def delete_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
    ) -> None:
        """Delete an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                delete the API key.
            api_key_name_or_id: The name or ID of the API key to delete.
        """
        with Session(self.engine) as session:
            api_key = self._get_api_key(
                service_account_id=service_account_id,
                api_key_name_or_id=api_key_name_or_id,
                session=session,
            )

            session.delete(api_key)
            session.commit()

    # -------------------- Artifacts --------------------

    def create_artifact(self, artifact: ArtifactRequest) -> ArtifactResponse:
        """Creates a new artifact.

        Args:
            artifact: The artifact to create.

        Returns:
            The newly created artifact.

        Raises:
            EntityExistsError: If an artifact with the same name already exists.
        """
        with Session(self.engine) as session:
            # Check if an artifact with the given name already exists
            existing_artifact = session.exec(
                select(ArtifactSchema).where(
                    ArtifactSchema.name == artifact.name
                )
            ).first()
            if existing_artifact is not None:
                raise EntityExistsError(
                    f"Unable to create artifact with name '{artifact.name}': "
                    "An artifact with the same name already exists."
                )

            # Create the artifact.
            artifact_schema = ArtifactSchema.from_request(artifact)

            # Save tags of the artifact.
            if artifact.tags:
                self._attach_tags_to_resource(
                    tag_names=artifact.tags,
                    resource_id=artifact_schema.id,
                    resource_type=TaggableResourceTypes.ARTIFACT,
                )

            session.add(artifact_schema)
            session.commit()
            return artifact_schema.to_model(include_metadata=True)

    def get_artifact(
        self, artifact_id: UUID, hydrate: bool = True
    ) -> ArtifactResponse:
        """Gets an artifact.

        Args:
            artifact_id: The ID of the artifact to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact.

        Raises:
            KeyError: if the artifact doesn't exist.
        """
        with Session(self.engine) as session:
            artifact = session.exec(
                select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
            ).first()
            if artifact is None:
                raise KeyError(
                    f"Unable to get artifact with ID {artifact_id}: No "
                    "artifact with this ID found."
                )
            return artifact.to_model(include_metadata=hydrate)

    def list_artifacts(
        self, filter_model: ArtifactFilter, hydrate: bool = False
    ) -> Page[ArtifactResponse]:
        """List all artifacts matching the given filter criteria.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all artifacts matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(ArtifactSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=ArtifactSchema,
                filter_model=filter_model,
                hydrate=hydrate,
            )

    def update_artifact(
        self, artifact_id: UUID, artifact_update: ArtifactUpdate
    ) -> ArtifactResponse:
        """Updates an artifact.

        Args:
            artifact_id: The ID of the artifact to update.
            artifact_update: The update to be applied to the artifact.

        Returns:
            The updated artifact.

        Raises:
            KeyError: if the artifact doesn't exist.
        """
        with Session(self.engine) as session:
            existing_artifact = session.exec(
                select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
            ).first()
            if not existing_artifact:
                raise KeyError(f"Artifact with ID {artifact_id} not found.")

            # Handle tag updates.
            if artifact_update.add_tags:
                self._attach_tags_to_resource(
                    tag_names=artifact_update.add_tags,
                    resource_id=existing_artifact.id,
                    resource_type=TaggableResourceTypes.ARTIFACT,
                )
            if artifact_update.remove_tags:
                self._detach_tags_from_resource(
                    tag_names=artifact_update.remove_tags,
                    resource_id=existing_artifact.id,
                    resource_type=TaggableResourceTypes.ARTIFACT,
                )

            # Update the schema itself.
            existing_artifact.update(artifact_update=artifact_update)
            session.add(existing_artifact)
            session.commit()
            session.refresh(existing_artifact)
            return existing_artifact.to_model(include_metadata=True)

    def delete_artifact(self, artifact_id: UUID) -> None:
        """Deletes an artifact.

        Args:
            artifact_id: The ID of the artifact to delete.

        Raises:
            KeyError: if the artifact doesn't exist.
        """
        with Session(self.engine) as session:
            existing_artifact = session.exec(
                select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
            ).first()
            if not existing_artifact:
                raise KeyError(f"Artifact with ID {artifact_id} not found.")
            session.delete(existing_artifact)
            session.commit()

    # -------------------- Artifact Versions --------------------

    def create_artifact_version(
        self, artifact_version: ArtifactVersionRequest
    ) -> ArtifactVersionResponse:
        """Creates an artifact version.

        Args:
            artifact_version: The artifact version to create.

        Returns:
            The created artifact version.

        Raises:
            EntityExistsError: if an artifact with the same name and version
                already exists.
        """
        with Session(self.engine) as session:
            # Check if an artifact with the given name and version exists
            def _check(tolerance: int = 0) -> None:
                query = session.exec(
                    select(ArtifactVersionSchema)
                    .where(
                        ArtifactVersionSchema.artifact_id
                        == artifact_version.artifact_id
                    )
                    .where(
                        ArtifactVersionSchema.version
                        == artifact_version.version
                    )
                )
                existing_artifact = query.fetchmany(tolerance + 1)
                if (
                    existing_artifact is not None
                    and len(existing_artifact) > tolerance
                ):
                    raise EntityExistsError(
                        f"Unable to create artifact with name "
                        f"'{existing_artifact[0].artifact.name}' and version "
                        f"'{artifact_version.version}': An artifact with the same "
                        "name and version already exists."
                    )

            _check()
            # Create the artifact version.
            artifact_version_schema = ArtifactVersionSchema.from_request(
                artifact_version
            )
            session.add(artifact_version_schema)

            # Save visualizations of the artifact.
            if artifact_version.visualizations:
                for vis in artifact_version.visualizations:
                    vis_schema = ArtifactVisualizationSchema.from_model(
                        artifact_visualization_request=vis,
                        artifact_version_id=artifact_version_schema.id,
                    )
                    session.add(vis_schema)

            # Save tags of the artifact.
            if artifact_version.tags:
                self._attach_tags_to_resource(
                    tag_names=artifact_version.tags,
                    resource_id=artifact_version_schema.id,
                    resource_type=TaggableResourceTypes.ARTIFACT_VERSION,
                )

            try:
                _check(1)
                session.commit()
            except EntityExistsError as e:
                session.rollback()
                raise e

            return artifact_version_schema.to_model(include_metadata=True)

    def get_artifact_version(
        self, artifact_version_id: UUID, hydrate: bool = True
    ) -> ArtifactVersionResponse:
        """Gets an artifact version.

        Args:
            artifact_version_id: The ID of the artifact version to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact version.

        Raises:
            KeyError: if the artifact version doesn't exist.
        """
        with Session(self.engine) as session:
            artifact_version = session.exec(
                select(ArtifactVersionSchema).where(
                    ArtifactVersionSchema.id == artifact_version_id
                )
            ).first()
            if artifact_version is None:
                raise KeyError(
                    f"Unable to get artifact version with ID "
                    f"{artifact_version_id}: No artifact version with this ID "
                    f"found."
                )
            return artifact_version.to_model(include_metadata=hydrate)

    def list_artifact_versions(
        self,
        artifact_version_filter_model: ArtifactVersionFilter,
        hydrate: bool = False,
    ) -> Page[ArtifactVersionResponse]:
        """List all artifact versions matching the given filter criteria.

        Args:
            artifact_version_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all artifact versions matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(ArtifactVersionSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=ArtifactVersionSchema,
                filter_model=artifact_version_filter_model,
                hydrate=hydrate,
            )

    def update_artifact_version(
        self,
        artifact_version_id: UUID,
        artifact_version_update: ArtifactVersionUpdate,
    ) -> ArtifactVersionResponse:
        """Updates an artifact version.

        Args:
            artifact_version_id: The ID of the artifact version to update.
            artifact_version_update: The update to be applied to the artifact
                version.

        Returns:
            The updated artifact version.

        Raises:
            KeyError: if the artifact version doesn't exist.
        """
        with Session(self.engine) as session:
            existing_artifact_version = session.exec(
                select(ArtifactVersionSchema).where(
                    ArtifactVersionSchema.id == artifact_version_id
                )
            ).first()
            if not existing_artifact_version:
                raise KeyError(
                    f"Artifact version with ID {artifact_version_id} not found."
                )

            # Handle tag updates.
            if artifact_version_update.add_tags:
                self._attach_tags_to_resource(
                    tag_names=artifact_version_update.add_tags,
                    resource_id=existing_artifact_version.id,
                    resource_type=TaggableResourceTypes.ARTIFACT_VERSION,
                )
            if artifact_version_update.remove_tags:
                self._detach_tags_from_resource(
                    tag_names=artifact_version_update.remove_tags,
                    resource_id=existing_artifact_version.id,
                    resource_type=TaggableResourceTypes.ARTIFACT_VERSION,
                )

            # Update the schema itself.
            existing_artifact_version.update(
                artifact_version_update=artifact_version_update
            )
            session.add(existing_artifact_version)
            session.commit()
            session.refresh(existing_artifact_version)
            return existing_artifact_version.to_model(include_metadata=True)

    def delete_artifact_version(self, artifact_version_id: UUID) -> None:
        """Deletes an artifact version.

        Args:
            artifact_version_id: The ID of the artifact version to delete.

        Raises:
            KeyError: if the artifact version doesn't exist.
        """
        with Session(self.engine) as session:
            artifact_version = session.exec(
                select(ArtifactVersionSchema).where(
                    ArtifactVersionSchema.id == artifact_version_id
                )
            ).first()
            if artifact_version is None:
                raise KeyError(
                    f"Unable to delete artifact version with ID "
                    f"{artifact_version_id}: No artifact version with this ID "
                    "found."
                )
            session.delete(artifact_version)
            session.commit()

    def prune_artifact_versions(
        self,
        only_versions: bool = True,
    ) -> None:
        """Prunes unused artifact versions and their artifacts.

        Args:
            only_versions: Only delete artifact versions, keeping artifacts
        """
        with Session(self.engine) as session:
            unused_artifact_versions = [
                a[0]
                for a in session.execute(
                    select(ArtifactVersionSchema.id).where(
                        and_(
                            col(ArtifactVersionSchema.id).notin_(
                                select(StepRunOutputArtifactSchema.artifact_id)
                            ),
                            col(ArtifactVersionSchema.id).notin_(
                                select(StepRunInputArtifactSchema.artifact_id)
                            ),
                        )
                    )
                ).fetchall()
            ]
            session.execute(
                delete(ArtifactVersionSchema).where(
                    col(ArtifactVersionSchema.id).in_(
                        unused_artifact_versions
                    ),
                )
            )
            if not only_versions:
                unused_artifacts = [
                    a[0]
                    for a in session.execute(
                        select(ArtifactSchema.id).where(
                            col(ArtifactSchema.id).notin_(
                                select(ArtifactVersionSchema.artifact_id)
                            )
                        )
                    ).fetchall()
                ]
                session.execute(
                    delete(ArtifactSchema).where(
                        col(ArtifactSchema.id).in_(unused_artifacts)
                    )
                )
            session.commit()

    # ------------------------ Artifact Visualizations ------------------------

    def get_artifact_visualization(
        self, artifact_visualization_id: UUID, hydrate: bool = True
    ) -> ArtifactVisualizationResponse:
        """Gets an artifact visualization.

        Args:
            artifact_visualization_id: The ID of the artifact visualization to
                get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact visualization.

        Raises:
            KeyError: if the code reference doesn't exist.
        """
        with Session(self.engine) as session:
            artifact_visualization = session.exec(
                select(ArtifactVisualizationSchema).where(
                    ArtifactVisualizationSchema.id == artifact_visualization_id
                )
            ).first()
            if artifact_visualization is None:
                raise KeyError(
                    f"Unable to get artifact visualization with ID "
                    f"{artifact_visualization_id}: "
                    f"No artifact visualization with this ID found."
                )
            return artifact_visualization.to_model(include_metadata=hydrate)

    # ------------------------ Code References ------------------------

    def get_code_reference(
        self, code_reference_id: UUID, hydrate: bool = True
    ) -> CodeReferenceResponse:
        """Gets a code reference.

        Args:
            code_reference_id: The ID of the code reference to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The code reference.

        Raises:
            KeyError: if the code reference doesn't exist.
        """
        with Session(self.engine) as session:
            code_reference = session.exec(
                select(CodeReferenceSchema).where(
                    CodeRepositorySchema.id == code_reference_id
                )
            ).first()
            if code_reference is None:
                raise KeyError(
                    f"Unable to get code reference with ID "
                    f"{code_reference_id}: "
                    f"No code reference with this ID found."
                )
            return code_reference.to_model(include_metadata=hydrate)

    # --------------------------- Code Repositories ---------------------------

    @track_decorator(AnalyticsEvent.REGISTERED_CODE_REPOSITORY)
    def create_code_repository(
        self, code_repository: CodeRepositoryRequest
    ) -> CodeRepositoryResponse:
        """Creates a new code repository.

        Args:
            code_repository: Code repository to be created.

        Returns:
            The newly created code repository.

        Raises:
            EntityExistsError: If a code repository with the given name already
                exists.
        """
        with Session(self.engine) as session:
            existing_repo = session.exec(
                select(CodeRepositorySchema)
                .where(CodeRepositorySchema.name == code_repository.name)
                .where(
                    CodeRepositorySchema.workspace_id
                    == code_repository.workspace
                )
            ).first()
            if existing_repo is not None:
                raise EntityExistsError(
                    f"Unable to create code repository in workspace "
                    f"'{code_repository.workspace}': A code repository with "
                    "this name already exists."
                )

            new_repo = CodeRepositorySchema.from_request(code_repository)
            session.add(new_repo)
            session.commit()
            session.refresh(new_repo)

            return new_repo.to_model(include_metadata=True)

    def get_code_repository(
        self, code_repository_id: UUID, hydrate: bool = True
    ) -> CodeRepositoryResponse:
        """Gets a specific code repository.

        Args:
            code_repository_id: The ID of the code repository to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested code repository, if it was found.

        Raises:
            KeyError: If no code repository with the given ID exists.
        """
        with Session(self.engine) as session:
            repo = session.exec(
                select(CodeRepositorySchema).where(
                    CodeRepositorySchema.id == code_repository_id
                )
            ).first()
            if repo is None:
                raise KeyError(
                    f"Unable to get code repository with ID "
                    f"'{code_repository_id}': No code repository with this "
                    "ID found."
                )

            return repo.to_model(include_metadata=hydrate)

    def list_code_repositories(
        self,
        filter_model: CodeRepositoryFilter,
        hydrate: bool = False,
    ) -> Page[CodeRepositoryResponse]:
        """List all code repositories.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all code repositories.
        """
        with Session(self.engine) as session:
            query = select(CodeRepositorySchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=CodeRepositorySchema,
                filter_model=filter_model,
                hydrate=hydrate,
            )

    def update_code_repository(
        self, code_repository_id: UUID, update: CodeRepositoryUpdate
    ) -> CodeRepositoryResponse:
        """Updates an existing code repository.

        Args:
            code_repository_id: The ID of the code repository to update.
            update: The update to be applied to the code repository.

        Returns:
            The updated code repository.

        Raises:
            KeyError: If no code repository with the given name exists.
        """
        with Session(self.engine) as session:
            existing_repo = session.exec(
                select(CodeRepositorySchema).where(
                    CodeRepositorySchema.id == code_repository_id
                )
            ).first()
            if existing_repo is None:
                raise KeyError(
                    f"Unable to update code repository with ID "
                    f"{code_repository_id}: No code repository with this ID "
                    "found."
                )

            existing_repo.update(update)

            session.add(existing_repo)
            session.commit()

            return existing_repo.to_model(include_metadata=True)

    def delete_code_repository(self, code_repository_id: UUID) -> None:
        """Deletes a code repository.

        Args:
            code_repository_id: The ID of the code repository to delete.

        Raises:
            KeyError: If no code repository with the given ID exists.
        """
        with Session(self.engine) as session:
            existing_repo = session.exec(
                select(CodeRepositorySchema).where(
                    CodeRepositorySchema.id == code_repository_id
                )
            ).first()
            if existing_repo is None:
                raise KeyError(
                    f"Unable to delete code repository with ID "
                    f"{code_repository_id}: No code repository with this ID "
                    "found."
                )

            session.delete(existing_repo)
            session.commit()

    # ----------------------------- Components -----------------------------

    @track_decorator(AnalyticsEvent.REGISTERED_STACK_COMPONENT)
    def create_stack_component(
        self,
        component: ComponentRequest,
    ) -> ComponentResponse:
        """Create a stack component.

        Args:
            component: The stack component to create.

        Returns:
            The created stack component.

        Raises:
            KeyError: if the stack component references a non-existent
                connector.
        """
        with Session(self.engine) as session:
            self._fail_if_component_with_name_type_exists(
                name=component.name,
                component_type=component.type,
                workspace_id=component.workspace,
                session=session,
            )

            service_connector: Optional[ServiceConnectorSchema] = None
            if component.connector:
                service_connector = session.exec(
                    select(ServiceConnectorSchema).where(
                        ServiceConnectorSchema.id == component.connector
                    )
                ).first()

                if service_connector is None:
                    raise KeyError(
                        f"Service connector with ID {component.connector} not "
                        "found."
                    )

            # Create the component
            new_component = StackComponentSchema(
                name=component.name,
                workspace_id=component.workspace,
                user_id=component.user,
                component_spec_path=component.component_spec_path,
                type=component.type,
                flavor=component.flavor,
                configuration=base64.b64encode(
                    json.dumps(component.configuration).encode("utf-8")
                ),
                labels=base64.b64encode(
                    json.dumps(component.labels).encode("utf-8")
                ),
                connector=service_connector,
                connector_resource_id=component.connector_resource_id,
            )

            session.add(new_component)
            session.commit()

            session.refresh(new_component)

            return new_component.to_model(include_metadata=True)

    def get_stack_component(
        self, component_id: UUID, hydrate: bool = True
    ) -> ComponentResponse:
        """Get a stack component by ID.

        Args:
            component_id: The ID of the stack component to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack component.

        Raises:
            KeyError: if the stack component doesn't exist.
        """
        with Session(self.engine) as session:
            stack_component = session.exec(
                select(StackComponentSchema).where(
                    StackComponentSchema.id == component_id
                )
            ).first()

            if stack_component is None:
                raise KeyError(
                    f"Stack component with ID {component_id} not found."
                )

            return stack_component.to_model(include_metadata=hydrate)

    def list_stack_components(
        self,
        component_filter_model: ComponentFilter,
        hydrate: bool = False,
    ) -> Page[ComponentResponse]:
        """List all stack components matching the given filter criteria.

        Args:
            component_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all stack components matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(StackComponentSchema)
            paged_components: Page[ComponentResponse] = (
                self.filter_and_paginate(
                    session=session,
                    query=query,
                    table=StackComponentSchema,
                    filter_model=component_filter_model,
                    hydrate=hydrate,
                )
            )
            return paged_components

    def update_stack_component(
        self, component_id: UUID, component_update: ComponentUpdate
    ) -> ComponentResponse:
        """Update an existing stack component.

        Args:
            component_id: The ID of the stack component to update.
            component_update: The update to be applied to the stack component.

        Returns:
            The updated stack component.

        Raises:
            KeyError: if the stack component doesn't exist.
            IllegalOperationError: if the stack component is a default stack
                component.
        """
        with Session(self.engine) as session:
            existing_component = session.exec(
                select(StackComponentSchema).where(
                    StackComponentSchema.id == component_id
                )
            ).first()

            if existing_component is None:
                raise KeyError(
                    f"Unable to update component with id "
                    f"'{component_id}': Found no"
                    f"existing component with this id."
                )

            if (
                existing_component.name == DEFAULT_STACK_AND_COMPONENT_NAME
                and existing_component.type
                in [
                    StackComponentType.ORCHESTRATOR,
                    StackComponentType.ARTIFACT_STORE,
                ]
            ):
                raise IllegalOperationError(
                    f"The default {existing_component.type} cannot be modified."
                )

            # In case of a renaming update, make sure no component of the same
            # type already exists with that name
            if component_update.name:
                if existing_component.name != component_update.name:
                    self._fail_if_component_with_name_type_exists(
                        name=component_update.name,
                        component_type=existing_component.type,
                        workspace_id=existing_component.workspace_id,
                        session=session,
                    )

            existing_component.update(component_update=component_update)

            if component_update.connector:
                service_connector = session.exec(
                    select(ServiceConnectorSchema).where(
                        ServiceConnectorSchema.id == component_update.connector
                    )
                ).first()

                if service_connector is None:
                    raise KeyError(
                        "Service connector with ID "
                        f"{component_update.connector} not found."
                    )
                existing_component.connector = service_connector
                existing_component.connector_resource_id = (
                    component_update.connector_resource_id
                )
            else:
                existing_component.connector = None
                existing_component.connector_resource_id = None

            session.add(existing_component)
            session.commit()

            return existing_component.to_model(include_metadata=True)

    def delete_stack_component(self, component_id: UUID) -> None:
        """Delete a stack component.

        Args:
            component_id: The id of the stack component to delete.

        Raises:
            KeyError: if the stack component doesn't exist.
            IllegalOperationError: if the stack component is part of one or
                more stacks, or if it's a default stack component.
        """
        with Session(self.engine) as session:
            try:
                stack_component = session.exec(
                    select(StackComponentSchema).where(
                        StackComponentSchema.id == component_id
                    )
                ).one()

                if stack_component is None:
                    raise KeyError(f"Stack with ID {component_id} not found.")
                if (
                    stack_component.name == DEFAULT_STACK_AND_COMPONENT_NAME
                    and stack_component.type
                    in [
                        StackComponentType.ORCHESTRATOR,
                        StackComponentType.ARTIFACT_STORE,
                    ]
                ):
                    raise IllegalOperationError(
                        f"The default {stack_component.type} cannot be deleted."
                    )

                if len(stack_component.stacks) > 0:
                    raise IllegalOperationError(
                        f"Stack Component `{stack_component.name}` of type "
                        f"`{stack_component.type} cannot be "
                        f"deleted as it is part of "
                        f"{len(stack_component.stacks)} stacks. "
                        f"Before deleting this stack "
                        f"component, make sure to remove it "
                        f"from all stacks."
                    )
                else:
                    session.delete(stack_component)
            except NoResultFound as error:
                raise KeyError from error

            session.commit()

    def count_stack_components(
        self, filter_model: Optional[ComponentFilter] = None
    ) -> int:
        """Count all components.

        Args:
            filter_model: The filter model to use for counting components.

        Returns:
            The number of components.
        """
        return self._count_entity(
            schema=StackComponentSchema, filter_model=filter_model
        )

    @staticmethod
    def _fail_if_component_with_name_type_exists(
        name: str,
        component_type: StackComponentType,
        workspace_id: UUID,
        session: Session,
    ) -> None:
        """Raise an exception if a component with same name/type exists.

        Args:
            name: The name of the component
            component_type: The type of the component
            workspace_id: The ID of the workspace
            session: The Session

        Raises:
            StackComponentExistsError: If a component with the given name and
                type already exists.
        """
        # Check if component with the same domain key (name, type, workspace)
        # already exists
        existing_domain_component = session.exec(
            select(StackComponentSchema)
            .where(StackComponentSchema.name == name)
            .where(StackComponentSchema.workspace_id == workspace_id)
            .where(StackComponentSchema.type == component_type)
        ).first()
        if existing_domain_component is not None:
            raise StackComponentExistsError(
                f"Unable to register '{component_type}' component "
                f"with name '{name}': Found an existing "
                f"component with the same name and type in the same "
                f" workspace '{existing_domain_component.workspace.name}'."
            )

    # -------------------------- Devices -------------------------

    def create_authorized_device(
        self, device: OAuthDeviceInternalRequest
    ) -> OAuthDeviceInternalResponse:
        """Creates a new OAuth 2.0 authorized device.

        Args:
            device: The device to be created.

        Returns:
            The newly created device.

        Raises:
            EntityExistsError: If a device for the same client ID already
                exists.
        """
        with Session(self.engine) as session:
            existing_device = session.exec(
                select(OAuthDeviceSchema).where(
                    OAuthDeviceSchema.client_id == device.client_id
                )
            ).first()
            if existing_device is not None:
                raise EntityExistsError(
                    f"Unable to create device with client ID "
                    f"'{device.client_id}': A device with this client ID "
                    "already exists."
                )

            (
                new_device,
                user_code,
                device_code,
            ) = OAuthDeviceSchema.from_request(device)
            session.add(new_device)
            session.commit()
            session.refresh(new_device)

            device_model = new_device.to_internal_model(hydrate=True)
            # Replace the hashed user code with the original user code
            device_model.user_code = user_code
            # Replace the hashed device code with the original device code
            device_model.device_code = device_code

            return device_model

    def get_authorized_device(
        self, device_id: UUID, hydrate: bool = True
    ) -> OAuthDeviceResponse:
        """Gets a specific OAuth 2.0 authorized device.

        Args:
            device_id: The ID of the device to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested device, if it was found.

        Raises:
            KeyError: If no device with the given ID exists.
        """
        with Session(self.engine) as session:
            device = session.exec(
                select(OAuthDeviceSchema).where(
                    OAuthDeviceSchema.id == device_id
                )
            ).first()
            if device is None:
                raise KeyError(
                    f"Unable to get device with ID {device_id}: No device with "
                    "this ID found."
                )

            return device.to_model(include_metadata=hydrate)

    def get_internal_authorized_device(
        self,
        device_id: Optional[UUID] = None,
        client_id: Optional[UUID] = None,
        hydrate: bool = True,
    ) -> OAuthDeviceInternalResponse:
        """Gets a specific OAuth 2.0 authorized device for internal use.

        Args:
            client_id: The client ID of the device to get.
            device_id: The ID of the device to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested device, if it was found.

        Raises:
            KeyError: If no device with the given client ID exists.
            ValueError: If neither device ID nor client ID are provided.
        """
        with Session(self.engine) as session:
            if device_id is not None:
                device = session.exec(
                    select(OAuthDeviceSchema).where(
                        OAuthDeviceSchema.id == device_id
                    )
                ).first()
            elif client_id is not None:
                device = session.exec(
                    select(OAuthDeviceSchema).where(
                        OAuthDeviceSchema.client_id == client_id
                    )
                ).first()
            else:
                raise ValueError(
                    "Either device ID or client ID must be provided."
                )
            if device is None:
                raise KeyError(
                    f"Unable to get device with client ID {client_id}: No "
                    "device with this client ID found."
                )

            return device.to_internal_model(hydrate=hydrate)

    def list_authorized_devices(
        self,
        filter_model: OAuthDeviceFilter,
        hydrate: bool = False,
    ) -> Page[OAuthDeviceResponse]:
        """List all OAuth 2.0 authorized devices for a user.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all matching OAuth 2.0 authorized devices.
        """
        with Session(self.engine) as session:
            query = select(OAuthDeviceSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=OAuthDeviceSchema,
                filter_model=filter_model,
                hydrate=hydrate,
            )

    def update_authorized_device(
        self, device_id: UUID, update: OAuthDeviceUpdate
    ) -> OAuthDeviceResponse:
        """Updates an existing OAuth 2.0 authorized device for internal use.

        Args:
            device_id: The ID of the device to update.
            update: The update to be applied to the device.

        Returns:
            The updated OAuth 2.0 authorized device.

        Raises:
            KeyError: If no device with the given ID exists.
        """
        with Session(self.engine) as session:
            existing_device = session.exec(
                select(OAuthDeviceSchema).where(
                    OAuthDeviceSchema.id == device_id
                )
            ).first()
            if existing_device is None:
                raise KeyError(
                    f"Unable to update device with ID {device_id}: No "
                    "device with this ID found."
                )

            existing_device.update(update)

            session.add(existing_device)
            session.commit()

            return existing_device.to_model(include_metadata=True)

    def update_internal_authorized_device(
        self, device_id: UUID, update: OAuthDeviceInternalUpdate
    ) -> OAuthDeviceInternalResponse:
        """Updates an existing OAuth 2.0 authorized device.

        Args:
            device_id: The ID of the device to update.
            update: The update to be applied to the device.

        Returns:
            The updated OAuth 2.0 authorized device.

        Raises:
            KeyError: If no device with the given ID exists.
        """
        with Session(self.engine) as session:
            existing_device = session.exec(
                select(OAuthDeviceSchema).where(
                    OAuthDeviceSchema.id == device_id
                )
            ).first()
            if existing_device is None:
                raise KeyError(
                    f"Unable to update device with ID {device_id}: No device "
                    "with this ID found."
                )

            (
                _,
                user_code,
                device_code,
            ) = existing_device.internal_update(update)

            session.add(existing_device)
            session.commit()

            device_model = existing_device.to_internal_model(hydrate=True)
            if user_code:
                # Replace the hashed user code with the original user code
                device_model.user_code = user_code

            if device_code:
                # Replace the hashed device code with the original device code
                device_model.device_code = device_code

            return device_model

    def delete_authorized_device(self, device_id: UUID) -> None:
        """Deletes an OAuth 2.0 authorized device.

        Args:
            device_id: The ID of the device to delete.

        Raises:
            KeyError: If no device with the given ID exists.
        """
        with Session(self.engine) as session:
            existing_device = session.exec(
                select(OAuthDeviceSchema).where(
                    OAuthDeviceSchema.id == device_id
                )
            ).first()
            if existing_device is None:
                raise KeyError(
                    f"Unable to delete device with ID {device_id}: No device "
                    "with this ID found."
                )

            session.delete(existing_device)
            session.commit()

    def delete_expired_authorized_devices(self) -> None:
        """Deletes all expired OAuth 2.0 authorized devices."""
        with Session(self.engine) as session:
            expired_devices = session.exec(
                select(OAuthDeviceSchema).where(OAuthDeviceSchema.user is None)
            ).all()
            for device in expired_devices:
                # Delete devices that have expired
                if (
                    device.expires is not None
                    and device.expires < datetime.now()
                    and device.user_id is None
                ):
                    session.delete(device)
            session.commit()

    # ----------------------------- Flavors -----------------------------

    @track_decorator(AnalyticsEvent.CREATED_FLAVOR)
    def create_flavor(self, flavor: FlavorRequest) -> FlavorResponse:
        """Creates a new stack component flavor.

        Args:
            flavor: The stack component flavor to create.

        Returns:
            The newly created flavor.

        Raises:
            EntityExistsError: If a flavor with the same name and type
                is already owned by this user in this workspace.
            ValueError: In case the config_schema string exceeds the max length.
        """
        with Session(self.engine) as session:
            # Check if flavor with the same domain key (name, type, workspace,
            # owner) already exists
            existing_flavor = session.exec(
                select(FlavorSchema)
                .where(FlavorSchema.name == flavor.name)
                .where(FlavorSchema.type == flavor.type)
                .where(FlavorSchema.workspace_id == flavor.workspace)
                .where(FlavorSchema.user_id == flavor.user)
            ).first()

            if existing_flavor is not None:
                raise EntityExistsError(
                    f"Unable to register '{flavor.type.value}' flavor "
                    f"with name '{flavor.name}': Found an existing "
                    f"flavor with the same name and type in the same "
                    f"'{flavor.workspace}' workspace owned by the same "
                    f"'{flavor.user}' user."
                )

            config_schema = json.dumps(flavor.config_schema)

            if len(config_schema) > TEXT_FIELD_MAX_LENGTH:
                raise ValueError(
                    "Json representation of configuration schema"
                    "exceeds max length."
                )

            else:
                new_flavor = FlavorSchema(
                    name=flavor.name,
                    type=flavor.type,
                    source=flavor.source,
                    config_schema=config_schema,
                    integration=flavor.integration,
                    connector_type=flavor.connector_type,
                    connector_resource_type=flavor.connector_resource_type,
                    connector_resource_id_attr=flavor.connector_resource_id_attr,
                    workspace_id=flavor.workspace,
                    user_id=flavor.user,
                    logo_url=flavor.logo_url,
                    docs_url=flavor.docs_url,
                    sdk_docs_url=flavor.sdk_docs_url,
                    is_custom=flavor.is_custom,
                )
                session.add(new_flavor)
                session.commit()

                return new_flavor.to_model(include_metadata=True)

    def get_flavor(
        self, flavor_id: UUID, hydrate: bool = True
    ) -> FlavorResponse:
        """Get a flavor by ID.

        Args:
            flavor_id: The ID of the flavor to fetch.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack component flavor.

        Raises:
            KeyError: if the stack component flavor doesn't exist.
        """
        with Session(self.engine) as session:
            flavor_in_db = session.exec(
                select(FlavorSchema).where(FlavorSchema.id == flavor_id)
            ).first()
            if flavor_in_db is None:
                raise KeyError(f"Flavor with ID {flavor_id} not found.")
            return flavor_in_db.to_model(include_metadata=hydrate)

    def list_flavors(
        self,
        flavor_filter_model: FlavorFilter,
        hydrate: bool = False,
    ) -> Page[FlavorResponse]:
        """List all stack component flavors matching the given filter criteria.

        Args:
            flavor_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            List of all the stack component flavors matching the given criteria.
        """
        with Session(self.engine) as session:
            query = select(FlavorSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=FlavorSchema,
                filter_model=flavor_filter_model,
                hydrate=hydrate,
            )

    def update_flavor(
        self, flavor_id: UUID, flavor_update: FlavorUpdate
    ) -> FlavorResponse:
        """Updates an existing user.

        Args:
            flavor_id: The id of the flavor to update.
            flavor_update: The update to be applied to the flavor.

        Returns:
            The updated flavor.

        Raises:
            KeyError: If no flavor with the given id exists.
        """
        with Session(self.engine) as session:
            existing_flavor = session.exec(
                select(FlavorSchema).where(FlavorSchema.id == flavor_id)
            ).first()

            if not existing_flavor:
                raise KeyError(f"Flavor with ID {flavor_id} not found.")

            existing_flavor.update(flavor_update=flavor_update)
            session.add(existing_flavor)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(existing_flavor)
            return existing_flavor.to_model(include_metadata=True)

    def delete_flavor(self, flavor_id: UUID) -> None:
        """Delete a flavor.

        Args:
            flavor_id: The id of the flavor to delete.

        Raises:
            KeyError: if the flavor doesn't exist.
            IllegalOperationError: if the flavor is used by a stack component.
        """
        with Session(self.engine) as session:
            try:
                flavor_in_db = session.exec(
                    select(FlavorSchema).where(FlavorSchema.id == flavor_id)
                ).one()

                if flavor_in_db is None:
                    raise KeyError(f"Flavor with ID {flavor_id} not found.")
                components_of_flavor = session.exec(
                    select(StackComponentSchema).where(
                        StackComponentSchema.flavor == flavor_in_db.name
                    )
                ).all()
                if len(components_of_flavor) > 0:
                    raise IllegalOperationError(
                        f"Stack Component `{flavor_in_db.name}` of type "
                        f"`{flavor_in_db.type} cannot be "
                        f"deleted as it is used by "
                        f"{len(components_of_flavor)} "
                        f"components. Before deleting this "
                        f"flavor, make sure to delete all "
                        f"associated components."
                    )
                else:
                    session.delete(flavor_in_db)
                    session.commit()
            except NoResultFound as error:
                raise KeyError from error

    # ------------------------ Logs ------------------------

    def get_logs(self, logs_id: UUID, hydrate: bool = True) -> LogsResponse:
        """Gets logs with the given ID.

        Args:
            logs_id: The ID of the logs to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The logs.

        Raises:
            KeyError: if the logs doesn't exist.
        """
        with Session(self.engine) as session:
            logs = session.exec(
                select(LogsSchema).where(LogsSchema.id == logs_id)
            ).first()
            if logs is None:
                raise KeyError(
                    f"Unable to get logs with ID "
                    f"{logs_id}: "
                    f"No logs with this ID found."
                )
            return logs.to_model(include_metadata=hydrate)

    # ----------------------------- Pipelines -----------------------------

    @track_decorator(AnalyticsEvent.CREATE_PIPELINE)
    def create_pipeline(
        self,
        pipeline: PipelineRequest,
    ) -> PipelineResponse:
        """Creates a new pipeline in a workspace.

        Args:
            pipeline: The pipeline to create.

        Returns:
            The newly created pipeline.

        Raises:
            EntityExistsError: If an identical pipeline already exists.
        """
        with Session(self.engine) as session:
            # Check if pipeline with the given name already exists
            existing_pipeline = session.exec(
                select(PipelineSchema)
                .where(PipelineSchema.name == pipeline.name)
                .where(PipelineSchema.version_hash == pipeline.version_hash)
                .where(PipelineSchema.workspace_id == pipeline.workspace)
            ).first()
            if existing_pipeline is not None:
                raise EntityExistsError(
                    f"Unable to create pipeline in workspace "
                    f"'{pipeline.workspace}': A pipeline with this name and "
                    f"version already exists."
                )

            # Create the pipeline
            new_pipeline = PipelineSchema.from_request(pipeline)
            session.add(new_pipeline)
            session.commit()
            session.refresh(new_pipeline)

            return new_pipeline.to_model(include_metadata=True)

    def get_pipeline(
        self, pipeline_id: UUID, hydrate: bool = True
    ) -> PipelineResponse:
        """Get a pipeline with a given ID.

        Args:
            pipeline_id: ID of the pipeline.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The pipeline.

        Raises:
            KeyError: if the pipeline does not exist.
        """
        with Session(self.engine) as session:
            # Check if pipeline with the given ID exists
            pipeline = session.exec(
                select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
            ).first()
            if pipeline is None:
                raise KeyError(
                    f"Unable to get pipeline with ID '{pipeline_id}': "
                    "No pipeline with this ID found."
                )

            return pipeline.to_model(include_metadata=hydrate)

    def list_pipeline_namespaces(
        self,
        filter_model: PipelineNamespaceFilter,
        hydrate: bool = False,
    ) -> Page[PipelineNamespaceResponse]:
        """List all pipeline namespaces matching the given filter criteria.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all pipeline namespaces matching the filter criteria.
        """

        def _custom_conversion(
            row: Tuple[str, UUID, str],
        ) -> PipelineNamespaceResponse:
            name, latest_run_id, latest_run_status = row

            body = PipelineNamespaceResponseBody(
                latest_run_id=latest_run_id,
                latest_run_status=latest_run_status,
            )

            return PipelineNamespaceResponse(name=name, body=body)

        def _custom_fetch(
            session: Session,
            query: Union[Select[Any], SelectOfScalar[Any]],
            filter: BaseFilter,
        ) -> List[Any]:
            return session.exec(query).unique().all()

        with Session(self.engine) as session:
            max_date_subquery = (
                select(  # type: ignore[call-overload]
                    PipelineSchema.name,
                    func.max(PipelineRunSchema.created).label("max_created"),
                )
                .outerjoin(
                    PipelineRunSchema,
                    PipelineSchema.id == PipelineRunSchema.pipeline_id,
                )
                .group_by(PipelineSchema.name)
                .subquery()
            )

            query = (
                select(
                    max_date_subquery.c.name,
                    PipelineRunSchema.id,
                    PipelineRunSchema.status,
                )
                .outerjoin(
                    PipelineSchema,
                    PipelineSchema.name == max_date_subquery.c.name,
                )
                .outerjoin(
                    PipelineRunSchema,
                    PipelineRunSchema.created
                    == max_date_subquery.c.max_created,
                )
            )

            return self.filter_and_paginate(
                session=session,
                query=query,
                table=PipelineSchema,
                filter_model=filter_model,
                hydrate=hydrate,
                custom_fetch=_custom_fetch,
                custom_schema_to_model_conversion=_custom_conversion,
            )

    def list_pipelines(
        self,
        pipeline_filter_model: PipelineFilter,
        hydrate: bool = False,
    ) -> Page[PipelineResponse]:
        """List all pipelines matching the given filter criteria.

        Args:
            pipeline_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all pipelines matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(PipelineSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=PipelineSchema,
                filter_model=pipeline_filter_model,
                hydrate=hydrate,
            )

    def count_pipelines(self, filter_model: Optional[PipelineFilter]) -> int:
        """Count all pipelines.

        Args:
            filter_model: The filter model to use for counting pipelines.

        Returns:
            The number of pipelines.
        """
        return self._count_entity(
            schema=PipelineSchema, filter_model=filter_model
        )

    def update_pipeline(
        self,
        pipeline_id: UUID,
        pipeline_update: PipelineUpdate,
    ) -> PipelineResponse:
        """Updates a pipeline.

        Args:
            pipeline_id: The ID of the pipeline to be updated.
            pipeline_update: The update to be applied.

        Returns:
            The updated pipeline.

        Raises:
            KeyError: if the pipeline doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if pipeline with the given ID exists
            existing_pipeline = session.exec(
                select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
            ).first()
            if existing_pipeline is None:
                raise KeyError(
                    f"Unable to update pipeline with ID {pipeline_id}: "
                    f"No pipeline with this ID found."
                )

            # Update the pipeline
            existing_pipeline.update(pipeline_update)

            session.add(existing_pipeline)
            session.commit()

            return existing_pipeline.to_model(include_metadata=True)

    def delete_pipeline(self, pipeline_id: UUID) -> None:
        """Deletes a pipeline.

        Args:
            pipeline_id: The ID of the pipeline to delete.

        Raises:
            KeyError: if the pipeline doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if pipeline with the given ID exists
            pipeline = session.exec(
                select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
            ).first()
            if pipeline is None:
                raise KeyError(
                    f"Unable to delete pipeline with ID {pipeline_id}: "
                    f"No pipeline with this ID found."
                )

            session.delete(pipeline)
            session.commit()

    # --------------------------- Pipeline Builds ---------------------------

    def create_build(
        self,
        build: PipelineBuildRequest,
    ) -> PipelineBuildResponse:
        """Creates a new build in a workspace.

        Args:
            build: The build to create.

        Returns:
            The newly created build.
        """
        with Session(self.engine) as session:
            # Create the build
            new_build = PipelineBuildSchema.from_request(build)
            session.add(new_build)
            session.commit()
            session.refresh(new_build)

            return new_build.to_model(include_metadata=True)

    def get_build(
        self, build_id: UUID, hydrate: bool = True
    ) -> PipelineBuildResponse:
        """Get a build with a given ID.

        Args:
            build_id: ID of the build.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The build.

        Raises:
            KeyError: If the build does not exist.
        """
        with Session(self.engine) as session:
            # Check if build with the given ID exists
            build = session.exec(
                select(PipelineBuildSchema).where(
                    PipelineBuildSchema.id == build_id
                )
            ).first()
            if build is None:
                raise KeyError(
                    f"Unable to get build with ID '{build_id}': "
                    "No build with this ID found."
                )

            return build.to_model(include_metadata=hydrate)

    def list_builds(
        self,
        build_filter_model: PipelineBuildFilter,
        hydrate: bool = False,
    ) -> Page[PipelineBuildResponse]:
        """List all builds matching the given filter criteria.

        Args:
            build_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all builds matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(PipelineBuildSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=PipelineBuildSchema,
                filter_model=build_filter_model,
                hydrate=hydrate,
            )

    def delete_build(self, build_id: UUID) -> None:
        """Deletes a build.

        Args:
            build_id: The ID of the build to delete.

        Raises:
            KeyError: if the build doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if build with the given ID exists
            build = session.exec(
                select(PipelineBuildSchema).where(
                    PipelineBuildSchema.id == build_id
                )
            ).first()
            if build is None:
                raise KeyError(
                    f"Unable to delete build with ID {build_id}: "
                    f"No build with this ID found."
                )

            session.delete(build)
            session.commit()

    # -------------------------- Pipeline Deployments --------------------------

    def create_deployment(
        self,
        deployment: PipelineDeploymentRequest,
    ) -> PipelineDeploymentResponse:
        """Creates a new deployment in a workspace.

        Args:
            deployment: The deployment to create.

        Returns:
            The newly created deployment.
        """
        with Session(self.engine) as session:
            code_reference_id = self._create_or_reuse_code_reference(
                session=session,
                workspace_id=deployment.workspace,
                code_reference=deployment.code_reference,
            )

            new_deployment = PipelineDeploymentSchema.from_request(
                deployment, code_reference_id=code_reference_id
            )
            session.add(new_deployment)
            session.commit()
            session.refresh(new_deployment)

            return new_deployment.to_model(include_metadata=True)

    def get_deployment(
        self, deployment_id: UUID, hydrate: bool = True
    ) -> PipelineDeploymentResponse:
        """Get a deployment with a given ID.

        Args:
            deployment_id: ID of the deployment.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The deployment.

        Raises:
            KeyError: If the deployment does not exist.
        """
        with Session(self.engine) as session:
            # Check if deployment with the given ID exists
            deployment = session.exec(
                select(PipelineDeploymentSchema).where(
                    PipelineDeploymentSchema.id == deployment_id
                )
            ).first()
            if deployment is None:
                raise KeyError(
                    f"Unable to get deployment with ID '{deployment_id}': "
                    "No deployment with this ID found."
                )

            return deployment.to_model(include_metadata=hydrate)

    def list_deployments(
        self,
        deployment_filter_model: PipelineDeploymentFilter,
        hydrate: bool = False,
    ) -> Page[PipelineDeploymentResponse]:
        """List all deployments matching the given filter criteria.

        Args:
            deployment_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all deployments matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(PipelineDeploymentSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=PipelineDeploymentSchema,
                filter_model=deployment_filter_model,
                hydrate=hydrate,
            )

    def delete_deployment(self, deployment_id: UUID) -> None:
        """Deletes a deployment.

        Args:
            deployment_id: The ID of the deployment to delete.

        Raises:
            KeyError: If the deployment doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if build with the given ID exists
            deployment = session.exec(
                select(PipelineDeploymentSchema).where(
                    PipelineDeploymentSchema.id == deployment_id
                )
            ).first()
            if deployment is None:
                raise KeyError(
                    f"Unable to delete deployment with ID {deployment_id}: "
                    f"No deployment with this ID found."
                )

            session.delete(deployment)
            session.commit()

    # -------------------- Event Sources  --------------------

    def _fail_if_event_source_with_name_exists(
        self, event_source: EventSourceRequest, session: Session
    ) -> None:
        """Raise an exception if a stack with same name exists.

        Args:
            event_source: The event_source to create.
            session: The Session

        Returns:
            None

        Raises:
            EventSourceExistsError: In case the event source already exists
        """
        existing_domain_event_source = session.exec(
            select(EventSourceSchema)
            .where(EventSourceSchema.name == event_source.name)
            .where(EventSourceSchema.workspace_id == event_source.workspace)
        ).first()
        if existing_domain_event_source is not None:
            workspace = self._get_workspace_schema(
                workspace_name_or_id=event_source.workspace, session=session
            )
            raise EventSourceExistsError(
                f"Unable to register event source with name "
                f"'{event_source.name}': Found an existing event source with "
                f"the same name in the active workspace, '{workspace.name}'."
            )
        return None

    def create_event_source(
        self, event_source: EventSourceRequest
    ) -> EventSourceResponse:
        """Create an event_source.

        Args:
            event_source: The event_source to create.

        Returns:
            The created event_source.
        """
        with Session(self.engine) as session:
            self._fail_if_event_source_with_name_exists(
                event_source=event_source,
                session=session,
            )
            new_event_source = EventSourceSchema.from_request(event_source)
            session.add(new_event_source)
            session.commit()
            session.refresh(new_event_source)

            return new_event_source.to_model(
                include_metadata=True, include_resources=True
            )

    def _get_event_source(
        self,
        event_source_id: UUID,
        session: Session,
    ) -> EventSourceSchema:
        """Get an event_source by ID.

        Args:
            event_source_id: The ID of the event_source to get.
            session: The DB session.

        Returns:
            The event_source schema.
        """
        return self._get_schema_by_name_or_id(
            object_name_or_id=event_source_id,
            schema_class=EventSourceSchema,
            schema_name="event_source",
            session=session,
        )

    def get_event_source(
        self,
        event_source_id: UUID,
        hydrate: bool = True,
    ) -> EventSourceResponse:
        """Get an event_source by ID.

        Args:
            event_source_id: The ID of the event_source to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The event_source.
        """
        with Session(self.engine) as session:
            return self._get_event_source(
                event_source_id=event_source_id, session=session
            ).to_model(include_metadata=hydrate, include_resources=True)

    def list_event_sources(
        self,
        event_source_filter_model: EventSourceFilter,
        hydrate: bool = False,
    ) -> Page[EventSourceResponse]:
        """List all event_sources matching the given filter criteria.

        Args:
            event_source_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all event_sources matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(EventSourceSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=EventSourceSchema,
                filter_model=event_source_filter_model,
                hydrate=hydrate,
            )

    def update_event_source(
        self,
        event_source_id: UUID,
        event_source_update: EventSourceUpdate,
    ) -> EventSourceResponse:
        """Update an existing event_source.

        Args:
            event_source_id: The ID of the event_source to update.
            event_source_update: The update to be applied to the event_source.

        Returns:
            The updated event_source.
        """
        with Session(self.engine) as session:
            event_source = self._get_event_source(
                session=session, event_source_id=event_source_id
            )
            event_source.update(update=event_source_update)
            session.add(event_source)
            session.commit()

            # Refresh the event_source that was just created
            session.refresh(event_source)
            return event_source.to_model(
                include_metadata=True, include_resources=True
            )

    def delete_event_source(self, event_source_id: UUID) -> None:
        """Delete an event_source.

        Args:
            event_source_id: The ID of the event_source to delete.

        Raises:
            KeyError: if the event_source doesn't exist.
        """
        with Session(self.engine) as session:
            event_source = self._get_event_source(
                event_source_id=event_source_id, session=session
            )
            if event_source is None:
                raise KeyError(
                    f"Unable to delete event_source with ID `{event_source_id}`: "
                    f"No event_source with this ID found."
                )
            session.delete(event_source)
            session.commit()

            # TODO: catch and throw proper error if it can't be deleted due to
            #  not-null constraints on triggers

    # ----------------------------- Pipeline runs -----------------------------

    def create_run(
        self, pipeline_run: PipelineRunRequest
    ) -> PipelineRunResponse:
        """Creates a pipeline run.

        Args:
            pipeline_run: The pipeline run to create.

        Returns:
            The created pipeline run.

        Raises:
            EntityExistsError: If a run with the same name already exists.
        """
        with Session(self.engine) as session:
            # Check if pipeline run with same name already exists.
            existing_domain_run = session.exec(
                select(PipelineRunSchema).where(
                    PipelineRunSchema.name == pipeline_run.name
                )
            ).first()
            if existing_domain_run is not None:
                raise EntityExistsError(
                    f"Unable to create pipeline run: A pipeline run with name "
                    f"'{pipeline_run.name}' already exists."
                )

            # Create the pipeline run
            new_run = PipelineRunSchema.from_request(pipeline_run)
            session.add(new_run)
            session.commit()

            return new_run.to_model(include_metadata=True)

    def get_run(
        self, run_name_or_id: Union[str, UUID], hydrate: bool = True
    ) -> PipelineRunResponse:
        """Gets a pipeline run.

        Args:
            run_name_or_id: The name or ID of the pipeline run to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The pipeline run.
        """
        with Session(self.engine) as session:
            return self._get_run_schema(
                run_name_or_id, session=session
            ).to_model(include_metadata=hydrate)

    def _replace_placeholder_run(
        self, pipeline_run: PipelineRunRequest
    ) -> PipelineRunResponse:
        """Replace a placeholder run with the requested pipeline run.

        Args:
            pipeline_run: Pipeline run request.

        Raises:
            KeyError: If no placeholder run exists.

        Returns:
            The run model.
        """
        with Session(self.engine) as session:
            run_schema = session.exec(
                select(PipelineRunSchema)
                # The following line locks the row in the DB, so anyone else
                # calling `SELECT ... FOR UPDATE` will wait until the first
                # transaction to do so finishes. After the first transaction
                # finishes, the subsequent queries will not be able to find a
                # placeholder run anymore, as we already updated the
                # orchestrator_run_id.
                # Note: This only locks a single row if the where clause of
                # the query is indexed (we have a unique index due to the
                # unique constraint on those columns). Otherwise this will lock
                # multiple rows or even the complete table which we want to
                # avoid.
                .with_for_update()
                .where(
                    PipelineRunSchema.deployment_id == pipeline_run.deployment
                )
                .where(
                    PipelineRunSchema.orchestrator_run_id.is_(None)  # type: ignore[union-attr]
                )
            ).first()

            if not run_schema:
                raise KeyError("No placeholder run found.")

            run_schema.update_placeholder(pipeline_run)
            session.add(run_schema)
            session.commit()

            return run_schema.to_model(include_metadata=True)

    def _get_run_by_orchestrator_run_id(
        self, orchestrator_run_id: str, deployment_id: UUID
    ) -> PipelineRunResponse:
        """Get a pipeline run based on deployment and orchestrator run ID.

        Args:
            orchestrator_run_id: The orchestrator run ID.
            deployment_id: The deployment ID.

        Raises:
            KeyError: If no run exists for the deployment and orchestrator run
                ID.

        Returns:
            The pipeline run.
        """
        with Session(self.engine) as session:
            run_schema = session.exec(
                select(PipelineRunSchema)
                .where(PipelineRunSchema.deployment_id == deployment_id)
                .where(
                    PipelineRunSchema.orchestrator_run_id
                    == orchestrator_run_id
                )
            ).first()

            if not run_schema:
                raise KeyError(
                    f"Unable to get run for orchestrator run ID "
                    f"{orchestrator_run_id} and deployment ID {deployment_id}."
                )

            return run_schema.to_model(include_metadata=True)

    def get_or_create_run(
        self, pipeline_run: PipelineRunRequest
    ) -> Tuple[PipelineRunResponse, bool]:
        """Gets or creates a pipeline run.

        If a run with the same ID or name already exists, it is returned.
        Otherwise, a new run is created.

        Args:
            pipeline_run: The pipeline run to get or create.

        # noqa: DAR401
        Raises:
            ValueError: If the request does not contain an orchestrator run ID.
            EntityExistsError: If a run with the same name already exists.
            RuntimeError: If the run fetching failed unexpectedly.

        Returns:
            The pipeline run, and a boolean indicating whether the run was
            created or not.
        """
        if not pipeline_run.orchestrator_run_id:
            raise ValueError(
                "Unable to get or create run for request with missing "
                "orchestrator run ID."
            )

        try:
            return (
                self._replace_placeholder_run(pipeline_run=pipeline_run),
                True,
            )
        except KeyError:
            # We were not able to find/replace a placeholder run. This could be
            # due to one of the following three reasons:
            # (1) There never was a placeholder run for the deployment. This is
            #     the case if the user ran the pipeline on a schedule.
            # (2) There was a placeholder run, but a previous pipeline run
            #     already used it. This is the case if users rerun a pipeline
            #     run e.g. from the orchestrator UI, as they will use the same
            #     deployment_id with a new orchestrator_run_id.
            # (3) A step of the same pipeline run already replaced the
            #     placeholder run.
            pass

        try:
            # We now try to create a new run. The following will happen in the
            # three cases described above:
            # (1) The behavior depends on whether we're the first step of the
            #     pipeline run that's trying to create the run. If yes, the
            #     `self.create_run(...)` will succeed. If no, a run with the
            #     same deployment_id and orchestrator_run_id already exists and
            #     the `self.create_run(...)` call will fail due to the unique
            #     constraint on those columns.
            # (2) Same as (1).
            # (3) A step of the same pipeline run replaced the placeholder
            #     run, which now contains the deployment_id and
            #     orchestrator_run_id of the run that we're trying to create.
            #     -> The `self.create_run(...) call will fail due to the unique
            #     constraint on those columns.
            return self.create_run(pipeline_run), True
        except (EntityExistsError, IntegrityError) as create_error:
            # Creating the run failed with an
            # - IntegrityError: This happens when we violated a unique
            #   constraint, which in turn means a run with the same
            #   deployment_id and orchestrator_run_id exists. We now fetch and
            #   return that run.
            # - EntityExistsError: This happens when a run with the same name
            #   already exists. This could be either a different run (in which
            #   case we want to fail) or a run created by a step of the same
            #   pipeline run (in which case we want to return it).
            # Note: The IntegrityError might also be raised when other unique
            # constraints get violated. The only other such constraint is the
            # primary key constraint on the run ID, which means we randomly
            # generated an existing UUID. In this case the call below will fail,
            # but the chance of that happening is so low we don't handle it.
            try:
                return (
                    self._get_run_by_orchestrator_run_id(
                        orchestrator_run_id=pipeline_run.orchestrator_run_id,
                        deployment_id=pipeline_run.deployment,
                    ),
                    False,
                )
            except KeyError:
                if isinstance(create_error, EntityExistsError):
                    # There was a run with the same name which does not share
                    # the deployment_id and orchestrator_run_id -> We fail with
                    # the error that run names must be unique.
                    raise create_error from None

                # This should never happen as the run creation failed with an
                # IntegrityError which means a run with the deployment_id and
                # orchestrator_run_id exists.
                raise RuntimeError(
                    f"Failed to get or create run: {create_error}"
                )

    def list_runs(
        self,
        runs_filter_model: PipelineRunFilter,
        hydrate: bool = False,
    ) -> Page[PipelineRunResponse]:
        """List all pipeline runs matching the given filter criteria.

        Args:
            runs_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all pipeline runs matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(PipelineRunSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=PipelineRunSchema,
                filter_model=runs_filter_model,
                hydrate=hydrate,
            )

    def update_run(
        self, run_id: UUID, run_update: PipelineRunUpdate
    ) -> PipelineRunResponse:
        """Updates a pipeline run.

        Args:
            run_id: The ID of the pipeline run to update.
            run_update: The update to be applied to the pipeline run.

        Returns:
            The updated pipeline run.

        Raises:
            KeyError: if the pipeline run doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if pipeline run with the given ID exists
            existing_run = session.exec(
                select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
            ).first()
            if existing_run is None:
                raise KeyError(
                    f"Unable to update pipeline run with ID {run_id}: "
                    f"No pipeline run with this ID found."
                )

            # Update the pipeline run
            existing_run.update(run_update=run_update)
            session.add(existing_run)
            session.commit()

            session.refresh(existing_run)
            return existing_run.to_model(include_metadata=True)

    def delete_run(self, run_id: UUID) -> None:
        """Deletes a pipeline run.

        Args:
            run_id: The ID of the pipeline run to delete.

        Raises:
            KeyError: if the pipeline run doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if pipeline run with the given ID exists
            existing_run = session.exec(
                select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
            ).first()
            if existing_run is None:
                raise KeyError(
                    f"Unable to delete pipeline run with ID {run_id}: "
                    f"No pipeline run with this ID found."
                )

            # Delete the pipeline run
            session.delete(existing_run)
            session.commit()

    def count_runs(self, filter_model: Optional[PipelineRunFilter]) -> int:
        """Count all pipeline runs.

        Args:
            filter_model: The filter model to filter the runs.

        Returns:
            The number of pipeline runs.
        """
        return self._count_entity(
            schema=PipelineRunSchema, filter_model=filter_model
        )

    # ----------------------------- Run Metadata -----------------------------

    def create_run_metadata(
        self, run_metadata: RunMetadataRequest
    ) -> List[RunMetadataResponse]:
        """Creates run metadata.

        Args:
            run_metadata: The run metadata to create.

        Returns:
            The created run metadata.
        """
        return_value: List[RunMetadataResponse] = []
        with Session(self.engine) as session:
            for key, value in run_metadata.values.items():
                type_ = run_metadata.types[key]
                run_metadata_schema = RunMetadataSchema(
                    workspace_id=run_metadata.workspace,
                    user_id=run_metadata.user,
                    resource_id=run_metadata.resource_id,
                    resource_type=run_metadata.resource_type.value,
                    stack_component_id=run_metadata.stack_component_id,
                    key=key,
                    value=json.dumps(value),
                    type=type_,
                )
                session.add(run_metadata_schema)
                session.commit()
                return_value.append(
                    run_metadata_schema.to_model(include_metadata=True)
                )
        return return_value

    def get_run_metadata(
        self, run_metadata_id: UUID, hydrate: bool = True
    ) -> RunMetadataResponse:
        """Gets run metadata with the given ID.

        Args:
            run_metadata_id: The ID of the run metadata to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The run metadata.

        Raises:
            KeyError: if the run metadata doesn't exist.
        """
        with Session(self.engine) as session:
            run_metadata = session.exec(
                select(RunMetadataSchema).where(
                    RunMetadataSchema.id == run_metadata_id
                )
            ).first()
            if run_metadata is None:
                raise KeyError(
                    f"Unable to get run metadata with ID "
                    f"{run_metadata_id}: "
                    f"No run metadata with this ID found."
                )
            return run_metadata.to_model(include_metadata=hydrate)

    def list_run_metadata(
        self,
        run_metadata_filter_model: RunMetadataFilter,
        hydrate: bool = False,
    ) -> Page[RunMetadataResponse]:
        """List run metadata.

        Args:
            run_metadata_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The run metadata.
        """
        with Session(self.engine) as session:
            query = select(RunMetadataSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=RunMetadataSchema,
                filter_model=run_metadata_filter_model,
                hydrate=hydrate,
            )

    # ----------------------------- Schedules -----------------------------

    def create_schedule(self, schedule: ScheduleRequest) -> ScheduleResponse:
        """Creates a new schedule.

        Args:
            schedule: The schedule to create.

        Returns:
            The newly created schedule.
        """
        with Session(self.engine) as session:
            new_schedule = ScheduleSchema.from_request(schedule)
            session.add(new_schedule)
            session.commit()
            return new_schedule.to_model(include_metadata=True)

    def get_schedule(
        self, schedule_id: UUID, hydrate: bool = True
    ) -> ScheduleResponse:
        """Get a schedule with a given ID.

        Args:
            schedule_id: ID of the schedule.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The schedule.

        Raises:
            KeyError: if the schedule does not exist.
        """
        with Session(self.engine) as session:
            # Check if schedule with the given ID exists
            schedule = session.exec(
                select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
            ).first()
            if schedule is None:
                raise KeyError(
                    f"Unable to get schedule with ID '{schedule_id}': "
                    "No schedule with this ID found."
                )
            return schedule.to_model(include_metadata=hydrate)

    def list_schedules(
        self,
        schedule_filter_model: ScheduleFilter,
        hydrate: bool = False,
    ) -> Page[ScheduleResponse]:
        """List all schedules in the workspace.

        Args:
            schedule_filter_model: All filter parameters including pagination
                params
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of schedules.
        """
        with Session(self.engine) as session:
            query = select(ScheduleSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=ScheduleSchema,
                filter_model=schedule_filter_model,
                hydrate=hydrate,
            )

    def update_schedule(
        self,
        schedule_id: UUID,
        schedule_update: ScheduleUpdate,
    ) -> ScheduleResponse:
        """Updates a schedule.

        Args:
            schedule_id: The ID of the schedule to be updated.
            schedule_update: The update to be applied.

        Returns:
            The updated schedule.

        Raises:
            KeyError: if the schedule doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if schedule with the given ID exists
            existing_schedule = session.exec(
                select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
            ).first()
            if existing_schedule is None:
                raise KeyError(
                    f"Unable to update schedule with ID {schedule_id}: "
                    f"No schedule with this ID found."
                )

            # Update the schedule
            existing_schedule = existing_schedule.update(schedule_update)
            session.add(existing_schedule)
            session.commit()
            return existing_schedule.to_model(include_metadata=True)

    def delete_schedule(self, schedule_id: UUID) -> None:
        """Deletes a schedule.

        Args:
            schedule_id: The ID of the schedule to delete.

        Raises:
            KeyError: if the schedule doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if schedule with the given ID exists
            schedule = session.exec(
                select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
            ).first()
            if schedule is None:
                raise KeyError(
                    f"Unable to delete schedule with ID {schedule_id}: "
                    f"No schedule with this ID found."
                )

            # Delete the schedule
            session.delete(schedule)
            session.commit()

    # ------------------------- Secrets -------------------------

    def _check_sql_secret_scope(
        self,
        session: Session,
        secret_name: str,
        scope: SecretScope,
        workspace: UUID,
        user: UUID,
        exclude_secret_id: Optional[UUID] = None,
    ) -> Tuple[bool, str]:
        """Checks if a secret with the given name already exists in the given scope.

        This method enforces the following scope rules:

          - only one workspace-scoped secret with the given name can exist
            in the target workspace.
          - only one user-scoped secret with the given name can exist in the
            target workspace for the target user.

        Args:
            session: The SQLAlchemy session.
            secret_name: The name of the secret.
            scope: The scope of the secret.
            workspace: The ID of the workspace to which the secret belongs.
            user: The ID of the user to which the secret belongs.
            exclude_secret_id: The ID of a secret to exclude from the check
                (used e.g. during an update to exclude the existing secret).

        Returns:
            True if a secret with the given name already exists in the given
            scope, False otherwise, and an error message.
        """
        scope_filter = (
            select(SecretSchema)
            .where(SecretSchema.name == secret_name)
            .where(SecretSchema.scope == scope.value)
        )

        if scope in [SecretScope.WORKSPACE, SecretScope.USER]:
            scope_filter = scope_filter.where(
                SecretSchema.workspace_id == workspace
            )
        if scope == SecretScope.USER:
            scope_filter = scope_filter.where(SecretSchema.user_id == user)
        if exclude_secret_id is not None:
            scope_filter = scope_filter.where(
                SecretSchema.id != exclude_secret_id
            )

        existing_secret = session.exec(scope_filter).first()

        if existing_secret is not None:
            existing_secret_model = existing_secret.to_model(
                include_metadata=True
            )

            msg = (
                f"Found an existing {scope.value} scoped secret with the "
                f"same '{secret_name}' name"
            )
            if scope in [SecretScope.WORKSPACE, SecretScope.USER]:
                msg += (
                    f" in the same '{existing_secret_model.workspace.name}' "
                    f"workspace"
                )
            if scope == SecretScope.USER:
                assert existing_secret_model.user
                msg += (
                    f" for the same '{existing_secret_model.user.name}' user"
                )

            return True, msg

        return False, ""

    def _set_secret_values(
        self, secret_id: UUID, values: Dict[str, str], backup: bool = True
    ) -> None:
        """Sets the values of a secret in the configured secrets store.

        Args:
            secret_id: The ID of the secret to set the values of.
            values: The values to set.
            backup: Whether to back up the values in the backup secrets store,
                if configured.

        # noqa: DAR401
        """

        def do_backup() -> bool:
            """Backs up the values of a secret in the configured backup secrets store.

            Returns:
                True if the backup succeeded, False otherwise.
            """
            if not backup or not self.backup_secrets_store:
                return False
            logger.info(
                f"Storing secret {secret_id} in the backup secrets store. "
            )
            try:
                self._backup_secret_values(secret_id=secret_id, values=values)
            except Exception:
                logger.exception(
                    f"Failed to store secret values for secret with ID "
                    f"{secret_id} in the backup secrets store. "
                )
                return False
            return True

        try:
            self.secrets_store.store_secret_values(
                secret_id=secret_id, secret_values=values
            )
        except Exception:
            logger.exception(
                f"Failed to store secret values for secret with ID "
                f"{secret_id} in the primary secrets store. "
            )
            if not do_backup():
                raise
        else:
            do_backup()

    def _backup_secret_values(
        self, secret_id: UUID, values: Dict[str, str]
    ) -> None:
        """Backs up the values of a secret in the configured backup secrets store.

        Args:
            secret_id: The ID of the secret the values of which to backup.
            values: The values to back up.
        """
        if self.backup_secrets_store:
            # We attempt either an update or a create operation depending on
            # whether the secret values are already stored in the backup secrets
            # store. This is to account for any inconsistencies in the backup
            # secrets store without impairing the backup functionality.
            try:
                self.backup_secrets_store.get_secret_values(
                    secret_id=secret_id,
                )
            except KeyError:
                self.backup_secrets_store.store_secret_values(
                    secret_id=secret_id, secret_values=values
                )
            else:
                self.backup_secrets_store.update_secret_values(
                    secret_id=secret_id, secret_values=values
                )

    def _get_secret_values(
        self, secret_id: UUID, use_backup: bool = True
    ) -> Dict[str, str]:
        """Gets the values of a secret from the configured secrets store.

        Args:
            secret_id: The ID of the secret to get the values of.
            use_backup: Whether to use the backup secrets store if the primary
                secrets store fails to retrieve the values and if a backup
                secrets store is configured.

        Returns:
            The values of the secret.

        # noqa: DAR401
        """
        try:
            return self.secrets_store.get_secret_values(
                secret_id=secret_id,
            )
        except Exception as e:
            if use_backup and self.backup_secrets_store:
                logger.exception(
                    f"Failed to get secret values for secret with ID "
                    f"{secret_id} from the primary secrets store. "
                    f"Trying to get them from the backup secrets store. "
                )
                try:
                    backup_values = self._get_backup_secret_values(
                        secret_id=secret_id
                    )
                    if isinstance(e, KeyError):
                        # Attempt to automatically restore the values in the
                        # primary secrets store if the backup secrets store
                        # succeeds in retrieving them and if the values are
                        # missing in the primary secrets store.
                        try:
                            self.secrets_store.store_secret_values(
                                secret_id=secret_id,
                                secret_values=backup_values,
                            )
                        except Exception:
                            logger.exception(
                                f"Failed to restore secret values for secret "
                                f"with ID {secret_id} in the primary secrets "
                                "store. "
                            )
                    return backup_values
                except Exception:
                    logger.exception(
                        f"Failed to get secret values for secret with ID "
                        f"{secret_id} from the backup secrets store. "
                    )
            raise

    def _get_backup_secret_values(self, secret_id: UUID) -> Dict[str, str]:
        """Gets the backup values of a secret from the configured backup secrets store.

        Args:
            secret_id: The ID of the secret to get the values of.

        Returns:
            The backup values of the secret.

        Raises:
            KeyError: If no backup secrets store is configured.
        """
        if self.backup_secrets_store:
            return self.backup_secrets_store.get_secret_values(
                secret_id=secret_id,
            )
        raise KeyError(
            f"Unable to get backup secret values for secret with ID "
            f"{secret_id}: No backup secrets store is configured."
        )

    def _update_secret_values(
        self,
        secret_id: UUID,
        values: Dict[str, Optional[str]],
        overwrite: bool = False,
        backup: bool = True,
    ) -> Dict[str, str]:
        """Updates the values of a secret in the configured secrets store.

        This method will update the existing values with the new values
        and drop `None` values.

        Args:
            secret_id: The ID of the secret to set the values of.
            values: The updated values to set.
            overwrite: Whether to overwrite the existing values with the new
                values. If set to False, the new values will be merged with the
                existing values.
            backup: Whether to back up the updated values in the backup secrets
                store, if configured.

        Returns:
            The updated values.

        # noqa: DAR401
        """
        try:
            existing_values = self._get_secret_values(
                secret_id=secret_id, use_backup=backup
            )
        except KeyError:
            logger.error(
                f"Unable to update secret values for secret with ID "
                f"{secret_id}: No secret with this ID found in the secrets "
                f"store back-end. Creating a new secret instead."
            )
            # If no secret values are yet stored in the secrets store,
            # we simply treat this as a create operation. This is to account
            # for cases in which secrets are manually deleted in the secrets
            # store backend or when the secrets store backend is reconfigured to
            # a different account, provider, region etc. without migrating
            # the actual existing secrets themselves.
            new_values: Dict[str, str] = {
                k: v for k, v in values.items() if v is not None
            }
            self._set_secret_values(
                secret_id=secret_id, values=new_values, backup=backup
            )
            return new_values

        if overwrite:
            existing_values = {
                k: v for k, v in values.items() if v is not None
            }
        else:
            for k, v in values.items():
                if v is not None:
                    existing_values[k] = v
                # Drop values removed in the update
                if v is None and k in existing_values:
                    del existing_values[k]

        def do_backup() -> bool:
            """Backs up the values of a secret in the configured backup secrets store.

            Returns:
                True if the backup succeeded, False otherwise.
            """
            if not backup or not self.backup_secrets_store:
                return False
            logger.info(
                f"Storing secret {secret_id} in the backup secrets store. "
            )
            try:
                self._backup_secret_values(
                    secret_id=secret_id, values=existing_values
                )
            except Exception:
                logger.exception(
                    f"Failed to store secret values for secret with ID "
                    f"{secret_id} in the backup secrets store. "
                )
                return False
            return True

        try:
            self.secrets_store.update_secret_values(
                secret_id=secret_id, secret_values=existing_values
            )
        except Exception:
            logger.exception(
                f"Failed to update secret values for secret with ID "
                f"{secret_id} in the primary secrets store. "
            )
            if not do_backup():
                raise
        else:
            do_backup()

        return existing_values

    def _delete_secret_values(
        self,
        secret_id: UUID,
        delete_backup: bool = True,
    ) -> None:
        """Deletes the values of a secret in the configured secrets store.

        Args:
            secret_id: The ID of the secret for which to delete the values.
            delete_backup: Whether to delete the backup values of the secret
                from the backup secrets store, if configured.

        # noqa: DAR401
        """

        def do_delete_backup() -> bool:
            """Deletes the backup values of a secret in the configured backup secrets store.

            Returns:
                True if the backup deletion succeeded, False otherwise.
            """
            if not delete_backup or not self.backup_secrets_store:
                return False

            logger.info(
                f"Deleting secret {secret_id} from the backup secrets store."
            )
            try:
                self._delete_backup_secret_values(secret_id=secret_id)
            except KeyError:
                # If the secret doesn't exist in the backup secrets store, we
                # consider this a success.
                return True
            except Exception:
                logger.exception(
                    f"Failed to delete secret values for secret with ID "
                    f"{secret_id} from the backup secrets store. "
                )
                return False

            return True

        try:
            self.secrets_store.delete_secret_values(secret_id=secret_id)
        except KeyError:
            # If the secret doesn't exist in the primary secrets store, we
            # consider this a success.
            do_delete_backup()
        except Exception:
            logger.exception(
                f"Failed to delete secret values for secret with ID "
                f"{secret_id} from the primary secrets store. "
            )
            if not do_delete_backup():
                raise
        else:
            do_delete_backup()

    def _delete_backup_secret_values(
        self,
        secret_id: UUID,
    ) -> None:
        """Deletes the backup values of a secret in the configured backup secrets store.

        Args:
            secret_id: The ID of the secret for which to delete the backup values.
        """
        if self.backup_secrets_store:
            self.backup_secrets_store.delete_secret_values(secret_id=secret_id)

    @track_decorator(AnalyticsEvent.CREATED_SECRET)
    def create_secret(self, secret: SecretRequest) -> SecretResponse:
        """Creates a new secret.

        The new secret is also validated against the scoping rules enforced in
        the secrets store:

          - only one workspace-scoped secret with the given name can exist
            in the target workspace.
          - only one user-scoped secret with the given name can exist in the
            target workspace for the target user.

        Args:
            secret: The secret to create.

        Returns:
            The newly created secret.

        Raises:
            EntityExistsError: If a secret with the same name already exists in
                the same scope.
        """
        with Session(self.engine) as session:
            # Check if a secret with the same name already exists in the same
            # scope.
            secret_exists, msg = self._check_sql_secret_scope(
                session=session,
                secret_name=secret.name,
                scope=secret.scope,
                workspace=secret.workspace,
                user=secret.user,
            )
            if secret_exists:
                raise EntityExistsError(msg)

            new_secret = SecretSchema.from_request(
                secret,
            )
            session.add(new_secret)
            session.commit()

            secret_model = new_secret.to_model(include_metadata=True)

        try:
            # Set the secret values in the configured secrets store
            self._set_secret_values(
                secret_id=new_secret.id, values=secret.secret_values
            )
        except:
            # If setting the secret values fails, delete the secret from the
            # database.
            with Session(self.engine) as session:
                session.delete(new_secret)
                session.commit()
            raise

        secret_model.set_secrets(secret.secret_values)
        return secret_model

    def get_secret(
        self, secret_id: UUID, hydrate: bool = True
    ) -> SecretResponse:
        """Get a secret by ID.

        Args:
            secret_id: The ID of the secret to fetch.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The secret.

        Raises:
            KeyError: if the secret doesn't exist.
        """
        with Session(self.engine) as session:
            secret_in_db = session.exec(
                select(SecretSchema).where(SecretSchema.id == secret_id)
            ).first()
            if secret_in_db is None:
                raise KeyError(f"Secret with ID {secret_id} not found.")
            secret_model = secret_in_db.to_model(include_metadata=hydrate)

        secret_model.set_secrets(self._get_secret_values(secret_id=secret_id))

        return secret_model

    def list_secrets(
        self, secret_filter_model: SecretFilter, hydrate: bool = False
    ) -> Page[SecretResponse]:
        """List all secrets matching the given filter criteria.

        Note that returned secrets do not include any secret values. To fetch
        the secret values, use `get_secret`.

        Args:
            secret_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all secrets matching the filter criteria, with pagination
            information and sorted according to the filter criteria. The
            returned secrets do not include any secret values, only metadata. To
            fetch the secret values, use `get_secret` individually with each
            secret.
        """
        with Session(self.engine) as session:
            query = select(SecretSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=SecretSchema,
                filter_model=secret_filter_model,
                hydrate=hydrate,
            )

    def update_secret(
        self, secret_id: UUID, secret_update: SecretUpdate
    ) -> SecretResponse:
        """Updates a secret.

        Secret values that are specified as `None` in the update that are
        present in the existing secret are removed from the existing secret.
        Values that are present in both secrets are overwritten. All other
        values in both the existing secret and the update are kept (merged).

        If the update includes a change of name or scope, the scoping rules
        enforced in the secrets store are used to validate the update:

          - only one workspace-scoped secret with the given name can exist
            in the target workspace.
          - only one user-scoped secret with the given name can exist in the
            target workspace for the target user.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_update: The update to be applied.

        Returns:
            The updated secret.

        Raises:
            KeyError: if the secret doesn't exist.
            EntityExistsError: If a secret with the same name already exists in
                the same scope.
        """
        with Session(self.engine) as session:
            existing_secret = session.exec(
                select(SecretSchema).where(SecretSchema.id == secret_id)
            ).first()

            if not existing_secret:
                raise KeyError(f"Secret with ID {secret_id} not found.")

            # A change in name or scope requires a check of the scoping rules.
            if (
                secret_update.name is not None
                and existing_secret.name != secret_update.name
                or secret_update.scope is not None
                and existing_secret.scope != secret_update.scope
            ):
                secret_exists, msg = self._check_sql_secret_scope(
                    session=session,
                    secret_name=secret_update.name or existing_secret.name,
                    scope=secret_update.scope
                    or SecretScope(existing_secret.scope),
                    workspace=existing_secret.workspace.id,
                    user=existing_secret.user.id,
                    exclude_secret_id=secret_id,
                )

                if secret_exists:
                    raise EntityExistsError(msg)

            existing_secret.update(
                secret_update=secret_update,
            )
            session.add(existing_secret)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(existing_secret)
            secret_model = existing_secret.to_model(include_metadata=True)

        if secret_update.values is not None:
            # Update the secret values in the configured secrets store
            updated_values = self._update_secret_values(
                secret_id=secret_id,
                values=secret_update.get_secret_values_update(),
            )
            secret_model.set_secrets(updated_values)
        else:
            secret_model.set_secrets(self._get_secret_values(secret_id))

        return secret_model

    def delete_secret(self, secret_id: UUID) -> None:
        """Delete a secret.

        Args:
            secret_id: The id of the secret to delete.

        Raises:
            KeyError: if the secret doesn't exist.
        """
        # Delete the secret values in the configured secrets store
        try:
            self._delete_secret_values(secret_id=secret_id)
        except KeyError:
            # If the secret values don't exist in the secrets store, we don't
            # need to raise an error.
            pass

        with Session(self.engine) as session:
            try:
                secret_in_db = session.exec(
                    select(SecretSchema).where(SecretSchema.id == secret_id)
                ).one()
                session.delete(secret_in_db)
                session.commit()
            except NoResultFound:
                raise KeyError(f"Secret with ID {secret_id} not found.")

    def backup_secrets(
        self, ignore_errors: bool = True, delete_secrets: bool = False
    ) -> None:
        """Backs up all secrets to the configured backup secrets store.

        Args:
            ignore_errors: Whether to ignore individual errors during the backup
                process and attempt to backup all secrets.
            delete_secrets: Whether to delete the secrets that have been
                successfully backed up from the primary secrets store. Setting
                this flag effectively moves all secrets from the primary secrets
                store to the backup secrets store.

        # noqa: DAR401
        Raises:
            BackupSecretsStoreNotConfiguredError: if no backup secrets store is
                configured.
        """
        if not self.backup_secrets_store:
            raise BackupSecretsStoreNotConfiguredError(
                "Unable to backup secrets: No backup secrets store is "
                "configured."
            )

        with Session(self.engine) as session:
            secrets_in_db = session.exec(select(SecretSchema)).all()

        for secret in secrets_in_db:
            try:
                values = self._get_secret_values(
                    secret_id=secret.id, use_backup=False
                )
            except Exception:
                logger.exception(
                    f"Failed to get secret values for secret with ID "
                    f"{secret.id}."
                )
                if ignore_errors:
                    continue
                raise

            try:
                self._backup_secret_values(secret_id=secret.id, values=values)
            except Exception:
                logger.exception(
                    f"Failed to backup secret with ID {secret.id}. "
                )
                if ignore_errors:
                    continue
                raise

            if delete_secrets:
                try:
                    self._delete_secret_values(
                        secret_id=secret.id, delete_backup=False
                    )
                except Exception:
                    logger.exception(
                        f"Failed to delete secret with ID {secret.id} from the "
                        f"primary secrets store after backing it up to the "
                        f"backup secrets store."
                    )
                    if ignore_errors:
                        continue
                    raise

    def restore_secrets(
        self, ignore_errors: bool = False, delete_secrets: bool = False
    ) -> None:
        """Restore all secrets from the configured backup secrets store.

        Args:
            ignore_errors: Whether to ignore individual errors during the
                restore process and attempt to restore all secrets.
            delete_secrets: Whether to delete the secrets that have been
                successfully restored from the backup secrets store. Setting
                this flag effectively moves all secrets from the backup secrets
                store to the primary secrets store.

        # noqa: DAR401
        Raises:
            BackupSecretsStoreNotConfiguredError: if no backup secrets store is
                configured.
        """
        if not self.backup_secrets_store:
            raise BackupSecretsStoreNotConfiguredError(
                "Unable to restore secrets: No backup secrets store is "
                "configured."
            )

        with Session(self.engine) as session:
            secrets_in_db = session.exec(select(SecretSchema)).all()

        for secret in secrets_in_db:
            try:
                values = self._get_backup_secret_values(secret_id=secret.id)
            except Exception:
                logger.exception(
                    f"Failed to get backup secret values for secret with ID "
                    f"{secret.id}."
                )
                if ignore_errors:
                    continue
                raise

            try:
                self._update_secret_values(
                    secret_id=secret.id,
                    values=cast(Dict[str, Optional[str]], values),
                    overwrite=True,
                    backup=False,
                )
            except Exception:
                logger.exception(
                    f"Failed to restore secret with ID {secret.id}. "
                )
                if ignore_errors:
                    continue
                raise

            if delete_secrets:
                try:
                    self._delete_backup_secret_values(secret_id=secret.id)
                except Exception:
                    logger.exception(
                        f"Failed to delete backup secret with ID {secret.id} "
                        f"from the backup secrets store after restoring it to "
                        f"the primary secrets store."
                    )
                    if ignore_errors:
                        continue
                    raise

    # ------------------------- Service Accounts -------------------------

    @track_decorator(AnalyticsEvent.CREATED_SERVICE_ACCOUNT)
    def create_service_account(
        self, service_account: ServiceAccountRequest
    ) -> ServiceAccountResponse:
        """Creates a new service account.

        Args:
            service_account: Service account to be created.

        Returns:
            The newly created service account.

        Raises:
            EntityExistsError: If a user or service account with the given name
                already exists.
        """
        with Session(self.engine) as session:
            # Check if a service account with the given name already
            # exists
            err_msg = (
                f"Unable to create service account with name "
                f"'{service_account.name}': Found existing service "
                "account with this name."
            )
            try:
                self._get_account_schema(
                    service_account.name, session=session, service_account=True
                )
                raise EntityExistsError(err_msg)
            except KeyError:
                pass

            # Create the service account
            new_account = UserSchema.from_service_account_request(
                service_account
            )
            session.add(new_account)
            # on commit an IntegrityError may arise we let it bubble up
            session.commit()

            return new_account.to_service_account_model(include_metadata=True)

    def get_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
        hydrate: bool = True,
    ) -> ServiceAccountResponse:
        """Gets a specific service account.

        Raises a KeyError in case a service account with that id does not exist.

        Args:
            service_account_name_or_id: The name or ID of the service account to
                get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested service account, if it was found.
        """
        with Session(self.engine) as session:
            account = self._get_account_schema(
                service_account_name_or_id,
                session=session,
                service_account=True,
            )

            return account.to_service_account_model(include_metadata=hydrate)

    def list_service_accounts(
        self,
        filter_model: ServiceAccountFilter,
        hydrate: bool = False,
    ) -> Page[ServiceAccountResponse]:
        """List all service accounts.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of filtered service accounts.
        """
        with Session(self.engine) as session:
            query = select(UserSchema)
            paged_service_accounts: Page[ServiceAccountResponse] = (
                self.filter_and_paginate(
                    session=session,
                    query=query,
                    table=UserSchema,
                    filter_model=filter_model,
                    custom_schema_to_model_conversion=lambda user: user.to_service_account_model(
                        include_metadata=hydrate
                    ),
                    hydrate=hydrate,
                )
            )
            return paged_service_accounts

    def update_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
        service_account_update: ServiceAccountUpdate,
    ) -> ServiceAccountResponse:
        """Updates an existing service account.

        Args:
            service_account_name_or_id: The name or the ID of the service
                account to update.
            service_account_update: The update to be applied to the service
                account.

        Returns:
            The updated service account.

        Raises:
            EntityExistsError: If a user or service account with the given name
                already exists.
        """
        with Session(self.engine) as session:
            existing_service_account = self._get_account_schema(
                service_account_name_or_id,
                session=session,
                service_account=True,
            )

            if (
                service_account_update.name is not None
                and service_account_update.name
                != existing_service_account.name
            ):
                try:
                    self._get_account_schema(
                        service_account_update.name,
                        session=session,
                        service_account=True,
                    )
                    raise EntityExistsError(
                        f"Unable to update service account with name "
                        f"'{service_account_update.name}': Found an existing "
                        "service account with this name."
                    )
                except KeyError:
                    pass

            existing_service_account.update_service_account(
                service_account_update=service_account_update
            )
            session.add(existing_service_account)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(existing_service_account)
            return existing_service_account.to_service_account_model(
                include_metadata=True
            )

    def delete_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
    ) -> None:
        """Delete a service account.

        Args:
            service_account_name_or_id: The name or the ID of the service
                account to delete.

        Raises:
            IllegalOperationError: if the service account has already been used
                to create other resources.
        """
        with Session(self.engine) as session:
            service_account = self._get_account_schema(
                service_account_name_or_id,
                session=session,
                service_account=True,
            )
            # Check if the service account has any resources associated with it
            # and raise an error if it does.
            if self._account_owns_resources(service_account, session=session):
                raise IllegalOperationError(
                    "The service account has already been used to create "
                    "other resources that it now owns and therefore cannot be "
                    "deleted. Please delete all resources owned by the service "
                    "account or consider deactivating it instead."
                )

            session.delete(service_account)
            session.commit()

    # --------------------------- Service Connectors ---------------------------

    @track_decorator(AnalyticsEvent.CREATED_SERVICE_CONNECTOR)
    def create_service_connector(
        self, service_connector: ServiceConnectorRequest
    ) -> ServiceConnectorResponse:
        """Creates a new service connector.

        Args:
            service_connector: Service connector to be created.

        Returns:
            The newly created service connector.

        Raises:
            Exception: If anything goes wrong during the creation of the
                service connector.
        """
        # If the connector type is locally available, we validate the request
        # against the connector type schema before storing it in the database
        if service_connector_registry.is_registered(service_connector.type):
            connector_type = (
                service_connector_registry.get_service_connector_type(
                    service_connector.type
                )
            )
            service_connector.validate_and_configure_resources(
                connector_type=connector_type,
                resource_types=service_connector.resource_types,
                resource_id=service_connector.resource_id,
                configuration=service_connector.configuration,
                secrets=service_connector.secrets,
            )

        with Session(self.engine) as session:
            self._fail_if_service_connector_with_name_exists(
                name=service_connector.name,
                workspace_id=service_connector.workspace,
                session=session,
            )

            # Create the secret
            secret_id = self._create_connector_secret(
                connector_name=service_connector.name,
                user=service_connector.user,
                workspace=service_connector.workspace,
                secrets=service_connector.secrets,
            )
            try:
                # Create the service connector
                new_service_connector = ServiceConnectorSchema.from_request(
                    service_connector,
                    secret_id=secret_id,
                )

                session.add(new_service_connector)
                session.commit()

                session.refresh(new_service_connector)
            except Exception:
                # Delete the secret if it was created
                if secret_id:
                    try:
                        self.delete_secret(secret_id)
                    except Exception:
                        # Ignore any errors that occur while deleting the
                        # secret
                        pass

                raise

            connector = new_service_connector.to_model(include_metadata=True)
            self._populate_connector_type(connector)
            return connector

    def get_service_connector(
        self, service_connector_id: UUID, hydrate: bool = True
    ) -> ServiceConnectorResponse:
        """Gets a specific service connector.

        Args:
            service_connector_id: The ID of the service connector to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested service connector, if it was found.

        Raises:
            KeyError: If no service connector with the given ID exists.
        """
        with Session(self.engine) as session:
            service_connector = session.exec(
                select(ServiceConnectorSchema).where(
                    ServiceConnectorSchema.id == service_connector_id
                )
            ).first()

            if service_connector is None:
                raise KeyError(
                    f"Service connector with ID {service_connector_id} not "
                    "found."
                )

            connector = service_connector.to_model(include_metadata=hydrate)
            self._populate_connector_type(connector)
            return connector

    def list_service_connectors(
        self,
        filter_model: ServiceConnectorFilter,
        hydrate: bool = False,
    ) -> Page[ServiceConnectorResponse]:
        """List all service connectors.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all service connectors.
        """

        def fetch_connectors(
            session: Session,
            query: Union[
                Select[ServiceConnectorSchema],
                SelectOfScalar[ServiceConnectorSchema],
            ],
            filter_model: BaseFilter,
        ) -> List[ServiceConnectorSchema]:
            """Custom fetch function for connector filtering and pagination.

            Applies resource type and label filters to the query.

            Args:
                session: The database session.
                query: The query to filter.
                filter_model: The filter model.

            Returns:
                The filtered and paginated results.
            """
            assert isinstance(filter_model, ServiceConnectorFilter)
            items = self._list_filtered_service_connectors(
                session=session, query=query, filter_model=filter_model
            )

            return items

        with Session(self.engine) as session:
            query = select(ServiceConnectorSchema)
            paged_connectors: Page[ServiceConnectorResponse] = (
                self.filter_and_paginate(
                    session=session,
                    query=query,
                    table=ServiceConnectorSchema,
                    filter_model=filter_model,
                    custom_fetch=fetch_connectors,
                    hydrate=hydrate,
                )
            )

            self._populate_connector_type(*paged_connectors.items)
            return paged_connectors

    def update_service_connector(
        self, service_connector_id: UUID, update: ServiceConnectorUpdate
    ) -> ServiceConnectorResponse:
        """Updates an existing service connector.

        The update model contains the fields to be updated. If a field value is
        set to None in the model, the field is not updated, but there are
        special rules concerning some fields:

        * the `configuration` and `secrets` fields together represent a full
        valid configuration update, not just a partial update. If either is
        set (i.e. not None) in the update, their values are merged together and
        will replace the existing configuration and secrets values.
        * the `resource_id` field value is also a full replacement value: if set
        to `None`, the resource ID is removed from the service connector.
        * the `expiration_seconds` field value is also a full replacement value:
        if set to `None`, the expiration is removed from the service connector.
        * the `secret_id` field value in the update is ignored, given that
        secrets are managed internally by the ZenML store.
        * the `labels` field is also a full labels update: if set (i.e. not
        `None`), all existing labels are removed and replaced by the new labels
        in the update.

        Args:
            service_connector_id: The ID of the service connector to update.
            update: The update to be applied to the service connector.

        Returns:
            The updated service connector.

        Raises:
            KeyError: If no service connector with the given ID exists.
            IllegalOperationError: If the service connector is referenced by
                one or more stack components and the update would change the
                connector type, resource type or resource ID.
        """
        with Session(self.engine) as session:
            existing_connector = session.exec(
                select(ServiceConnectorSchema).where(
                    ServiceConnectorSchema.id == service_connector_id
                )
            ).first()

            if existing_connector is None:
                raise KeyError(
                    f"Unable to update service connector with ID "
                    f"'{service_connector_id}': Found no existing service "
                    "connector with this ID."
                )

            # In case of a renaming update, make sure no service connector uses
            # that name already
            if update.name and existing_connector.name != update.name:
                self._fail_if_service_connector_with_name_exists(
                    name=update.name,
                    workspace_id=existing_connector.workspace_id,
                    session=session,
                )

            existing_connector_model = existing_connector.to_model(
                include_metadata=True
            )

            if len(existing_connector.components):
                # If the service connector is already used in one or more
                # stack components, the update is no longer allowed to change
                # the service connector's authentication method, connector type,
                # resource type, or resource ID
                if (
                    update.connector_type
                    and update.type != existing_connector_model.connector_type
                ):
                    raise IllegalOperationError(
                        "The service type of a service connector that is "
                        "already actively used in one or more stack components "
                        "cannot be changed."
                    )

                if (
                    update.auth_method
                    and update.auth_method
                    != existing_connector_model.auth_method
                ):
                    raise IllegalOperationError(
                        "The authentication method of a service connector that "
                        "is already actively used in one or more stack "
                        "components cannot be changed."
                    )

                if (
                    update.resource_types
                    and update.resource_types
                    != existing_connector_model.resource_types
                ):
                    raise IllegalOperationError(
                        "The resource type of a service connector that is "
                        "already actively used in one or more stack components "
                        "cannot be changed."
                    )

                # The resource ID field cannot be used as a partial update: if
                # set to None, the existing resource ID is also removed
                if update.resource_id != existing_connector_model.resource_id:
                    raise IllegalOperationError(
                        "The resource ID of a service connector that is "
                        "already actively used in one or more stack components "
                        "cannot be changed."
                    )

            # If the connector type is locally available, we validate the update
            # against the connector type schema before storing it in the
            # database
            if service_connector_registry.is_registered(
                existing_connector.connector_type
            ):
                connector_type = (
                    service_connector_registry.get_service_connector_type(
                        existing_connector.connector_type
                    )
                )
                # We need the auth method to be set to be able to validate the
                # configuration
                update.auth_method = (
                    update.auth_method or existing_connector_model.auth_method
                )
                # Validate the configuration update. If the configuration or
                # secrets fields are set, together they are merged into a
                # full configuration that is validated against the connector
                # type schema and replaces the existing configuration and
                # secrets values
                update.validate_and_configure_resources(
                    connector_type=connector_type,
                    resource_types=update.resource_types,
                    resource_id=update.resource_id,
                    configuration=update.configuration,
                    secrets=update.secrets,
                )

            # Update secret
            secret_id = self._update_connector_secret(
                existing_connector=existing_connector_model,
                updated_connector=update,
            )

            existing_connector.update(
                connector_update=update, secret_id=secret_id
            )
            session.add(existing_connector)
            session.commit()

            connector = existing_connector.to_model(include_metadata=True)
            self._populate_connector_type(connector)
            return connector

    def delete_service_connector(self, service_connector_id: UUID) -> None:
        """Deletes a service connector.

        Args:
            service_connector_id: The ID of the service connector to delete.

        Raises:
            KeyError: If no service connector with the given ID exists.
            IllegalOperationError: If the service connector is still referenced
                by one or more stack components.
        """
        with Session(self.engine) as session:
            try:
                service_connector = session.exec(
                    select(ServiceConnectorSchema).where(
                        ServiceConnectorSchema.id == service_connector_id
                    )
                ).one()

                if service_connector is None:
                    raise KeyError(
                        f"Service connector with ID {service_connector_id} not "
                        "found."
                    )

                if len(service_connector.components) > 0:
                    raise IllegalOperationError(
                        f"Service connector with ID {service_connector_id} "
                        f"cannot be deleted as it is still referenced by "
                        f"{len(service_connector.components)} "
                        "stack components. Before deleting this service "
                        "connector, make sure to remove it from all stack "
                        "components."
                    )
                else:
                    session.delete(service_connector)

                if service_connector.secret_id:
                    try:
                        self.delete_secret(service_connector.secret_id)
                    except KeyError:
                        # If the secret doesn't exist anymore, we can ignore
                        # this error
                        pass
            except NoResultFound as error:
                raise KeyError from error

            session.commit()

    @staticmethod
    def _fail_if_service_connector_with_name_exists(
        name: str,
        workspace_id: UUID,
        session: Session,
    ) -> None:
        """Raise an exception if a service connector with same name exists.

        Args:
            name: The name of the service connector
            workspace_id: The ID of the workspace
            session: The Session

        Raises:
            EntityExistsError: If a service connector with the given name
                already exists.
        """
        # Check if service connector with the same domain key (name, workspace)
        # already exists
        existing_domain_connector = session.exec(
            select(ServiceConnectorSchema)
            .where(ServiceConnectorSchema.name == name)
            .where(ServiceConnectorSchema.workspace_id == workspace_id)
        ).first()
        if existing_domain_connector is not None:
            raise EntityExistsError(
                f"Unable to register service connector with name '{name}': "
                "Found an existing service connector with the same name in the "
                f"same workspace '{existing_domain_connector.workspace.name}'."
            )

    def _create_connector_secret(
        self,
        connector_name: str,
        user: UUID,
        workspace: UUID,
        secrets: Optional[Dict[str, Optional[SecretStr]]],
    ) -> Optional[UUID]:
        """Creates a new secret to store the service connector secret credentials.

        Args:
            connector_name: The name of the service connector for which to
                create a secret.
            user: The ID of the user who owns the service connector.
            workspace: The ID of the workspace in which the service connector
                is registered.
            secrets: The secret credentials to store.

        Returns:
            The ID of the newly created secret or None, if the service connector
            does not contain any secret credentials.
        """
        if not secrets:
            return None

        # Generate a unique name for the secret
        # Replace all non-alphanumeric characters with a dash because
        # the secret name must be a valid DNS subdomain name in some
        # secrets stores
        connector_name = re.sub(r"[^a-zA-Z0-9-]", "-", connector_name)
        # Generate unique names using a random suffix until we find a name
        # that is not already in use
        while True:
            secret_name = f"connector-{connector_name}-{random_str(4)}".lower()
            existing_secrets = self.list_secrets(
                SecretFilter(
                    name=secret_name,
                )
            )
            if not existing_secrets.size:
                try:
                    return self.create_secret(
                        SecretRequest(
                            name=secret_name,
                            user=user,
                            workspace=workspace,
                            scope=SecretScope.WORKSPACE,
                            values=secrets,
                        )
                    ).id
                except KeyError:
                    # The secret already exists, try again
                    continue

    @staticmethod
    def _populate_connector_type(
        *service_connectors: ServiceConnectorResponse,
    ) -> None:
        """Populates the connector type of the given service connectors.

        If the connector type is not locally available, the connector type
        field is left as is.

        Args:
            service_connectors: The service connectors to populate.
        """
        for service_connector in service_connectors:
            if not service_connector_registry.is_registered(
                service_connector.type
            ):
                continue
            service_connector.set_connector_type(
                service_connector_registry.get_service_connector_type(
                    service_connector.type
                )
            )

    @staticmethod
    def _list_filtered_service_connectors(
        session: Session,
        query: Union[
            Select[ServiceConnectorSchema],
            SelectOfScalar[ServiceConnectorSchema],
        ],
        filter_model: ServiceConnectorFilter,
    ) -> List[ServiceConnectorSchema]:
        """Refine a service connector query.

        Applies resource type and label filters to the query.

        Args:
            session: The database session.
            query: The query to filter.
            filter_model: The filter model.

        Returns:
            The filtered list of service connectors.
        """
        items: List[ServiceConnectorSchema] = (
            session.exec(query).unique().all()
        )

        # filter out items that don't match the resource type
        if filter_model.resource_type:
            items = [
                item
                for item in items
                if filter_model.resource_type in item.resource_types_list
            ]

        # filter out items that don't match the labels
        if filter_model.labels:
            items = [
                item for item in items if item.has_labels(filter_model.labels)
            ]

        return items

    def _update_connector_secret(
        self,
        existing_connector: ServiceConnectorResponse,
        updated_connector: ServiceConnectorUpdate,
    ) -> Optional[UUID]:
        """Updates the secret for a service connector.

        If the secrets field in the service connector update is set (i.e. not
        None), the existing secret, if any, is replaced. If the secrets field is
        set to an empty dict, the existing secret is deleted.

        Args:
            existing_connector: Existing service connector for which to update a
                secret.
            updated_connector: Updated service connector.

        Returns:
            The ID of the updated secret or None, if the new service connector
            does not contain any secret credentials.
        """
        if updated_connector.secrets is None:
            # If the connector update does not contain a secrets update, keep
            # the existing secret (if any)
            return existing_connector.secret_id

        # Delete the existing secret (if any), to be replaced by the new secret
        if existing_connector.secret_id:
            try:
                self.delete_secret(existing_connector.secret_id)
            except KeyError:
                # Ignore if the secret no longer exists
                pass

        # If the new service connector does not contain any secret credentials,
        # return None
        if not updated_connector.secrets:
            return None

        assert existing_connector.user is not None
        # A secret does not exist yet, create a new one
        return self._create_connector_secret(
            connector_name=updated_connector.name or existing_connector.name,
            user=existing_connector.user.id,
            workspace=existing_connector.workspace.id,
            secrets=updated_connector.secrets,
        )

    def verify_service_connector_config(
        self,
        service_connector: ServiceConnectorRequest,
        list_resources: bool = True,
    ) -> ServiceConnectorResourcesModel:
        """Verifies if a service connector configuration has access to resources.

        Args:
            service_connector: The service connector configuration to verify.
            list_resources: If True, the list of all resources accessible
                through the service connector is returned.

        Returns:
            The list of resources that the service connector configuration has
            access to.
        """
        connector_instance = service_connector_registry.instantiate_connector(
            model=service_connector
        )
        return connector_instance.verify(list_resources=list_resources)

    def verify_service_connector(
        self,
        service_connector_id: UUID,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
        list_resources: bool = True,
    ) -> ServiceConnectorResourcesModel:
        """Verifies if a service connector instance has access to one or more resources.

        Args:
            service_connector_id: The ID of the service connector to verify.
            resource_type: The type of resource to verify access to.
            resource_id: The ID of the resource to verify access to.
            list_resources: If True, the list of all resources accessible
                through the service connector and matching the supplied resource
                type and ID are returned.

        Returns:
            The list of resources that the service connector has access to,
            scoped to the supplied resource type and ID, if provided.
        """
        connector = self.get_service_connector(service_connector_id)

        connector_instance = service_connector_registry.instantiate_connector(
            model=connector
        )

        return connector_instance.verify(
            resource_type=resource_type,
            resource_id=resource_id,
            list_resources=list_resources,
        )

    def get_service_connector_client(
        self,
        service_connector_id: UUID,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
    ) -> ServiceConnectorResponse:
        """Get a service connector client for a service connector and given resource.

        Args:
            service_connector_id: The ID of the base service connector to use.
            resource_type: The type of resource to get a client for.
            resource_id: The ID of the resource to get a client for.

        Returns:
            A service connector client that can be used to access the given
            resource.
        """
        connector = self.get_service_connector(service_connector_id)

        connector_instance = service_connector_registry.instantiate_connector(
            model=connector
        )

        # Fetch the connector client
        connector_client = connector_instance.get_connector_client(
            resource_type=resource_type,
            resource_id=resource_id,
        )

        # Return the model for the connector client
        connector = connector_client.to_response_model(
            user=connector.user,
            workspace=connector.workspace,
            description=connector.description,
            labels=connector.labels,
        )

        self._populate_connector_type(connector)

        return connector

    def list_service_connector_resources(
        self,
        workspace_name_or_id: Union[str, UUID],
        connector_type: Optional[str] = None,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
        filter_model: Optional[ServiceConnectorFilter] = None,
    ) -> List[ServiceConnectorResourcesModel]:
        """List resources that can be accessed by service connectors.

        Args:
            workspace_name_or_id: The name or ID of the workspace to scope to.
            connector_type: The type of service connector to scope to.
            resource_type: The type of resource to scope to.
            resource_id: The ID of the resource to scope to.
            filter_model: Optional filter model to use when fetching service
                connectors.

        Returns:
            The matching list of resources that available service
            connectors have access to.
        """
        workspace = self.get_workspace(workspace_name_or_id)

        if not filter_model:
            filter_model = ServiceConnectorFilter(
                connector_type=connector_type,
                resource_type=resource_type,
                workspace_id=workspace.id,
            )

        service_connectors = self.list_service_connectors(
            filter_model=filter_model
        ).items

        resource_list: List[ServiceConnectorResourcesModel] = []

        for connector in service_connectors:
            if not service_connector_registry.is_registered(connector.type):
                # For connectors that we can instantiate, i.e. those that have a
                # connector type available locally, we return complete
                # information about the resources that they have access to.
                #
                # For those that are not locally available, we only return
                # rudimentary information extracted from the connector model
                # without actively trying to discover the resources that they
                # have access to.

                if resource_id and connector.resource_id != resource_id:
                    # If an explicit resource ID is required, the connector
                    # has to be configured with it.
                    continue

                resources = (
                    ServiceConnectorResourcesModel.from_connector_model(
                        connector,
                        resource_type=resource_type,
                    )
                )
                for r in resources.resources:
                    if not r.resource_ids:
                        r.error = (
                            f"The service '{connector.type}' connector type is "
                            "not available."
                        )

            else:
                try:
                    connector_instance = (
                        service_connector_registry.instantiate_connector(
                            model=connector
                        )
                    )

                    resources = connector_instance.verify(
                        resource_type=resource_type,
                        resource_id=resource_id,
                        list_resources=True,
                    )
                except (ValueError, AuthorizationException) as e:
                    error = (
                        f'Failed to fetch {resource_type or "available"} '
                        f"resources from service connector {connector.name}/"
                        f"{connector.id}: {e}"
                    )
                    # Log an exception if debug logging is enabled
                    if logger.isEnabledFor(logging.DEBUG):
                        logger.exception(error)
                    else:
                        logger.error(error)
                    continue

            resource_list.append(resources)

        return resource_list

    def list_service_connector_types(
        self,
        connector_type: Optional[str] = None,
        resource_type: Optional[str] = None,
        auth_method: Optional[str] = None,
    ) -> List[ServiceConnectorTypeModel]:
        """Get a list of service connector types.

        Args:
            connector_type: Filter by connector type.
            resource_type: Filter by resource type.
            auth_method: Filter by authentication method.

        Returns:
            List of service connector types.
        """
        return service_connector_registry.list_service_connector_types(
            connector_type=connector_type,
            resource_type=resource_type,
            auth_method=auth_method,
        )

    def get_service_connector_type(
        self,
        connector_type: str,
    ) -> ServiceConnectorTypeModel:
        """Returns the requested service connector type.

        Args:
            connector_type: the service connector type identifier.

        Returns:
            The requested service connector type.
        """
        return service_connector_registry.get_service_connector_type(
            connector_type
        )

    # ----------------------------- Stacks -----------------------------

    @track_decorator(AnalyticsEvent.REGISTERED_STACK)
    def create_stack(self, stack: StackRequest) -> StackResponse:
        """Register a new stack.

        Args:
            stack: The stack to register.

        Returns:
            The registered stack.
        """
        with Session(self.engine) as session:
            self._fail_if_stack_with_name_exists(stack=stack, session=session)

            # Get the Schemas of all components mentioned
            component_ids = (
                [
                    component_id
                    for list_of_component_ids in stack.components.values()
                    for component_id in list_of_component_ids
                ]
                if stack.components is not None
                else []
            )
            filters = [
                (StackComponentSchema.id == component_id)
                for component_id in component_ids
            ]

            defined_components = session.exec(
                select(StackComponentSchema).where(or_(*filters))
            ).all()

            new_stack_schema = StackSchema(
                workspace_id=stack.workspace,
                user_id=stack.user,
                stack_spec_path=stack.stack_spec_path,
                name=stack.name,
                description=stack.description,
                components=defined_components,
            )

            session.add(new_stack_schema)
            session.commit()
            session.refresh(new_stack_schema)

            return new_stack_schema.to_model(include_metadata=True)

    def get_stack(self, stack_id: UUID, hydrate: bool = True) -> StackResponse:
        """Get a stack by its unique ID.

        Args:
            stack_id: The ID of the stack to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack with the given ID.

        Raises:
            KeyError: if the stack doesn't exist.
        """
        with Session(self.engine) as session:
            stack = session.exec(
                select(StackSchema).where(StackSchema.id == stack_id)
            ).first()

            if stack is None:
                raise KeyError(f"Stack with ID {stack_id} not found.")
            return stack.to_model(include_metadata=hydrate)

    def list_stacks(
        self,
        stack_filter_model: StackFilter,
        hydrate: bool = False,
    ) -> Page[StackResponse]:
        """List all stacks matching the given filter criteria.

        Args:
            stack_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all stacks matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(StackSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=StackSchema,
                filter_model=stack_filter_model,
                hydrate=hydrate,
            )

    @track_decorator(AnalyticsEvent.UPDATED_STACK)
    def update_stack(
        self, stack_id: UUID, stack_update: StackUpdate
    ) -> StackResponse:
        """Update a stack.

        Args:
            stack_id: The ID of the stack update.
            stack_update: The update request on the stack.

        Returns:
            The updated stack.

        Raises:
            KeyError: if the stack doesn't exist.
            IllegalOperationError: if the stack is a default stack.
        """
        with Session(self.engine) as session:
            # Check if stack with the domain key (name, workspace, owner)
            # already exists
            existing_stack = session.exec(
                select(StackSchema).where(StackSchema.id == stack_id)
            ).first()
            if existing_stack is None:
                raise KeyError(
                    f"Unable to update stack with id '{stack_id}': Found no"
                    f"existing stack with this id."
                )
            if existing_stack.name == DEFAULT_STACK_AND_COMPONENT_NAME:
                raise IllegalOperationError(
                    "The default stack cannot be modified."
                )
            # In case of a renaming update, make sure no stack already exists
            # with that name
            if stack_update.name:
                if existing_stack.name != stack_update.name:
                    self._fail_if_stack_with_name_exists(
                        stack=stack_update,
                        session=session,
                    )

            components = []
            if stack_update.components:
                filters = [
                    (StackComponentSchema.id == component_id)
                    for list_of_component_ids in stack_update.components.values()
                    for component_id in list_of_component_ids
                ]
                components = session.exec(
                    select(StackComponentSchema).where(or_(*filters))
                ).all()

            existing_stack.update(
                stack_update=stack_update,
                components=components,
            )

            session.add(existing_stack)
            session.commit()
            session.refresh(existing_stack)

            return existing_stack.to_model(include_metadata=True)

    def delete_stack(self, stack_id: UUID) -> None:
        """Delete a stack.

        Args:
            stack_id: The ID of the stack to delete.

        Raises:
            KeyError: if the stack doesn't exist.
            IllegalOperationError: if the stack is a default stack.
        """
        with Session(self.engine) as session:
            try:
                stack = session.exec(
                    select(StackSchema).where(StackSchema.id == stack_id)
                ).one()

                if stack is None:
                    raise KeyError(f"Stack with ID {stack_id} not found.")
                if stack.name == DEFAULT_STACK_AND_COMPONENT_NAME:
                    raise IllegalOperationError(
                        "The default stack cannot be deleted."
                    )
                session.delete(stack)
            except NoResultFound as error:
                raise KeyError from error

            session.commit()

    def count_stacks(self, filter_model: Optional[StackFilter]) -> int:
        """Count all stacks.

        Args:
            filter_model: The filter model to filter the stacks.

        Returns:
            The number of stacks.
        """
        return self._count_entity(
            schema=StackSchema, filter_model=filter_model
        )

    def _fail_if_stack_with_name_exists(
        self,
        stack: StackRequest,
        session: Session,
    ) -> None:
        """Raise an exception if a stack with same name exists.

        Args:
            stack: The Stack
            session: The Session

        Returns:
            None

        Raises:
            StackExistsError: If a stack with the given name already exists.
        """
        existing_domain_stack = session.exec(
            select(StackSchema)
            .where(StackSchema.name == stack.name)
            .where(StackSchema.workspace_id == stack.workspace)
        ).first()
        if existing_domain_stack is not None:
            workspace = self._get_workspace_schema(
                workspace_name_or_id=stack.workspace, session=session
            )
            raise StackExistsError(
                f"Unable to register stack with name "
                f"'{stack.name}': Found an existing stack with the same "
                f"name in the active workspace, '{workspace.name}'."
            )
        return None

    def _create_default_stack(
        self,
        workspace_id: UUID,
    ) -> StackResponse:
        """Create the default stack components and stack.

        The default stack contains a local orchestrator and a local artifact
        store.

        Args:
            workspace_id: ID of the workspace to which the stack
                belongs.

        Returns:
            The model of the created default stack.
        """
        with analytics_disabler():
            workspace = self.get_workspace(workspace_name_or_id=workspace_id)

            logger.info(
                f"Creating default stack in workspace {workspace.name}..."
            )

            orchestrator = self.create_stack_component(
                component=InternalComponentRequest(
                    # Passing `None` for the user here means the orchestrator
                    # is owned by the server, which for RBAC indicates that
                    # everyone can read it
                    user=None,
                    workspace=workspace.id,
                    name=DEFAULT_STACK_AND_COMPONENT_NAME,
                    type=StackComponentType.ORCHESTRATOR,
                    flavor="local",
                    configuration={},
                ),
            )

            artifact_store = self.create_stack_component(
                component=InternalComponentRequest(
                    # Passing `None` for the user here means the stack is owned
                    # by the server, which for RBAC indicates that everyone can
                    # read it
                    user=None,
                    workspace=workspace.id,
                    name=DEFAULT_STACK_AND_COMPONENT_NAME,
                    type=StackComponentType.ARTIFACT_STORE,
                    flavor="local",
                    configuration={},
                ),
            )

            components = {
                c.type: [c.id] for c in [orchestrator, artifact_store]
            }

            stack = InternalStackRequest(
                # Passing `None` for the user here means the stack is owned by
                # the server, which for RBAC indicates that everyone can read it
                user=None,
                name=DEFAULT_STACK_AND_COMPONENT_NAME,
                components=components,
                workspace=workspace.id,
            )
            return self.create_stack(stack=stack)

    def _get_or_create_default_stack(
        self, workspace: WorkspaceResponse
    ) -> StackResponse:
        """Get or create the default stack if it doesn't exist.

        Args:
            workspace: The workspace for which to create the default stack.

        Returns:
            The default stack.
        """
        try:
            return self._get_default_stack(
                workspace_id=workspace.id,
            )
        except KeyError:
            return self._create_default_stack(
                workspace_id=workspace.id,
            )

    # ----------------------------- Step runs -----------------------------

    def create_run_step(self, step_run: StepRunRequest) -> StepRunResponse:
        """Creates a step run.

        Args:
            step_run: The step run to create.

        Returns:
            The created step run.

        Raises:
            EntityExistsError: if the step run already exists.
            KeyError: if the pipeline run doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if the pipeline run exists
            run = session.exec(
                select(PipelineRunSchema).where(
                    PipelineRunSchema.id == step_run.pipeline_run_id
                )
            ).first()
            if run is None:
                raise KeyError(
                    f"Unable to create step '{step_run.name}': No pipeline run "
                    f"with ID '{step_run.pipeline_run_id}' found."
                )

            # Check if the step name already exists in the pipeline run
            existing_step_run = session.exec(
                select(StepRunSchema)
                .where(StepRunSchema.name == step_run.name)
                .where(
                    StepRunSchema.pipeline_run_id == step_run.pipeline_run_id
                )
            ).first()
            if existing_step_run is not None:
                raise EntityExistsError(
                    f"Unable to create step '{step_run.name}': A step with "
                    f"this name already exists in the pipeline run with ID "
                    f"'{step_run.pipeline_run_id}'."
                )

            # Create the step
            step_schema = StepRunSchema.from_request(step_run)
            session.add(step_schema)

            # Add logs entry for the step if exists
            if step_run.logs is not None:
                log_entry = LogsSchema(
                    uri=step_run.logs.uri,
                    step_run_id=step_schema.id,
                    artifact_store_id=step_run.logs.artifact_store_id,
                )
                session.add(log_entry)

            # Save parent step IDs into the database.
            for parent_step_id in step_run.parent_step_ids:
                self._set_run_step_parent_step(
                    child_id=step_schema.id,
                    parent_id=parent_step_id,
                    session=session,
                )

            # Save input artifact IDs into the database.
            for input_name, artifact_version_id in step_run.inputs.items():
                self._set_run_step_input_artifact(
                    run_step_id=step_schema.id,
                    artifact_version_id=artifact_version_id,
                    name=input_name,
                    input_type=StepRunInputArtifactType.DEFAULT,
                    session=session,
                )

            # Save output artifact IDs into the database.
            for output_name, artifact_version_id in step_run.outputs.items():
                self._set_run_step_output_artifact(
                    step_run_id=step_schema.id,
                    artifact_version_id=artifact_version_id,
                    name=output_name,
                    output_type=StepRunOutputArtifactType.DEFAULT,
                    session=session,
                )

            if step_run.status != ExecutionStatus.RUNNING:
                self._update_pipeline_run_status(
                    pipeline_run_id=step_run.pipeline_run_id, session=session
                )

            session.commit()

            return step_schema.to_model(include_metadata=True)

    def get_run_step(
        self, step_run_id: UUID, hydrate: bool = True
    ) -> StepRunResponse:
        """Get a step run by ID.

        Args:
            step_run_id: The ID of the step run to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The step run.

        Raises:
            KeyError: if the step run doesn't exist.
        """
        with Session(self.engine) as session:
            step_run = session.exec(
                select(StepRunSchema).where(StepRunSchema.id == step_run_id)
            ).first()
            if step_run is None:
                raise KeyError(
                    f"Unable to get step run with ID {step_run_id}: No step "
                    "run with this ID found."
                )
            return step_run.to_model(include_metadata=hydrate)

    def list_run_steps(
        self,
        step_run_filter_model: StepRunFilter,
        hydrate: bool = False,
    ) -> Page[StepRunResponse]:
        """List all step runs matching the given filter criteria.

        Args:
            step_run_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all step runs matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(StepRunSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=StepRunSchema,
                filter_model=step_run_filter_model,
                hydrate=hydrate,
            )

    def update_run_step(
        self,
        step_run_id: UUID,
        step_run_update: StepRunUpdate,
    ) -> StepRunResponse:
        """Updates a step run.

        Args:
            step_run_id: The ID of the step to update.
            step_run_update: The update to be applied to the step.

        Returns:
            The updated step run.

        Raises:
            KeyError: if the step run doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if the step exists
            existing_step_run = session.exec(
                select(StepRunSchema).where(StepRunSchema.id == step_run_id)
            ).first()
            if existing_step_run is None:
                raise KeyError(
                    f"Unable to update step with ID {step_run_id}: "
                    f"No step with this ID found."
                )

            # Update the step
            existing_step_run.update(step_run_update)
            session.add(existing_step_run)

            # Update the output artifacts.
            for name, artifact_version_id in step_run_update.outputs.items():
                self._set_run_step_output_artifact(
                    step_run_id=step_run_id,
                    artifact_version_id=artifact_version_id,
                    name=name,
                    output_type=StepRunOutputArtifactType.DEFAULT,
                    session=session,
                )

            # Update saved artifacts
            for (
                artifact_name,
                artifact_version_id,
            ) in step_run_update.saved_artifact_versions.items():
                self._set_run_step_output_artifact(
                    step_run_id=step_run_id,
                    artifact_version_id=artifact_version_id,
                    name=artifact_name,
                    output_type=StepRunOutputArtifactType.MANUAL,
                    session=session,
                )

            # Update loaded artifacts.
            for (
                artifact_name,
                artifact_version_id,
            ) in step_run_update.loaded_artifact_versions.items():
                self._set_run_step_input_artifact(
                    run_step_id=step_run_id,
                    artifact_version_id=artifact_version_id,
                    name=artifact_name,
                    input_type=StepRunInputArtifactType.MANUAL,
                    session=session,
                )

            self._update_pipeline_run_status(
                pipeline_run_id=existing_step_run.pipeline_run_id,
                session=session,
            )

            session.commit()
            session.refresh(existing_step_run)

            return existing_step_run.to_model(include_metadata=True)

    @staticmethod
    def _set_run_step_parent_step(
        child_id: UUID, parent_id: UUID, session: Session
    ) -> None:
        """Sets the parent step run for a step run.

        Args:
            child_id: The ID of the child step run to set the parent for.
            parent_id: The ID of the parent step run to set a child for.
            session: The database session to use.

        Raises:
            KeyError: if the child step run or parent step run doesn't exist.
        """
        # Check if the child step exists.
        child_step_run = session.exec(
            select(StepRunSchema).where(StepRunSchema.id == child_id)
        ).first()
        if child_step_run is None:
            raise KeyError(
                f"Unable to set parent step for step with ID "
                f"{child_id}: No step with this ID found."
            )

        # Check if the parent step exists.
        parent_step_run = session.exec(
            select(StepRunSchema).where(StepRunSchema.id == parent_id)
        ).first()
        if parent_step_run is None:
            raise KeyError(
                f"Unable to set parent step for step with ID "
                f"{child_id}: No parent step with ID {parent_id} "
                "found."
            )

        # Check if the parent step is already set.
        assignment = session.exec(
            select(StepRunParentsSchema)
            .where(StepRunParentsSchema.child_id == child_id)
            .where(StepRunParentsSchema.parent_id == parent_id)
        ).first()
        if assignment is not None:
            return

        # Save the parent step assignment in the database.
        assignment = StepRunParentsSchema(
            child_id=child_id, parent_id=parent_id
        )
        session.add(assignment)

    @staticmethod
    def _set_run_step_input_artifact(
        run_step_id: UUID,
        artifact_version_id: UUID,
        name: str,
        input_type: StepRunInputArtifactType,
        session: Session,
    ) -> None:
        """Sets an artifact as an input of a step run.

        Args:
            run_step_id: The ID of the step run.
            artifact_version_id: The ID of the artifact.
            name: The name of the input in the step run.
            input_type: In which way the artifact was loaded in the step.
            session: The database session to use.

        Raises:
            KeyError: if the step run or artifact doesn't exist.
        """
        # Check if the step exists.
        step_run = session.exec(
            select(StepRunSchema).where(StepRunSchema.id == run_step_id)
        ).first()
        if step_run is None:
            raise KeyError(
                f"Unable to set input artifact: No step run with ID "
                f"'{run_step_id}' found."
            )

        # Check if the artifact exists.
        artifact = session.exec(
            select(ArtifactVersionSchema).where(
                ArtifactVersionSchema.id == artifact_version_id
            )
        ).first()
        if artifact is None:
            raise KeyError(
                f"Unable to set input artifact: No artifact with ID "
                f"'{artifact_version_id}' found."
            )

        # Check if the input is already set.
        assignment = session.exec(
            select(StepRunInputArtifactSchema)
            .where(StepRunInputArtifactSchema.step_id == run_step_id)
            .where(
                StepRunInputArtifactSchema.artifact_id == artifact_version_id
            )
            .where(StepRunInputArtifactSchema.name == name)
        ).first()
        if assignment is not None:
            return

        # Save the input assignment in the database.
        assignment = StepRunInputArtifactSchema(
            step_id=run_step_id,
            artifact_id=artifact_version_id,
            name=name,
            type=input_type,
        )
        session.add(assignment)

    @staticmethod
    def _set_run_step_output_artifact(
        step_run_id: UUID,
        artifact_version_id: UUID,
        name: str,
        output_type: StepRunOutputArtifactType,
        session: Session,
    ) -> None:
        """Sets an artifact as an output of a step run.

        Args:
            step_run_id: The ID of the step run.
            artifact_version_id: The ID of the artifact version.
            name: The name of the output in the step run.
            output_type: In which way the artifact was saved by the step.
            session: The database session to use.

        Raises:
            KeyError: if the step run or artifact doesn't exist.
        """
        # Check if the step exists.
        step_run = session.exec(
            select(StepRunSchema).where(StepRunSchema.id == step_run_id)
        ).first()
        if step_run is None:
            raise KeyError(
                f"Unable to set output artifact: No step run with ID "
                f"'{step_run_id}' found."
            )

        # Check if the artifact exists.
        artifact = session.exec(
            select(ArtifactVersionSchema).where(
                ArtifactVersionSchema.id == artifact_version_id
            )
        ).first()
        if artifact is None:
            raise KeyError(
                f"Unable to set output artifact: No artifact with ID "
                f"'{artifact_version_id}' found."
            )

        # Check if the output is already set.
        assignment = session.exec(
            select(StepRunOutputArtifactSchema)
            .where(StepRunOutputArtifactSchema.step_id == step_run_id)
            .where(
                StepRunOutputArtifactSchema.artifact_id == artifact_version_id
            )
        ).first()
        if assignment is not None:
            return

        # Save the output assignment in the database.
        assignment = StepRunOutputArtifactSchema(
            step_id=step_run_id,
            artifact_id=artifact_version_id,
            name=name,
            type=output_type,
        )
        session.add(assignment)

    def _update_pipeline_run_status(
        self,
        pipeline_run_id: UUID,
        session: Session,
    ) -> None:
        """Updates the status of a pipeline run.

        Args:
            pipeline_run_id: The ID of the pipeline run to update.
            session: The database session to use.
        """
        from zenml.orchestrators.publish_utils import get_pipeline_run_status

        pipeline_run = session.exec(
            select(PipelineRunSchema).where(
                PipelineRunSchema.id == pipeline_run_id
            )
        ).one()
        step_runs = session.exec(
            select(StepRunSchema).where(
                StepRunSchema.pipeline_run_id == pipeline_run_id
            )
        ).all()

        # Deployment always exists for pipeline runs of newer versions
        assert pipeline_run.deployment
        num_steps = len(pipeline_run.deployment.to_model().step_configurations)
        new_status = get_pipeline_run_status(
            step_statuses=[step_run.status for step_run in step_runs],
            num_steps=num_steps,
        )

        if new_status != pipeline_run.status:
            run_update = PipelineRunUpdate(status=new_status)
            if new_status in {
                ExecutionStatus.COMPLETED,
                ExecutionStatus.FAILED,
            }:
                run_update.end_time = datetime.utcnow()
                if pipeline_run.start_time and isinstance(
                    pipeline_run.start_time, datetime
                ):
                    duration_time = (
                        run_update.end_time - pipeline_run.start_time
                    )
                    duration_seconds = duration_time.total_seconds()
                    start_time_str = pipeline_run.start_time.strftime(
                        "%Y-%m-%dT%H:%M:%S.%fZ"
                    )
                else:
                    start_time_str = None
                    duration_seconds = None

                stack = pipeline_run.deployment.stack
                assert stack
                stack_metadata = {
                    str(component.type): component.flavor
                    for component in stack.components
                }
                with track_handler(
                    AnalyticsEvent.RUN_PIPELINE_ENDED
                ) as analytics_handler:
                    analytics_handler.metadata = {
                        "pipeline_run_id": pipeline_run_id,
                        "status": new_status,
                        "num_steps": num_steps,
                        "start_time": start_time_str,
                        "end_time": run_update.end_time.strftime(
                            "%Y-%m-%dT%H:%M:%S.%fZ"
                        ),
                        "duration_seconds": duration_seconds,
                        **stack_metadata,
                    }
            pipeline_run.update(run_update)
            session.add(pipeline_run)

    # --------------------------- Triggers ---------------------------

    @track_decorator(AnalyticsEvent.CREATED_TRIGGER)
    def create_trigger(self, trigger: TriggerRequest) -> TriggerResponse:
        """Creates a new trigger.

        Args:
            trigger: Trigger to be created.

        Returns:
            The newly created trigger.
        """
        with Session(self.engine) as session:
            # Verify that the given event_source exists
            self._get_event_source(
                event_source_id=trigger.event_source_id, session=session
            )

            # Verify that the given service account exists
            self._get_account_schema(
                account_name_or_id=trigger.service_account_id,
                session=session,
                service_account=True,
            )

            # Verify that the trigger won't validate Unique
            self._fail_if_trigger_with_name_exists(
                trigger_name=trigger.name,
                workspace_id=trigger.workspace,
                session=session,
            )

            new_trigger = TriggerSchema.from_request(trigger)
            session.add(new_trigger)
            session.commit()
            session.refresh(new_trigger)

            return new_trigger.to_model(
                include_metadata=True, include_resources=True
            )

    def get_trigger(
        self, trigger_id: UUID, hydrate: bool = True
    ) -> TriggerResponse:
        """Get a trigger by its unique ID.

        Args:
            trigger_id: The ID of the trigger to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The trigger with the given ID.

        Raises:
            KeyError: if the trigger doesn't exist.
        """
        with Session(self.engine) as session:
            trigger = session.exec(
                select(TriggerSchema).where(TriggerSchema.id == trigger_id)
            ).first()

            if trigger is None:
                raise KeyError(f"Trigger with ID {trigger_id} not found.")
            return trigger.to_model(
                include_metadata=hydrate, include_resources=hydrate
            )

    def list_triggers(
        self,
        trigger_filter_model: TriggerFilter,
        hydrate: bool = False,
    ) -> Page[TriggerResponse]:
        """List all trigger matching the given filter criteria.

        Args:
            trigger_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all triggers matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(TriggerSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=TriggerSchema,
                filter_model=trigger_filter_model,
                hydrate=hydrate,
            )

    @track_decorator(AnalyticsEvent.UPDATED_TRIGGER)
    def update_trigger(
        self, trigger_id: UUID, trigger_update: TriggerUpdate
    ) -> TriggerResponse:
        """Update a trigger.

        Args:
            trigger_id: The ID of the trigger update.
            trigger_update: The update request on the trigger.

        Returns:
            The updated trigger.

        Raises:
            KeyError: if the trigger doesn't exist.
        """
        with Session(self.engine) as session:
            # Check if trigger with the domain key (name, workspace, owner)
            # already exists
            existing_trigger = session.exec(
                select(TriggerSchema).where(TriggerSchema.id == trigger_id)
            ).first()
            if existing_trigger is None:
                raise KeyError(
                    f"Unable to update trigger with id '{trigger_id}': Found no"
                    f"existing trigger with this id."
                )

            if trigger_update.service_account_id:
                # Verify that the given service account exists
                self._get_account_schema(
                    account_name_or_id=trigger_update.service_account_id,
                    session=session,
                    service_account=True,
                )

            # In case of a renaming update, make sure no trigger already exists
            # with that name
            if trigger_update.name:
                if existing_trigger.name != trigger_update.name:
                    self._fail_if_trigger_with_name_exists(
                        trigger_name=trigger_update.name,
                        workspace_id=existing_trigger.workspace.id,
                        session=session,
                    )

            existing_trigger.update(
                trigger_update=trigger_update,
            )

            session.add(existing_trigger)
            session.commit()
            session.refresh(existing_trigger)

            return existing_trigger.to_model(
                include_metadata=True, include_resources=True
            )

    def delete_trigger(self, trigger_id: UUID) -> None:
        """Delete a trigger.

        Args:
            trigger_id: The ID of the trigger to delete.

        Raises:
            KeyError: if the trigger doesn't exist.
        """
        with Session(self.engine) as session:
            try:
                trigger = session.exec(
                    select(TriggerSchema).where(TriggerSchema.id == trigger_id)
                ).one()

                if trigger is None:
                    raise KeyError(f"Trigger with ID {trigger_id} not found.")
                session.delete(trigger)
            except NoResultFound as error:
                raise KeyError from error

            session.commit()

    def _fail_if_trigger_with_name_exists(
        self,
        trigger_name: str,
        workspace_id: UUID,
        session: Session,
    ) -> None:
        """Raise an exception if a trigger with same name exists.

        Args:
            trigger_name: The Trigger name
            workspace_id: The workspace ID
            session: The Session

        Returns:
            None

        Raises:
            TriggerExistsError: If a trigger with the given name already exists.
        """
        existing_domain_trigger = session.exec(
            select(TriggerSchema)
            .where(TriggerSchema.name == trigger_name)
            .where(TriggerSchema.workspace_id == workspace_id)
        ).first()
        if existing_domain_trigger is not None:
            workspace = self._get_workspace_schema(
                workspace_name_or_id=workspace_id, session=session
            )
            raise TriggerExistsError(
                f"Unable to register trigger with name "
                f"'{trigger_name}': Found an existing trigger with the same "
                f"name in the active workspace, '{workspace.name}'."
            )
        return None

    # -------------------- Trigger Executions --------------------

    def create_trigger_execution(
        self, trigger_execution: TriggerExecutionRequest
    ) -> TriggerExecutionResponse:
        """Create a trigger execution.

        Args:
            trigger_execution: The trigger execution to create.

        Returns:
            The created trigger execution.
        """
        with Session(self.engine) as session:
            # TODO: Verify that the given trigger exists
            new_execution = TriggerExecutionSchema.from_request(
                trigger_execution
            )
            session.add(new_execution)
            session.commit()
            session.refresh(new_execution)

            return new_execution.to_model(
                include_metadata=True, include_resources=True
            )

    def get_trigger_execution(
        self,
        trigger_execution_id: UUID,
        hydrate: bool = True,
    ) -> TriggerExecutionResponse:
        """Get an trigger execution by ID.

        Args:
            trigger_execution_id: The ID of the trigger execution to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The trigger execution.

        Raises:
            KeyError: If the trigger execution doesn't exist.
        """
        with Session(self.engine) as session:
            execution = session.exec(
                select(TriggerExecutionSchema).where(
                    TriggerExecutionSchema.id == trigger_execution_id
                )
            ).first()

            if execution is None:
                raise KeyError(
                    f"Trigger execution with ID {trigger_execution_id} not found."
                )
            return execution.to_model(
                include_metadata=hydrate, include_resources=True
            )

    def list_trigger_executions(
        self,
        trigger_execution_filter_model: TriggerExecutionFilter,
        hydrate: bool = False,
    ) -> Page[TriggerExecutionResponse]:
        """List all trigger executions matching the given filter criteria.

        Args:
            trigger_execution_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all trigger executions matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(TriggerExecutionSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=TriggerExecutionSchema,
                filter_model=trigger_execution_filter_model,
                hydrate=hydrate,
            )

    def delete_trigger_execution(self, trigger_execution_id: UUID) -> None:
        """Delete a trigger execution.

        Args:
            trigger_execution_id: The ID of the trigger execution to delete.

        Raises:
            KeyError: If the trigger execution doesn't exist.
        """
        with Session(self.engine) as session:
            try:
                execution = session.exec(
                    select(TriggerExecutionSchema).where(
                        TriggerExecutionSchema.id == trigger_execution_id
                    )
                ).one()

                session.delete(execution)
                session.commit()
            except NoResultFound:
                raise KeyError(
                    f"Execution with ID {trigger_execution_id} not found."
                )

    # ----------------------------- Users -----------------------------

    @classmethod
    @lru_cache(maxsize=1)
    def _get_resource_references(
        cls,
    ) -> List[Tuple[Type[SQLModel], str]]:
        """Get a list of all other table columns that reference the user table.

        Given that this list doesn't change at runtime, we cache it.

        Returns:
            A list of all other table columns that reference the user table
            as a list of tuples of the form
            (<sqlmodel-schema-class>, <attribute-name>).
        """
        from zenml.zen_stores import schemas as zenml_schemas

        # Get a list of attributes that represent relationships to other
        # resources
        resource_attrs = [
            attr
            for attr in UserSchema.__sqlmodel_relationships__.keys()
            if not attr.startswith("_")
            and attr
            not in
            # These are not resources owned by the user or  are resources that
            # are deleted automatically when the user is deleted.
            ["api_keys", "auth_devices"]
        ]

        # This next part is crucial in preserving scalability: we don't fetch
        # the values of the relationship attributes, because this would
        # potentially load a huge amount of data into memory through
        # lazy-loading. Instead, we use a DB query to count resources
        # associated with the user for each individual resource attribute.

        # To create this query, we need a list of all tables and their foreign
        # keys that point to the user table.
        foreign_keys: List[Tuple[Type[SQLModel], str]] = []
        for resource_attr in resource_attrs:
            # Extract the target schema from the annotation
            annotation = UserSchema.__annotations__[resource_attr]

            # The annotation must be of the form
            # `typing.List[ForwardRef('<schema-class>')]`
            # We need to recover the schema class from the ForwardRef
            assert annotation._name == "List"
            assert annotation.__args__
            schema_ref = annotation.__args__[0]
            assert isinstance(schema_ref, ForwardRef)
            # We pass the zenml_schemas module as the globals dict to
            # _evaluate, because this is where the schema classes are
            # defined
            if sys.version_info < (3, 9):
                # For Python versions <3.9, leave out the third parameter to
                # _evaluate
                target_schema = schema_ref._evaluate(vars(zenml_schemas), {})
            else:
                target_schema = schema_ref._evaluate(
                    vars(zenml_schemas), {}, frozenset()
                )
            assert target_schema is not None
            assert issubclass(target_schema, SQLModel)

            # Next, we need to identify the foreign key attribute in the
            # target table
            table = UserSchema.metadata.tables[target_schema.__tablename__]
            foreign_key_attr = None
            for fk in table.foreign_keys:
                if fk.column.table.name != UserSchema.__tablename__:
                    continue
                if fk.column.name != "id":
                    continue
                assert fk.parent is not None
                foreign_key_attr = fk.parent.name
                break

            assert foreign_key_attr is not None

            foreign_keys.append((target_schema, foreign_key_attr))

        return foreign_keys

    def _account_owns_resources(
        self, account: UserSchema, session: Session
    ) -> bool:
        """Check if the account owns any resources.

        Args:
            account: The account to check.
            session: The database session to use for the query.

        Returns:
            Whether the account owns any resources.
        """
        # Get a list of all other table columns that reference the user table
        resource_attrs = self._get_resource_references()
        for schema, resource_attr in resource_attrs:
            # Check if the user owns any resources of this type
            count = session.scalar(
                select([func.count("*")])
                .select_from(schema)
                .where(getattr(schema, resource_attr) == account.id)
            )
            if count > 0:
                logger.debug(
                    f"User {account.name} owns {count} resources of type "
                    f"{schema.__tablename__}"
                )
                return True

        return False

    def create_user(self, user: UserRequest) -> UserResponse:
        """Creates a new user.

        Args:
            user: User to be created.

        Returns:
            The newly created user.

        Raises:
            EntityExistsError: If a user or service account with the given name
                already exists.
        """
        with Session(self.engine) as session:
            # Check if a user account with the given name already exists
            err_msg = (
                f"Unable to create user with name '{user.name}': "
                f"Found an existing user account with this name."
            )
            try:
                self._get_account_schema(
                    user.name,
                    session=session,
                    # Filter out service accounts
                    service_account=False,
                )
                raise EntityExistsError(err_msg)
            except KeyError:
                pass

            # Create the user
            new_user = UserSchema.from_user_request(user)
            session.add(new_user)
            # on commit an IntegrityError may arise we let it bubble up
            session.commit()
            return new_user.to_model(include_metadata=True)

    def get_user(
        self,
        user_name_or_id: Optional[Union[str, UUID]] = None,
        include_private: bool = False,
        hydrate: bool = True,
    ) -> UserResponse:
        """Gets a specific user, when no id is specified the active user is returned.

        # noqa: DAR401
        # noqa: DAR402

        Raises a KeyError in case a user with that name or id does not exist.

        For backwards-compatibility reasons, this method can also be called
        to fetch service accounts by their ID.

        Args:
            user_name_or_id: The name or ID of the user to get.
            include_private: Whether to include private user information
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested user, if it was found.

        Raises:
            KeyError: If the user does not exist.
        """
        if not user_name_or_id:
            user_name_or_id = self._default_user_name

        with Session(self.engine) as session:
            # If a UUID is passed, we also allow fetching service accounts
            # with that ID.
            service_account: Optional[bool] = False
            if uuid_utils.is_valid_uuid(user_name_or_id):
                service_account = None
            user = self._get_account_schema(
                user_name_or_id,
                session=session,
                service_account=service_account,
            )

            return user.to_model(
                include_private=include_private, include_metadata=hydrate
            )

    def get_auth_user(
        self, user_name_or_id: Union[str, UUID]
    ) -> UserAuthModel:
        """Gets the auth model to a specific user.

        Args:
            user_name_or_id: The name or ID of the user to get.

        Returns:
            The requested user, if it was found.
        """
        with Session(self.engine) as session:
            user = self._get_account_schema(
                user_name_or_id, session=session, service_account=False
            )
            return UserAuthModel(
                id=user.id,
                name=user.name,
                full_name=user.full_name,
                email_opted_in=user.email_opted_in,
                active=user.active,
                created=user.created,
                updated=user.updated,
                password=user.password,
                activation_token=user.activation_token,
                is_service_account=False,
            )

    def list_users(
        self,
        user_filter_model: UserFilter,
        hydrate: bool = False,
    ) -> Page[UserResponse]:
        """List all users.

        Args:
            user_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all users.
        """
        with Session(self.engine) as session:
            query = select(UserSchema)
            paged_user: Page[UserResponse] = self.filter_and_paginate(
                session=session,
                query=query,
                table=UserSchema,
                filter_model=user_filter_model,
                hydrate=hydrate,
            )
            return paged_user

    def update_user(
        self, user_id: UUID, user_update: UserUpdate
    ) -> UserResponse:
        """Updates an existing user.

        Args:
            user_id: The id of the user to update.
            user_update: The update to be applied to the user.

        Returns:
            The updated user.

        Raises:
            IllegalOperationError: If the request tries to update the username
                for the default user account.
            EntityExistsError: If the request tries to update the username to
                a name that is already taken by another user or service account.
        """
        with Session(self.engine) as session:
            existing_user = self._get_account_schema(
                user_id, session=session, service_account=False
            )

            if (
                user_update.name is not None
                and user_update.name != existing_user.name
            ):
                if existing_user.name == self._default_user_name:
                    raise IllegalOperationError(
                        "The username of the default user account cannot be "
                        "changed."
                    )

                try:
                    self._get_account_schema(
                        user_update.name,
                        session=session,
                        service_account=False,
                    )
                    raise EntityExistsError(
                        f"Unable to update user account with name "
                        f"'{user_update.name}': Found an existing user "
                        "account with this name."
                    )
                except KeyError:
                    pass

            existing_user.update_user(user_update=user_update)
            session.add(existing_user)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(existing_user)
            return existing_user.to_model(include_metadata=True)

    def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
        """Deletes a user.

        Args:
            user_name_or_id: The name or the ID of the user to delete.

        Raises:
            IllegalOperationError: If the user is the default user account or
                if the user already owns resources.
        """
        with Session(self.engine) as session:
            user = self._get_account_schema(
                user_name_or_id, session=session, service_account=False
            )
            if user.name == self._default_user_name:
                raise IllegalOperationError(
                    "The default user account cannot be deleted."
                )
            if self._account_owns_resources(user, session=session):
                raise IllegalOperationError(
                    "The user account has already been used to create "
                    "other resources that it now owns and therefore cannot be "
                    "deleted. Please delete all resources owned by the user "
                    "account or consider deactivating it instead."
                )

            session.delete(user)
            session.commit()

    @property
    def _default_user_name(self) -> str:
        """Get the default user name.

        Returns:
            The default user name.
        """
        return os.getenv(ENV_ZENML_DEFAULT_USER_NAME, DEFAULT_USERNAME)

    def _get_or_create_default_user(self) -> UserResponse:
        """Get or create the default user if it doesn't exist.

        Returns:
            The default user.
        """
        default_user_name = self._default_user_name
        try:
            return self.get_user(default_user_name)
        except KeyError:
            password = os.getenv(
                ENV_ZENML_DEFAULT_USER_PASSWORD, DEFAULT_PASSWORD
            )

            logger.info(f"Creating default user '{default_user_name}' ...")
            return self.create_user(
                UserRequest(
                    name=default_user_name,
                    active=True,
                    password=password,
                )
            )

    # ----------------------------- Workspaces -----------------------------

    @track_decorator(AnalyticsEvent.CREATED_WORKSPACE)
    def create_workspace(
        self, workspace: WorkspaceRequest
    ) -> WorkspaceResponse:
        """Creates a new workspace.

        Args:
            workspace: The workspace to create.

        Returns:
            The newly created workspace.

        Raises:
            EntityExistsError: If a workspace with the given name already exists.
        """
        with Session(self.engine) as session:
            # Check if workspace with the given name already exists
            existing_workspace = session.exec(
                select(WorkspaceSchema).where(
                    WorkspaceSchema.name == workspace.name
                )
            ).first()
            if existing_workspace is not None:
                raise EntityExistsError(
                    f"Unable to create workspace {workspace.name}: "
                    "A workspace with this name already exists."
                )

            # Create the workspace
            new_workspace = WorkspaceSchema.from_request(workspace)
            session.add(new_workspace)
            session.commit()

            # Explicitly refresh the new_workspace schema
            session.refresh(new_workspace)

            workspace_model = new_workspace.to_model(include_metadata=True)

        self._get_or_create_default_stack(workspace=workspace_model)
        return workspace_model

    def get_workspace(
        self, workspace_name_or_id: Union[str, UUID], hydrate: bool = True
    ) -> WorkspaceResponse:
        """Get an existing workspace by name or ID.

        Args:
            workspace_name_or_id: Name or ID of the workspace to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested workspace if one was found.
        """
        with Session(self.engine) as session:
            workspace = self._get_workspace_schema(
                workspace_name_or_id, session=session
            )
        return workspace.to_model(include_metadata=hydrate)

    def list_workspaces(
        self,
        workspace_filter_model: WorkspaceFilter,
        hydrate: bool = False,
    ) -> Page[WorkspaceResponse]:
        """List all workspace matching the given filter criteria.

        Args:
            workspace_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all workspace matching the filter criteria.
        """
        with Session(self.engine) as session:
            query = select(WorkspaceSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=WorkspaceSchema,
                filter_model=workspace_filter_model,
                hydrate=hydrate,
            )

    def update_workspace(
        self, workspace_id: UUID, workspace_update: WorkspaceUpdate
    ) -> WorkspaceResponse:
        """Update an existing workspace.

        Args:
            workspace_id: The ID of the workspace to be updated.
            workspace_update: The update to be applied to the workspace.

        Returns:
            The updated workspace.

        Raises:
            IllegalOperationError: if the workspace is the default workspace.
            KeyError: if the workspace does not exist.
        """
        with Session(self.engine) as session:
            existing_workspace = session.exec(
                select(WorkspaceSchema).where(
                    WorkspaceSchema.id == workspace_id
                )
            ).first()
            if existing_workspace is None:
                raise KeyError(
                    f"Unable to update workspace with id "
                    f"'{workspace_id}': Found no"
                    f"existing workspaces with this id."
                )
            if (
                existing_workspace.name == self._default_workspace_name
                and "name" in workspace_update.__fields_set__
                and workspace_update.name != existing_workspace.name
            ):
                raise IllegalOperationError(
                    "The name of the default workspace cannot be changed."
                )

            # Update the workspace
            existing_workspace.update(workspace_update=workspace_update)
            session.add(existing_workspace)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(existing_workspace)
            return existing_workspace.to_model(include_metadata=True)

    def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
        """Deletes a workspace.

        Args:
            workspace_name_or_id: Name or ID of the workspace to delete.

        Raises:
            IllegalOperationError: If the workspace is the default workspace.
        """
        with Session(self.engine) as session:
            # Check if workspace with the given name exists
            workspace = self._get_workspace_schema(
                workspace_name_or_id, session=session
            )
            if workspace.name == self._default_workspace_name:
                raise IllegalOperationError(
                    "The default workspace cannot be deleted."
                )

            session.delete(workspace)
            session.commit()

    def _get_or_create_default_workspace(self) -> WorkspaceResponse:
        """Get or create the default workspace if it doesn't exist.

        Returns:
            The default workspace.
        """
        default_workspace_name = self._default_workspace_name

        try:
            return self.get_workspace(default_workspace_name)
        except KeyError:
            logger.info(
                f"Creating default workspace '{default_workspace_name}' ..."
            )
            return self.create_workspace(
                WorkspaceRequest(name=default_workspace_name)
            )

    # =======================
    # Internal helper methods
    # =======================

    def _count_entity(
        self,
        schema: Type[BaseSchema],
        filter_model: Optional[BaseFilter] = None,
    ) -> int:
        """Return count of a given entity.

        Args:
            schema: Schema of the Entity
            filter_model: The filter model to filter the entity table.

        Returns:
            Count of the entity as integer.
        """
        with Session(self.engine) as session:
            query = select([func.count(schema.id)])

            if filter_model:
                query = filter_model.apply_filter(query=query, table=schema)

            entity_count = session.scalar(query)

        return int(entity_count)

    def entity_exists(
        self, entity_id: UUID, schema_class: Type[AnySchema]
    ) -> bool:
        """Check whether an entity exists in the database.

        Args:
            entity_id: The ID of the entity to check.
            schema_class: The schema class.

        Returns:
            If the entity exists.
        """
        with Session(self.engine) as session:
            schema = session.exec(
                select(schema_class.id).where(schema_class.id == entity_id)
            ).first()

            return False if schema is None else True

    def get_entity_by_id(
        self, entity_id: UUID, schema_class: Type[AnySchema]
    ) -> Optional[AnyIdentifiedResponse]:
        """Get an entity by ID.

        Args:
            entity_id: The ID of the entity to get.
            schema_class: The schema class.

        Raises:
            RuntimeError: If the schema to model conversion failed.

        Returns:
            The entity if it exists, None otherwise
        """
        with Session(self.engine) as session:
            schema = session.exec(
                select(schema_class).where(schema_class.id == entity_id)
            ).first()

            if not schema:
                return None

            to_model = getattr(schema, "to_model", None)
            if callable(to_model):
                return cast(AnyIdentifiedResponse, to_model(hydrate=True))
            else:
                raise RuntimeError("Unable to convert schema to model.")

    @staticmethod
    def _get_schema_by_name_or_id(
        object_name_or_id: Union[str, UUID],
        schema_class: Type[AnyNamedSchema],
        schema_name: str,
        session: Session,
    ) -> AnyNamedSchema:
        """Query a schema by its 'name' or 'id' field.

        Args:
            object_name_or_id: The name or ID of the object to query.
            schema_class: The schema class to query. E.g., `WorkspaceSchema`.
            schema_name: The name of the schema used for error messages.
                E.g., "workspace".
            session: The database session to use.

        Returns:
            The schema object.

        Raises:
            KeyError: if the object couldn't be found.
            ValueError: if the schema_name isn't provided.
        """
        if object_name_or_id is None:
            raise ValueError(
                f"Unable to get {schema_name}: No {schema_name} ID or name "
                "provided."
            )
        if uuid_utils.is_valid_uuid(object_name_or_id):
            filter_params = schema_class.id == object_name_or_id
            error_msg = (
                f"Unable to get {schema_name} with name or ID "
                f"'{object_name_or_id}': No {schema_name} with this ID found."
            )
        else:
            filter_params = schema_class.name == object_name_or_id
            error_msg = (
                f"Unable to get {schema_name} with name or ID "
                f"'{object_name_or_id}': '{object_name_or_id}' is not a valid "
                f" UUID and no {schema_name} with this name exists."
            )

        schema = session.exec(
            select(schema_class).where(filter_params)
        ).first()

        if schema is None:
            raise KeyError(error_msg)
        return schema

    def _get_workspace_schema(
        self,
        workspace_name_or_id: Union[str, UUID],
        session: Session,
    ) -> WorkspaceSchema:
        """Gets a workspace schema by name or ID.

        This is a helper method that is used in various places to find the
        workspace associated to some other object.

        Args:
            workspace_name_or_id: The name or ID of the workspace to get.
            session: The database session to use.

        Returns:
            The workspace schema.
        """
        return self._get_schema_by_name_or_id(
            object_name_or_id=workspace_name_or_id,
            schema_class=WorkspaceSchema,
            schema_name="workspace",
            session=session,
        )

    def _get_account_schema(
        self,
        account_name_or_id: Union[str, UUID],
        session: Session,
        service_account: Optional[bool] = None,
    ) -> UserSchema:
        """Gets a user account or a service account schema by name or ID.

        This helper method is used to fetch both user accounts and service
        accounts by name or ID. It is required because in the DB, user accounts
        and service accounts are stored using the same UserSchema to make
        it easier to implement resource ownership.

        Args:
            account_name_or_id: The name or ID of the account to get.
            session: The database session to use.
            service_account: Whether to get a service account or a user
                account. If None, both are considered with a priority for
                user accounts if both exist (e.g. with the same name).

        Returns:
            The account schema.

        Raises:
            KeyError: If no account with the given name or ID exists.
        """
        account_type = ""
        query = select(UserSchema)
        if uuid_utils.is_valid_uuid(account_name_or_id):
            query = query.where(UserSchema.id == account_name_or_id)
        else:
            query = query.where(UserSchema.name == account_name_or_id)
        if service_account is not None:
            if service_account is True:
                account_type = "service "
            elif service_account is False:
                account_type = "user "
            query = query.where(
                UserSchema.is_service_account == service_account  # noqa: E712
            )
        error_msg = (
            f"No {account_type}account with the '{account_name_or_id}' name "
            "or ID was found"
        )

        results = session.exec(query).all()

        if len(results) == 0:
            raise KeyError(error_msg)

        if len(results) == 1:
            return results[0]

        # We could have two results if a service account and a user account
        # have the same name. In that case, we return the user account.
        for result in results:
            if not result.is_service_account:
                return result

        raise KeyError(error_msg)

    def _get_run_schema(
        self,
        run_name_or_id: Union[str, UUID],
        session: Session,
    ) -> PipelineRunSchema:
        """Gets a run schema by name or ID.

        This is a helper method that is used in various places to find a run
        by its name or ID.

        Args:
            run_name_or_id: The name or ID of the run to get.
            session: The database session to use.

        Returns:
            The run schema.
        """
        return self._get_schema_by_name_or_id(
            object_name_or_id=run_name_or_id,
            schema_class=PipelineRunSchema,
            schema_name="run",
            session=session,
        )

    def _get_model_schema(
        self,
        model_name_or_id: Union[str, UUID],
        session: Session,
    ) -> ModelSchema:
        """Gets a model schema by name or ID.

        This is a helper method that is used in various places to find a model
        by its name or ID.

        Args:
            model_name_or_id: The name or ID of the model to get.
            session: The database session to use.

        Returns:
            The model schema.
        """
        return self._get_schema_by_name_or_id(
            object_name_or_id=model_name_or_id,
            schema_class=ModelSchema,
            schema_name="model",
            session=session,
        )

    def _get_tag_schema(
        self,
        tag_name_or_id: Union[str, UUID],
        session: Session,
    ) -> TagSchema:
        """Gets a tag schema by name or ID.

        This is a helper method that is used in various places to find a tag
        by its name or ID.

        Args:
            tag_name_or_id: The name or ID of the tag to get.
            session: The database session to use.

        Returns:
            The tag schema.
        """
        return self._get_schema_by_name_or_id(
            object_name_or_id=tag_name_or_id,
            schema_class=TagSchema,
            schema_name=TagSchema.__tablename__,
            session=session,
        )

    def _get_tag_model_schema(
        self,
        tag_id: UUID,
        resource_id: UUID,
        resource_type: TaggableResourceTypes,
        session: Session,
    ) -> TagResourceSchema:
        """Gets a tag model schema by tag and resource.

        Args:
            tag_id: The ID of the tag to get.
            resource_id: The ID of the resource to get.
            resource_type: The type of the resource to get.
            session: The database session to use.

        Returns:
            The tag resource schema.

        Raises:
            KeyError: if entity not found.
        """
        with Session(self.engine) as session:
            schema = session.exec(
                select(TagResourceSchema).where(
                    TagResourceSchema.tag_id == tag_id,
                    TagResourceSchema.resource_id == resource_id,
                    TagResourceSchema.resource_type == resource_type.value,
                )
            ).first()
            if schema is None:
                raise KeyError(
                    f"Unable to get {TagResourceSchema.__tablename__} with IDs "
                    f"`tag_id`='{tag_id}' and `resource_id`='{resource_id}' and "
                    f"`resource_type`='{resource_type.value}': No "
                    f"{TagResourceSchema.__tablename__} with these IDs found."
                )
            return schema

    @staticmethod
    def _create_or_reuse_code_reference(
        session: Session,
        workspace_id: UUID,
        code_reference: Optional["CodeReferenceRequest"],
    ) -> Optional[UUID]:
        """Creates or reuses a code reference.

        Args:
            session: The database session to use.
            workspace_id: ID of the workspace in which the code reference
                should be.
            code_reference: Request of the reference to create.

        Returns:
            The code reference ID.
        """
        if not code_reference:
            return None

        existing_reference = session.exec(
            select(CodeReferenceSchema)
            .where(CodeReferenceSchema.workspace_id == workspace_id)
            .where(
                CodeReferenceSchema.code_repository_id
                == code_reference.code_repository
            )
            .where(CodeReferenceSchema.commit == code_reference.commit)
            .where(
                CodeReferenceSchema.subdirectory == code_reference.subdirectory
            )
        ).first()
        if existing_reference is not None:
            return existing_reference.id

        new_reference = CodeReferenceSchema.from_request(
            code_reference, workspace_id=workspace_id
        )

        session.add(new_reference)
        return new_reference.id

    # ----------------------------- Models -----------------------------

    @track_decorator(AnalyticsEvent.CREATED_MODEL)
    def create_model(self, model: ModelRequest) -> ModelResponse:
        """Creates a new model.

        Args:
            model: the Model to be created.

        Returns:
            The newly created model.

        Raises:
            EntityExistsError: If a workspace with the given name already exists.
        """
        with Session(self.engine) as session:
            existing_model = session.exec(
                select(ModelSchema).where(ModelSchema.name == model.name)
            ).first()
            if existing_model is not None:
                raise EntityExistsError(
                    f"Unable to create model {model.name}: "
                    "A model with this name already exists."
                )

            model_schema = ModelSchema.from_request(model)
            session.add(model_schema)

            if model.tags:
                self._attach_tags_to_resource(
                    tag_names=model.tags,
                    resource_id=model_schema.id,
                    resource_type=TaggableResourceTypes.MODEL,
                )
            session.commit()
            return model_schema.to_model(include_metadata=True)

    def get_model(
        self,
        model_name_or_id: Union[str, UUID],
        hydrate: bool = True,
    ) -> ModelResponse:
        """Get an existing model.

        Args:
            model_name_or_id: name or id of the model to be retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Raises:
            KeyError: specified ID or name not found.

        Returns:
            The model of interest.
        """
        with Session(self.engine) as session:
            model = self._get_model_schema(
                model_name_or_id=model_name_or_id, session=session
            )
            if model is None:
                raise KeyError(
                    f"Unable to get model with ID `{model_name_or_id}`: "
                    f"No model with this ID found."
                )
            return model.to_model(include_metadata=hydrate)

    def list_models(
        self,
        model_filter_model: ModelFilter,
        hydrate: bool = False,
    ) -> Page[ModelResponse]:
        """Get all models by filter.

        Args:
            model_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all models.
        """
        with Session(self.engine) as session:
            query = select(ModelSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=ModelSchema,
                filter_model=model_filter_model,
                hydrate=hydrate,
            )

    def delete_model(self, model_name_or_id: Union[str, UUID]) -> None:
        """Deletes a model.

        Args:
            model_name_or_id: name or id of the model to be deleted.

        Raises:
            KeyError: specified ID or name not found.
        """
        with Session(self.engine) as session:
            model = self._get_model_schema(
                model_name_or_id=model_name_or_id, session=session
            )
            if model is None:
                raise KeyError(
                    f"Unable to delete model with ID `{model_name_or_id}`: "
                    f"No model with this ID found."
                )
            session.delete(model)
            session.commit()

    def update_model(
        self,
        model_id: UUID,
        model_update: ModelUpdate,
    ) -> ModelResponse:
        """Updates an existing model.

        Args:
            model_id: UUID of the model to be updated.
            model_update: the Model to be updated.

        Raises:
            KeyError: specified ID not found.

        Returns:
            The updated model.
        """
        with Session(self.engine) as session:
            existing_model = session.exec(
                select(ModelSchema).where(ModelSchema.id == model_id)
            ).first()

            if not existing_model:
                raise KeyError(f"Model with ID {model_id} not found.")

            if model_update.add_tags:
                self._attach_tags_to_resource(
                    tag_names=model_update.add_tags,
                    resource_id=existing_model.id,
                    resource_type=TaggableResourceTypes.MODEL,
                )
            model_update.add_tags = None
            if model_update.remove_tags:
                self._detach_tags_from_resource(
                    tag_names=model_update.remove_tags,
                    resource_id=existing_model.id,
                    resource_type=TaggableResourceTypes.MODEL,
                )
            model_update.remove_tags = None

            existing_model.update(model_update=model_update)

            session.add(existing_model)
            session.commit()

            # Refresh the Model that was just created
            session.refresh(existing_model)
            return existing_model.to_model(include_metadata=True)

    # ----------------------------- Model Versions -----------------------------

    @track_decorator(AnalyticsEvent.CREATED_MODEL_VERSION)
    def create_model_version(
        self, model_version: ModelVersionRequest
    ) -> ModelVersionResponse:
        """Creates a new model version.

        Args:
            model_version: the Model Version to be created.

        Returns:
            The newly created model version.

        Raises:
            ValueError: If `number` is not None during model version creation.
            EntityExistsError: If a workspace with the given name already exists.
        """
        if model_version.number is not None:
            raise ValueError(
                "`number` field  must be None during model version creation."
            )
        with Session(self.engine) as session:
            model_version_ = model_version.copy()
            model = self.get_model(model_version_.model)

            def _check(tolerance: int = 0) -> None:
                query = session.exec(
                    select(ModelVersionSchema)
                    .where(ModelVersionSchema.model_id == model.id)
                    .where(ModelVersionSchema.name == model_version_.name)
                )
                existing_model_version = query.fetchmany(tolerance + 1)
                if (
                    existing_model_version is not None
                    and len(existing_model_version) > tolerance
                ):
                    raise EntityExistsError(
                        f"Unable to create model version {model_version_.name}: "
                        f"A model version with this name already exists in {model.name} model."
                    )

            _check()
            all_versions = session.exec(
                select(ModelVersionSchema)
                .where(ModelVersionSchema.model_id == model.id)
                .order_by(ModelVersionSchema.number.desc())  # type: ignore[attr-defined]
            ).first()

            model_version_.number = (
                all_versions.number + 1 if all_versions else 1
            )

            if model_version_.name is None:
                model_version_.name = str(model_version_.number)

            model_version_schema = ModelVersionSchema.from_request(
                model_version_
            )
            session.add(model_version_schema)

            if model_version_.tags:
                self._attach_tags_to_resource(
                    tag_names=model_version_.tags,
                    resource_id=model_version_schema.id,
                    resource_type=TaggableResourceTypes.MODEL_VERSION,
                )
            try:
                _check(1)
                session.commit()
            except EntityExistsError as e:
                session.rollback()
                raise e

            return model_version_schema.to_model(include_metadata=True)

    def get_model_version(
        self, model_version_id: UUID, hydrate: bool = True
    ) -> ModelVersionResponse:
        """Get an existing model version.

        Args:
            model_version_id: name, id, stage or number of the model version to
                be retrieved. If skipped - latest is retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The model version of interest.

        Raises:
            KeyError: specified ID or name not found.
        """
        with Session(self.engine) as session:
            model_version = self._get_schema_by_name_or_id(
                object_name_or_id=model_version_id,
                schema_class=ModelVersionSchema,
                schema_name="model_version",
                session=session,
            )
            if model_version is None:
                raise KeyError(
                    f"Unable to get model version with ID "
                    f"`{model_version_id}`: No model version with this "
                    f"ID found."
                )
            return model_version.to_model(include_metadata=hydrate)

    def list_model_versions(
        self,
        model_version_filter_model: ModelVersionFilter,
        model_name_or_id: Optional[Union[str, UUID]] = None,
        hydrate: bool = False,
    ) -> Page[ModelVersionResponse]:
        """Get all model versions by filter.

        Args:
            model_name_or_id: name or id of the model containing the model
                versions.
            model_version_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model versions.
        """
        with Session(self.engine) as session:
            if model_name_or_id:
                model = self.get_model(model_name_or_id)
                model_version_filter_model.set_scope_model(model.id)

            query = select(ModelVersionSchema)

            return self.filter_and_paginate(
                session=session,
                query=query,
                table=ModelVersionSchema,
                filter_model=model_version_filter_model,
                hydrate=hydrate,
            )

    def delete_model_version(
        self,
        model_version_id: UUID,
    ) -> None:
        """Deletes a model version.

        Args:
            model_version_id: name or id of the model version to be deleted.

        Raises:
            KeyError: specified ID or name not found.
        """
        with Session(self.engine) as session:
            query = select(ModelVersionSchema).where(
                ModelVersionSchema.id == model_version_id
            )
            model_version = session.exec(query).first()
            if model_version is None:
                raise KeyError(
                    "Unable to delete model version with id "
                    f"`{model_version_id}`: "
                    "No model version with this id found."
                )
            session.delete(model_version)
            session.commit()

    def update_model_version(
        self,
        model_version_id: UUID,
        model_version_update_model: ModelVersionUpdate,
    ) -> ModelVersionResponse:
        """Get all model versions by filter.

        Args:
            model_version_id: The ID of model version to be updated.
            model_version_update_model: The model version to be updated.

        Returns:
            An updated model version.

        Raises:
            KeyError: If the model version not found
            RuntimeError: If there is a model version with target stage,
                but `force` flag is off
        """
        with Session(self.engine) as session:
            existing_model_version = session.exec(
                select(ModelVersionSchema)
                .where(
                    ModelVersionSchema.model_id
                    == model_version_update_model.model
                )
                .where(ModelVersionSchema.id == model_version_id)
            ).first()

            if not existing_model_version:
                raise KeyError(f"Model version {model_version_id} not found.")

            stage = None
            if (stage_ := model_version_update_model.stage) is not None:
                stage = getattr(stage_, "value", stage_)

                existing_model_version_in_target_stage = session.exec(
                    select(ModelVersionSchema)
                    .where(
                        ModelVersionSchema.model_id
                        == model_version_update_model.model
                    )
                    .where(ModelVersionSchema.stage == stage)
                ).first()

                if (
                    existing_model_version_in_target_stage is not None
                    and existing_model_version_in_target_stage.id
                    != existing_model_version.id
                ):
                    if not model_version_update_model.force:
                        raise RuntimeError(
                            f"Model version {existing_model_version_in_target_stage.name} is "
                            f"in {stage}, but `force` flag is False."
                        )
                    else:
                        existing_model_version_in_target_stage.update(
                            target_stage=ModelStages.ARCHIVED.value
                        )
                        session.add(existing_model_version_in_target_stage)

                        logger.info(
                            f"Model version {existing_model_version_in_target_stage.name} has been set to {ModelStages.ARCHIVED.value}."
                        )

            if model_version_update_model.add_tags:
                self._attach_tags_to_resource(
                    tag_names=model_version_update_model.add_tags,
                    resource_id=existing_model_version.id,
                    resource_type=TaggableResourceTypes.MODEL_VERSION,
                )
            if model_version_update_model.remove_tags:
                self._detach_tags_from_resource(
                    tag_names=model_version_update_model.remove_tags,
                    resource_id=existing_model_version.id,
                    resource_type=TaggableResourceTypes.MODEL_VERSION,
                )

            existing_model_version.update(
                target_stage=stage,
                target_name=model_version_update_model.name,
                target_description=model_version_update_model.description,
            )
            session.add(existing_model_version)
            session.commit()
            session.refresh(existing_model_version)

            return existing_model_version.to_model(include_metadata=True)

    # ------------------------ Model Versions Artifacts ------------------------

    def create_model_version_artifact_link(
        self, model_version_artifact_link: ModelVersionArtifactRequest
    ) -> ModelVersionArtifactResponse:
        """Creates a new model version link.

        Args:
            model_version_artifact_link: the Model Version to Artifact Link
                to be created.

        Returns:
            The newly created model version to artifact link.
        """
        with Session(self.engine) as session:
            # If the link already exists, return it
            existing_model_version_artifact_link = session.exec(
                select(ModelVersionArtifactSchema)
                .where(
                    ModelVersionArtifactSchema.model_version_id
                    == model_version_artifact_link.model_version
                )
                .where(
                    ModelVersionArtifactSchema.artifact_version_id
                    == model_version_artifact_link.artifact_version,
                )
            ).first()
            if existing_model_version_artifact_link is not None:
                return existing_model_version_artifact_link.to_model()

            model_version_artifact_link_schema = (
                ModelVersionArtifactSchema.from_request(
                    model_version_artifact_request=model_version_artifact_link,
                )
            )
            session.add(model_version_artifact_link_schema)
            session.commit()
            return model_version_artifact_link_schema.to_model(
                include_metadata=True
            )

    def list_model_version_artifact_links(
        self,
        model_version_artifact_link_filter_model: ModelVersionArtifactFilter,
        hydrate: bool = False,
    ) -> Page[ModelVersionArtifactResponse]:
        """Get all model version to artifact links by filter.

        Args:
            model_version_artifact_link_filter_model: All filter parameters
                including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model version to artifact links.
        """
        with Session(self.engine) as session:
            query = select(ModelVersionArtifactSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=ModelVersionArtifactSchema,
                filter_model=model_version_artifact_link_filter_model,
                hydrate=hydrate,
            )

    def delete_model_version_artifact_link(
        self,
        model_version_id: UUID,
        model_version_artifact_link_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a model version to artifact link.

        Args:
            model_version_id: ID of the model version containing the link.
            model_version_artifact_link_name_or_id: name or ID of the model
                version to artifact link to be deleted.

        Raises:
            KeyError: specified ID or name not found.
        """
        with Session(self.engine) as session:
            model_version = self.get_model_version(model_version_id)
            query = select(ModelVersionArtifactSchema).where(
                ModelVersionArtifactSchema.model_version_id == model_version.id
            )
            try:
                UUID(str(model_version_artifact_link_name_or_id))
                query = query.where(
                    ModelVersionArtifactSchema.id
                    == model_version_artifact_link_name_or_id
                )
            except ValueError:
                query = (
                    query.where(
                        ModelVersionArtifactSchema.artifact_version_id
                        == ArtifactVersionSchema.id
                    )
                    .where(
                        ArtifactVersionSchema.artifact_id == ArtifactSchema.id
                    )
                    .where(
                        ArtifactSchema.name
                        == model_version_artifact_link_name_or_id
                    )
                )

            model_version_artifact_link = session.exec(query).first()
            if model_version_artifact_link is None:
                raise KeyError(
                    f"Unable to delete model version link with name or ID "
                    f"`{model_version_artifact_link_name_or_id}`: "
                    f"No model version link with this name found."
                )

            session.delete(model_version_artifact_link)
            session.commit()

    def delete_all_model_version_artifact_links(
        self,
        model_version_id: UUID,
        only_links: bool = True,
    ) -> None:
        """Deletes all model version to artifact links.

        Args:
            model_version_id: ID of the model version containing the link.
            only_links: Whether to only delete the link to the artifact.
        """
        with Session(self.engine) as session:
            if not only_links:
                artifact_version_ids = session.execute(
                    select(
                        ModelVersionArtifactSchema.artifact_version_id
                    ).where(
                        ModelVersionArtifactSchema.model_version_id
                        == model_version_id
                    )
                ).fetchall()
                session.execute(
                    delete(ArtifactVersionSchema).where(
                        col(ArtifactVersionSchema.id).in_(
                            [a[0] for a in artifact_version_ids]
                        )
                    ),
                )
            session.execute(
                delete(ModelVersionArtifactSchema).where(
                    ModelVersionArtifactSchema.model_version_id
                    == model_version_id
                )
            )

            session.commit()

    # ---------------------- Model Versions Pipeline Runs ----------------------

    def create_model_version_pipeline_run_link(
        self,
        model_version_pipeline_run_link: ModelVersionPipelineRunRequest,
    ) -> ModelVersionPipelineRunResponse:
        """Creates a new model version to pipeline run link.

        Args:
            model_version_pipeline_run_link: the Model Version to Pipeline Run
                Link to be created.

        Returns:
            - If Model Version to Pipeline Run Link already exists - returns
                the existing link.
            - Otherwise, returns the newly created model version to pipeline
                run link.
        """
        with Session(self.engine) as session:
            # If the link already exists, return it
            existing_model_version_pipeline_run_link = session.exec(
                select(ModelVersionPipelineRunSchema)
                .where(
                    ModelVersionPipelineRunSchema.model_version_id
                    == model_version_pipeline_run_link.model_version
                )
                .where(
                    ModelVersionPipelineRunSchema.pipeline_run_id
                    == model_version_pipeline_run_link.pipeline_run,
                )
            ).first()
            if existing_model_version_pipeline_run_link is not None:
                return existing_model_version_pipeline_run_link.to_model()

            # Otherwise, create a new link
            model_version_pipeline_run_link_schema = (
                ModelVersionPipelineRunSchema.from_request(
                    model_version_pipeline_run_link
                )
            )
            session.add(model_version_pipeline_run_link_schema)
            session.commit()
            return model_version_pipeline_run_link_schema.to_model(
                include_metadata=True
            )

    def list_model_version_pipeline_run_links(
        self,
        model_version_pipeline_run_link_filter_model: ModelVersionPipelineRunFilter,
        hydrate: bool = False,
    ) -> Page[ModelVersionPipelineRunResponse]:
        """Get all model version to pipeline run links by filter.

        Args:
            model_version_pipeline_run_link_filter_model: All filter parameters
                including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model version to pipeline run links.
        """
        query = select(ModelVersionPipelineRunSchema)
        with Session(self.engine) as session:
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=ModelVersionPipelineRunSchema,
                filter_model=model_version_pipeline_run_link_filter_model,
                hydrate=hydrate,
            )

    def delete_model_version_pipeline_run_link(
        self,
        model_version_id: UUID,
        model_version_pipeline_run_link_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a model version to pipeline run link.

        Args:
            model_version_id: name or ID of the model version containing the
                link.
            model_version_pipeline_run_link_name_or_id: name or ID of the model
                version to pipeline run link to be deleted.

        Raises:
            KeyError: specified ID not found.
        """
        with Session(self.engine) as session:
            model_version = self.get_model_version(
                model_version_id=model_version_id
            )
            query = select(ModelVersionPipelineRunSchema).where(
                ModelVersionPipelineRunSchema.model_version_id
                == model_version.id
            )
            try:
                UUID(str(model_version_pipeline_run_link_name_or_id))
                query = query.where(
                    ModelVersionPipelineRunSchema.id
                    == model_version_pipeline_run_link_name_or_id
                )
            except ValueError:
                query = query.where(
                    ModelVersionPipelineRunSchema.pipeline_run_id
                    == PipelineRunSchema.id
                ).where(
                    PipelineRunSchema.name
                    == model_version_pipeline_run_link_name_or_id
                )

            model_version_pipeline_run_link = session.exec(query).first()
            if model_version_pipeline_run_link is None:
                raise KeyError(
                    f"Unable to delete model version link with name "
                    f"`{model_version_pipeline_run_link_name_or_id}`: "
                    f"No model version link with this name found."
                )

            session.delete(model_version_pipeline_run_link)
            session.commit()

    #################
    # Tags
    #################

    def _attach_tags_to_resource(
        self,
        tag_names: List[str],
        resource_id: UUID,
        resource_type: TaggableResourceTypes,
    ) -> None:
        """Creates a tag<>resource link if not present.

        Args:
            tag_names: The list of names of the tags.
            resource_id: The id of the resource.
            resource_type: The type of the resource to create link with.
        """
        for tag_name in tag_names:
            try:
                tag = self.get_tag(tag_name)
            except KeyError:
                tag = self.create_tag(TagRequest(name=tag_name))
            try:
                self.create_tag_resource(
                    TagResourceRequest(
                        tag_id=tag.id,
                        resource_id=resource_id,
                        resource_type=resource_type,
                    )
                )
            except EntityExistsError:
                pass

    def _detach_tags_from_resource(
        self,
        tag_names: List[str],
        resource_id: UUID,
        resource_type: TaggableResourceTypes,
    ) -> None:
        """Deletes tag<>resource link if present.

        Args:
            tag_names: The list of names of the tags.
            resource_id: The id of the resource.
            resource_type: The type of the resource to create link with.
        """
        for tag_name in tag_names:
            try:
                tag = self.get_tag(tag_name)
                self.delete_tag_resource(
                    tag_id=tag.id,
                    resource_id=resource_id,
                    resource_type=resource_type,
                )
            except KeyError:
                pass

    @track_decorator(AnalyticsEvent.CREATED_TAG)
    def create_tag(self, tag: TagRequest) -> TagResponse:
        """Creates a new tag.

        Args:
            tag: the tag to be created.

        Returns:
            The newly created tag.

        Raises:
            EntityExistsError: If a tag with the given name already exists.
        """
        with Session(self.engine) as session:
            existing_tag = session.exec(
                select(TagSchema).where(TagSchema.name == tag.name)
            ).first()
            if existing_tag is not None:
                raise EntityExistsError(
                    f"Unable to create tag {tag.name}: "
                    "A tag with this name already exists."
                )

            tag_schema = TagSchema.from_request(tag)
            session.add(tag_schema)

            session.commit()
            return tag_schema.to_model(include_metadata=True)

    def delete_tag(
        self,
        tag_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a tag.

        Args:
            tag_name_or_id: name or id of the tag to delete.

        Raises:
            KeyError: specified ID or name not found.
        """
        with Session(self.engine) as session:
            tag = self._get_tag_schema(
                tag_name_or_id=tag_name_or_id, session=session
            )
            if tag is None:
                raise KeyError(
                    f"Unable to delete tag with ID `{tag_name_or_id}`: "
                    f"No tag with this ID found."
                )
            session.delete(tag)
            session.commit()

    def get_tag(
        self, tag_name_or_id: Union[str, UUID], hydrate: bool = True
    ) -> TagResponse:
        """Get an existing tag.

        Args:
            tag_name_or_id: name or id of the tag to be retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The tag of interest.

        Raises:
            KeyError: specified ID or name not found.
        """
        with Session(self.engine) as session:
            tag = self._get_tag_schema(
                tag_name_or_id=tag_name_or_id, session=session
            )
            if tag is None:
                raise KeyError(
                    f"Unable to get tag with ID `{tag_name_or_id}`: "
                    f"No tag with this ID found."
                )
            return tag.to_model(include_metadata=hydrate)

    def list_tags(
        self,
        tag_filter_model: TagFilter,
        hydrate: bool = False,
    ) -> Page[TagResponse]:
        """Get all tags by filter.

        Args:
            tag_filter_model: All filter parameters including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all tags.
        """
        with Session(self.engine) as session:
            query = select(TagSchema)
            return self.filter_and_paginate(
                session=session,
                query=query,
                table=TagSchema,
                filter_model=tag_filter_model,
                hydrate=hydrate,
            )

    def update_tag(
        self,
        tag_name_or_id: Union[str, UUID],
        tag_update_model: TagUpdate,
    ) -> TagResponse:
        """Update tag.

        Args:
            tag_name_or_id: name or id of the tag to be updated.
            tag_update_model: Tag to use for the update.

        Returns:
            An updated tag.

        Raises:
            KeyError: If the tag is not found
        """
        with Session(self.engine) as session:
            tag = self._get_tag_schema(
                tag_name_or_id=tag_name_or_id, session=session
            )

            if not tag:
                raise KeyError(f"Tag with ID `{tag_name_or_id}` not found.")

            tag.update(update=tag_update_model)
            session.add(tag)
            session.commit()

            # Refresh the tag that was just created
            session.refresh(tag)
            return tag.to_model(include_metadata=True)

    ####################
    # Tags <> resources
    ####################

    def create_tag_resource(
        self, tag_resource: TagResourceRequest
    ) -> TagResourceResponse:
        """Creates a new tag resource relationship.

        Args:
            tag_resource: the tag resource relationship to be created.

        Returns:
            The newly created tag resource relationship.

        Raises:
            EntityExistsError: If a tag resource relationship with the given
                configuration already exists.
        """
        with Session(self.engine) as session:
            existing_tag_resource = session.exec(
                select(TagResourceSchema).where(
                    TagResourceSchema.tag_id == tag_resource.tag_id,
                    TagResourceSchema.resource_id == tag_resource.resource_id,
                    TagResourceSchema.resource_type
                    == tag_resource.resource_type.value,
                )
            ).first()
            if existing_tag_resource is not None:
                raise EntityExistsError(
                    f"Unable to create a tag "
                    f"{tag_resource.resource_type.name.lower()} "
                    f"relationship with IDs "
                    f"`{tag_resource.tag_id}`|`{tag_resource.resource_id}`. "
                    "This relationship already exists."
                )

            tag_resource_schema = TagResourceSchema.from_request(tag_resource)
            session.add(tag_resource_schema)

            session.commit()
            return tag_resource_schema.to_model(include_metadata=True)

    def delete_tag_resource(
        self,
        tag_id: UUID,
        resource_id: UUID,
        resource_type: TaggableResourceTypes,
    ) -> None:
        """Deletes a tag resource relationship.

        Args:
            tag_id: The ID of the tag to delete.
            resource_id: The ID of the resource to delete.
            resource_type: The type of the resource to delete.

        Raises:
            KeyError: specified ID not found.
        """
        with Session(self.engine) as session:
            tag_model = self._get_tag_model_schema(
                tag_id=tag_id,
                resource_id=resource_id,
                resource_type=resource_type,
                session=session,
            )
            if tag_model is None:
                raise KeyError(
                    f"Unable to delete tag<>resource with IDs: "
                    f"`tag_id`='{tag_id}' and `resource_id`='{resource_id}' "
                    f"and `resource_type`='{resource_type.value}': No "
                    "tag<>resource with these IDs found."
                )
            session.delete(tag_model)
            session.commit()
alembic: Alembic property readonly

The Alembic wrapper.

Returns:

Type Description
Alembic

The Alembic wrapper.

Exceptions:

Type Description
ValueError

If the store is not initialized.

backup_secrets_store: Optional[BaseSecretsStore] property readonly

The backup secrets store associated with this store.

Returns:

Type Description
Optional[BaseSecretsStore]

The backup secrets store associated with this store.

engine: Engine property readonly

The SQLAlchemy engine.

Returns:

Type Description
Engine

The SQLAlchemy engine.

Exceptions:

Type Description
ValueError

If the store is not initialized.

migration_utils: MigrationUtils property readonly

The migration utils.

Returns:

Type Description
MigrationUtils

The migration utils.

Exceptions:

Type Description
ValueError

If the store is not initialized.

secrets_store: BaseSecretsStore property readonly

The secrets store associated with this store.

Returns:

Type Description
BaseSecretsStore

The secrets store associated with this store.

Exceptions:

Type Description
SecretsStoreNotConfiguredError

If no secrets store is configured.

CONFIG_TYPE (StoreConfiguration) pydantic-model

SQL ZenML store configuration.

Attributes:

Name Type Description
type StoreType

The type of the store.

secrets_store Optional[zenml.config.secrets_store_config.SecretsStoreConfiguration]

The configuration of the secrets store to use. This defaults to a SQL secrets store that extends the SQL ZenML store.

backup_secrets_store Optional[zenml.config.secrets_store_config.SecretsStoreConfiguration]

The configuration of a backup secrets store to use in addition to the primary one as an intermediate step during the migration to a new secrets store.

driver Optional[zenml.zen_stores.sql_zen_store.SQLDatabaseDriver]

The SQL database driver.

database Optional[str]

database name. If not already present on the server, it will be created automatically on first access.

username Optional[str]

The database username.

password Optional[str]

The database password.

ssl_ca Optional[str]

certificate authority certificate. Required for SSL enabled authentication if the CA certificate is not part of the certificates shipped by the operating system.

ssl_cert Optional[str]

client certificate. Required for SSL enabled authentication if client certificates are used.

ssl_key Optional[str]

client certificate private key. Required for SSL enabled if client certificates are used.

ssl_verify_server_cert bool

set to verify the identity of the server against the provided server certificate.

pool_size int

The maximum number of connections to keep in the SQLAlchemy pool.

max_overflow int

The maximum number of connections to allow in the SQLAlchemy pool in addition to the pool_size.

pool_pre_ping bool

Enable emitting a test statement on the SQL connection at the start of each connection pool checkout, to test that the database connection is still viable.

Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStoreConfiguration(StoreConfiguration):
    """SQL ZenML store configuration.

    Attributes:
        type: The type of the store.
        secrets_store: The configuration of the secrets store to use.
            This defaults to a SQL secrets store that extends the SQL ZenML
            store.
        backup_secrets_store: The configuration of a backup secrets store to
            use in addition to the primary one as an intermediate step during
            the migration to a new secrets store.
        driver: The SQL database driver.
        database: database name. If not already present on the server, it will
            be created automatically on first access.
        username: The database username.
        password: The database password.
        ssl_ca: certificate authority certificate. Required for SSL
            enabled authentication if the CA certificate is not part of the
            certificates shipped by the operating system.
        ssl_cert: client certificate. Required for SSL enabled
            authentication if client certificates are used.
        ssl_key: client certificate private key. Required for SSL
            enabled if client certificates are used.
        ssl_verify_server_cert: set to verify the identity of the server
            against the provided server certificate.
        pool_size: The maximum number of connections to keep in the SQLAlchemy
            pool.
        max_overflow: The maximum number of connections to allow in the
            SQLAlchemy pool in addition to the pool_size.
        pool_pre_ping: Enable emitting a test statement on the SQL connection
            at the start of each connection pool checkout, to test that the
            database connection is still viable.
    """

    type: StoreType = StoreType.SQL

    secrets_store: Optional[SecretsStoreConfiguration] = None
    backup_secrets_store: Optional[SecretsStoreConfiguration] = None

    driver: Optional[SQLDatabaseDriver] = None
    database: Optional[str] = None
    username: Optional[str] = None
    password: Optional[str] = None
    ssl_ca: Optional[str] = None
    ssl_cert: Optional[str] = None
    ssl_key: Optional[str] = None
    ssl_verify_server_cert: bool = False
    pool_size: int = 20
    max_overflow: int = 20
    pool_pre_ping: bool = True

    backup_strategy: DatabaseBackupStrategy = DatabaseBackupStrategy.IN_MEMORY
    # database backup directory
    backup_directory: str = Field(
        default_factory=lambda: os.path.join(
            GlobalConfiguration().config_directory,
            SQL_STORE_BACKUP_DIRECTORY_NAME,
        )
    )
    backup_database: Optional[str] = None

    @validator("secrets_store")
    def validate_secrets_store(
        cls, secrets_store: Optional[SecretsStoreConfiguration]
    ) -> SecretsStoreConfiguration:
        """Ensures that the secrets store is initialized with a default SQL secrets store.

        Args:
            secrets_store: The secrets store config to be validated.

        Returns:
            The validated secrets store config.
        """
        if secrets_store is None:
            secrets_store = SqlSecretsStoreConfiguration()

        return secrets_store

    @root_validator(pre=True)
    def _remove_grpc_attributes(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Removes old GRPC attributes.

        Args:
            values: All model attribute values.

        Returns:
            The model attribute values
        """
        grpc_attribute_keys = [
            "grpc_metadata_host",
            "grpc_metadata_port",
            "grpc_metadata_ssl_ca",
            "grpc_metadata_ssl_key",
            "grpc_metadata_ssl_cert",
        ]
        grpc_values = [values.pop(key, None) for key in grpc_attribute_keys]
        if any(grpc_values):
            logger.warning(
                "The GRPC attributes %s are unused and will be removed soon. "
                "Please remove them from SQLZenStore configuration. This will "
                "become an error in future versions of ZenML."
            )

        return values

    @root_validator
    def _validate_backup_strategy(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Validate the backup strategy.

        Args:
            values: All model attribute values.

        Returns:
            The model attribute values.

        Raises:
            ValueError: If the backup database name is not set when the backup
                database is requested.
        """
        backup_strategy = values.get("backup_strategy")
        if backup_strategy == DatabaseBackupStrategy.DATABASE and (
            not values.get("backup_database")
        ):
            raise ValueError(
                "The `backup_database` attribute must also be set if the "
                "backup strategy is set to use a backup database."
            )

        return values

    @root_validator
    def _validate_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validate the SQL URL.

        The validator also moves the MySQL username, password and database
        parameters from the URL into the other configuration arguments, if they
        are present in the URL.

        Args:
            values: The values to validate.

        Returns:
            The validated values.

        Raises:
            ValueError: If the URL is invalid or the SQL driver is not
                supported.
        """
        url = values.get("url")
        if url is None:
            return values

        # When running inside a container, if the URL uses localhost, the
        # target service will not be available. We try to replace localhost
        # with one of the special Docker or K3D internal hostnames.
        url = replace_localhost_with_internal_hostname(url)

        try:
            sql_url = make_url(url)
        except ArgumentError as e:
            raise ValueError(
                "Invalid SQL URL `%s`: %s. The URL must be in the format "
                "`driver://[[username:password@]hostname:port]/database["
                "?<extra-args>]`.",
                url,
                str(e),
            )

        if sql_url.drivername not in SQLDatabaseDriver.values():
            raise ValueError(
                "Invalid SQL driver value `%s`: The driver must be one of: %s.",
                url,
                ", ".join(SQLDatabaseDriver.values()),
            )
        values["driver"] = SQLDatabaseDriver(sql_url.drivername)
        if sql_url.drivername == SQLDatabaseDriver.SQLITE:
            if (
                sql_url.username
                or sql_url.password
                or sql_url.query
                or sql_url.database is None
            ):
                raise ValueError(
                    "Invalid SQLite URL `%s`: The URL must be in the "
                    "format `sqlite:///path/to/database.db`.",
                    url,
                )
            if values.get("username") or values.get("password"):
                raise ValueError(
                    "Invalid SQLite configuration: The username and password "
                    "must not be set",
                    url,
                )
            values["database"] = sql_url.database
        elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
            if sql_url.username:
                values["username"] = sql_url.username
                sql_url = sql_url._replace(username=None)
            if sql_url.password:
                values["password"] = sql_url.password
                sql_url = sql_url._replace(password=None)
            if sql_url.database:
                values["database"] = sql_url.database
                sql_url = sql_url._replace(database=None)
            if sql_url.query:
                for k, v in sql_url.query.items():
                    if k == "ssl_ca":
                        values["ssl_ca"] = v
                    elif k == "ssl_cert":
                        values["ssl_cert"] = v
                    elif k == "ssl_key":
                        values["ssl_key"] = v
                    elif k == "ssl_verify_server_cert":
                        values["ssl_verify_server_cert"] = v
                    else:
                        raise ValueError(
                            "Invalid MySQL URL query parameter `%s`: The "
                            "parameter must be one of: ssl_ca, ssl_cert, "
                            "ssl_key, or ssl_verify_server_cert.",
                            k,
                        )
                sql_url = sql_url._replace(query={})

            database = values.get("database")
            if (
                not values.get("username")
                or not values.get("password")
                or not database
            ):
                raise ValueError(
                    "Invalid MySQL configuration: The username, password and "
                    "database must be set in the URL or as configuration "
                    "attributes",
                )

            regexp = r"^[^\\/?%*:|\"<>.-]{1,64}$"
            match = re.match(regexp, database)
            if not match:
                raise ValueError(
                    f"The database name does not conform to the required "
                    f"format "
                    f"rules ({regexp}): {database}"
                )

            # Save the certificates in a secure location on disk
            secret_folder = Path(
                GlobalConfiguration().local_stores_path,
                "certificates",
            )
            for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
                content = values.get(key)
                if content and not os.path.isfile(content):
                    fileio.makedirs(str(secret_folder))
                    file_path = Path(secret_folder, f"{key}.pem")
                    with open(file_path, "w") as f:
                        f.write(content)
                    file_path.chmod(0o600)
                    values[key] = str(file_path)

        values["url"] = str(sql_url)
        return values

    @staticmethod
    def get_local_url(path: str) -> str:
        """Get a local SQL url for a given local path.

        Args:
            path: The path to the local sqlite file.

        Returns:
            The local SQL url for the given path.
        """
        return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"

    @classmethod
    def supports_url_scheme(cls, url: str) -> bool:
        """Check if a URL scheme is supported by this store.

        Args:
            url: The URL to check.

        Returns:
            True if the URL scheme is supported, False otherwise.
        """
        return make_url(url).drivername in SQLDatabaseDriver.values()

    def expand_certificates(self) -> None:
        """Expands the certificates in the verify_ssl field."""
        # Load the certificate values back into the configuration
        for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
            file_path = getattr(self, key, None)
            if file_path and os.path.isfile(file_path):
                with open(file_path, "r") as f:
                    setattr(self, key, f.read())

    def get_sqlalchemy_config(
        self,
        database: Optional[str] = None,
    ) -> Tuple[URL, Dict[str, Any], Dict[str, Any]]:
        """Get the SQLAlchemy engine configuration for the SQL ZenML store.

        Args:
            database: Custom database name to use. If not set, the database name
                from the configuration will be used.

        Returns:
            The URL and connection arguments for the SQLAlchemy engine.

        Raises:
            NotImplementedError: If the SQL driver is not supported.
        """
        sql_url = make_url(self.url)
        sqlalchemy_connect_args: Dict[str, Any] = {}
        engine_args = {}
        if sql_url.drivername == SQLDatabaseDriver.SQLITE:
            assert self.database is not None
            # The following default value is needed for sqlite to avoid the
            # Error:
            #   sqlite3.ProgrammingError: SQLite objects created in a thread can
            #   only be used in that same thread.
            sqlalchemy_connect_args = {"check_same_thread": False}
        elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
            # all these are guaranteed by our root validator
            assert self.database is not None
            assert self.username is not None
            assert self.password is not None
            assert sql_url.host is not None

            if not database:
                database = self.database

            engine_args = {
                "pool_size": self.pool_size,
                "max_overflow": self.max_overflow,
                "pool_pre_ping": self.pool_pre_ping,
            }

            sql_url = sql_url._replace(
                drivername="mysql+pymysql",
                username=self.username,
                password=self.password,
                database=database,
            )

            sqlalchemy_ssl_args: Dict[str, Any] = {}

            # Handle SSL params
            for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
                ssl_setting = getattr(self, key)
                if not ssl_setting:
                    continue
                if not os.path.isfile(ssl_setting):
                    logger.warning(
                        f"Database SSL setting `{key}` is not a file. "
                    )
                sqlalchemy_ssl_args[key.lstrip("ssl_")] = ssl_setting
            if len(sqlalchemy_ssl_args) > 0:
                sqlalchemy_ssl_args["check_hostname"] = (
                    self.ssl_verify_server_cert
                )
                sqlalchemy_connect_args["ssl"] = sqlalchemy_ssl_args
        else:
            raise NotImplementedError(
                f"SQL driver `{sql_url.drivername}` is not supported."
            )

        return sql_url, sqlalchemy_connect_args, engine_args

    class Config:
        """Pydantic configuration class."""

        # Don't validate attributes when assigning them. This is necessary
        # because the certificate attributes can be expanded to the contents
        # of the certificate files.
        validate_assignment = False
        # Forbid extra attributes set in the class.
        extra = "forbid"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/sql_zen_store.py
class Config:
    """Pydantic configuration class."""

    # Don't validate attributes when assigning them. This is necessary
    # because the certificate attributes can be expanded to the contents
    # of the certificate files.
    validate_assignment = False
    # Forbid extra attributes set in the class.
    extra = "forbid"
expand_certificates(self)

Expands the certificates in the verify_ssl field.

Source code in zenml/zen_stores/sql_zen_store.py
def expand_certificates(self) -> None:
    """Expands the certificates in the verify_ssl field."""
    # Load the certificate values back into the configuration
    for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
        file_path = getattr(self, key, None)
        if file_path and os.path.isfile(file_path):
            with open(file_path, "r") as f:
                setattr(self, key, f.read())
get_local_url(path) staticmethod

Get a local SQL url for a given local path.

Parameters:

Name Type Description Default
path str

The path to the local sqlite file.

required

Returns:

Type Description
str

The local SQL url for the given path.

Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def get_local_url(path: str) -> str:
    """Get a local SQL url for a given local path.

    Args:
        path: The path to the local sqlite file.

    Returns:
        The local SQL url for the given path.
    """
    return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"
get_sqlalchemy_config(self, database=None)

Get the SQLAlchemy engine configuration for the SQL ZenML store.

Parameters:

Name Type Description Default
database Optional[str]

Custom database name to use. If not set, the database name from the configuration will be used.

None

Returns:

Type Description
Tuple[sqlalchemy.engine.url.URL, Dict[str, Any], Dict[str, Any]]

The URL and connection arguments for the SQLAlchemy engine.

Exceptions:

Type Description
NotImplementedError

If the SQL driver is not supported.

Source code in zenml/zen_stores/sql_zen_store.py
def get_sqlalchemy_config(
    self,
    database: Optional[str] = None,
) -> Tuple[URL, Dict[str, Any], Dict[str, Any]]:
    """Get the SQLAlchemy engine configuration for the SQL ZenML store.

    Args:
        database: Custom database name to use. If not set, the database name
            from the configuration will be used.

    Returns:
        The URL and connection arguments for the SQLAlchemy engine.

    Raises:
        NotImplementedError: If the SQL driver is not supported.
    """
    sql_url = make_url(self.url)
    sqlalchemy_connect_args: Dict[str, Any] = {}
    engine_args = {}
    if sql_url.drivername == SQLDatabaseDriver.SQLITE:
        assert self.database is not None
        # The following default value is needed for sqlite to avoid the
        # Error:
        #   sqlite3.ProgrammingError: SQLite objects created in a thread can
        #   only be used in that same thread.
        sqlalchemy_connect_args = {"check_same_thread": False}
    elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
        # all these are guaranteed by our root validator
        assert self.database is not None
        assert self.username is not None
        assert self.password is not None
        assert sql_url.host is not None

        if not database:
            database = self.database

        engine_args = {
            "pool_size": self.pool_size,
            "max_overflow": self.max_overflow,
            "pool_pre_ping": self.pool_pre_ping,
        }

        sql_url = sql_url._replace(
            drivername="mysql+pymysql",
            username=self.username,
            password=self.password,
            database=database,
        )

        sqlalchemy_ssl_args: Dict[str, Any] = {}

        # Handle SSL params
        for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
            ssl_setting = getattr(self, key)
            if not ssl_setting:
                continue
            if not os.path.isfile(ssl_setting):
                logger.warning(
                    f"Database SSL setting `{key}` is not a file. "
                )
            sqlalchemy_ssl_args[key.lstrip("ssl_")] = ssl_setting
        if len(sqlalchemy_ssl_args) > 0:
            sqlalchemy_ssl_args["check_hostname"] = (
                self.ssl_verify_server_cert
            )
            sqlalchemy_connect_args["ssl"] = sqlalchemy_ssl_args
    else:
        raise NotImplementedError(
            f"SQL driver `{sql_url.drivername}` is not supported."
        )

    return sql_url, sqlalchemy_connect_args, engine_args
supports_url_scheme(url) classmethod

Check if a URL scheme is supported by this store.

Parameters:

Name Type Description Default
url str

The URL to check.

required

Returns:

Type Description
bool

True if the URL scheme is supported, False otherwise.

Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
    """Check if a URL scheme is supported by this store.

    Args:
        url: The URL to check.

    Returns:
        True if the URL scheme is supported, False otherwise.
    """
    return make_url(url).drivername in SQLDatabaseDriver.values()
validate_secrets_store(secrets_store) classmethod

Ensures that the secrets store is initialized with a default SQL secrets store.

Parameters:

Name Type Description Default
secrets_store Optional[zenml.config.secrets_store_config.SecretsStoreConfiguration]

The secrets store config to be validated.

required

Returns:

Type Description
SecretsStoreConfiguration

The validated secrets store config.

Source code in zenml/zen_stores/sql_zen_store.py
@validator("secrets_store")
def validate_secrets_store(
    cls, secrets_store: Optional[SecretsStoreConfiguration]
) -> SecretsStoreConfiguration:
    """Ensures that the secrets store is initialized with a default SQL secrets store.

    Args:
        secrets_store: The secrets store config to be validated.

    Returns:
        The validated secrets store config.
    """
    if secrets_store is None:
        secrets_store = SqlSecretsStoreConfiguration()

    return secrets_store
backup_database(self, strategy=None, location=None, overwrite=False)

Backup the database.

Parameters:

Name Type Description Default
strategy Optional[zenml.enums.DatabaseBackupStrategy]

Custom backup strategy to use. If not set, the backup strategy from the store configuration will be used.

None
location Optional[str]

Custom target location to backup the database to. If not set, the configured backup location will be used. Depending on the backup strategy, this can be a file path or a database name.

None
overwrite bool

Whether to overwrite an existing backup if it exists. If set to False, the existing backup will be reused.

False

Returns:

Type Description
Tuple[str, Any]

The location where the database was backed up to and an accompanying user-friendly message that describes the backup location, or None if no backup was created (i.e. because the backup already exists).

Exceptions:

Type Description
ValueError

If the backup database name is not set when the backup database is requested or if the backup strategy is invalid.

Source code in zenml/zen_stores/sql_zen_store.py
def backup_database(
    self,
    strategy: Optional[DatabaseBackupStrategy] = None,
    location: Optional[str] = None,
    overwrite: bool = False,
) -> Tuple[str, Any]:
    """Backup the database.

    Args:
        strategy: Custom backup strategy to use. If not set, the backup
            strategy from the store configuration will be used.
        location: Custom target location to backup the database to. If not
            set, the configured backup location will be used. Depending on
            the backup strategy, this can be a file path or a database name.
        overwrite: Whether to overwrite an existing backup if it exists.
            If set to False, the existing backup will be reused.

    Returns:
        The location where the database was backed up to and an accompanying
        user-friendly message that describes the backup location, or None
        if no backup was created (i.e. because the backup already exists).

    Raises:
        ValueError: If the backup database name is not set when the backup
            database is requested or if the backup strategy is invalid.
    """
    strategy = strategy or self.config.backup_strategy

    if (
        strategy == DatabaseBackupStrategy.DUMP_FILE
        or self.config.driver == SQLDatabaseDriver.SQLITE
    ):
        dump_file = location or self._get_db_backup_file_path()

        if not overwrite and os.path.isfile(dump_file):
            logger.warning(
                f"A previous backup file already exists at '{dump_file}'. "
                "Reusing the existing backup."
            )
        else:
            self.migration_utils.backup_database_to_file(
                dump_file=dump_file
            )
        return f"the '{dump_file}' backup file", dump_file
    elif strategy == DatabaseBackupStrategy.DATABASE:
        backup_db_name = location or self.config.backup_database
        if not backup_db_name:
            raise ValueError(
                "The backup database name must be set in the store "
                "configuration to use the backup database strategy."
            )

        if not overwrite and self.migration_utils.database_exists(
            backup_db_name
        ):
            logger.warning(
                "A previous backup database already exists at "
                f"'{backup_db_name}'. Reusing the existing backup."
            )
        else:
            self.migration_utils.backup_database_to_db(
                backup_db_name=backup_db_name
            )
        return f"the '{backup_db_name}' backup database", backup_db_name
    elif strategy == DatabaseBackupStrategy.IN_MEMORY:
        return (
            "memory",
            self.migration_utils.backup_database_to_memory(),
        )

    else:
        raise ValueError(f"Invalid backup strategy: {strategy}.")
backup_secrets(self, ignore_errors=True, delete_secrets=False)

Backs up all secrets to the configured backup secrets store.

Parameters:

Name Type Description Default
ignore_errors bool

Whether to ignore individual errors during the backup process and attempt to backup all secrets.

True
delete_secrets bool

Whether to delete the secrets that have been successfully backed up from the primary secrets store. Setting this flag effectively moves all secrets from the primary secrets store to the backup secrets store.

False
noqa: DAR401

Exceptions:

Type Description
BackupSecretsStoreNotConfiguredError

if no backup secrets store is configured.

Source code in zenml/zen_stores/sql_zen_store.py
def backup_secrets(
    self, ignore_errors: bool = True, delete_secrets: bool = False
) -> None:
    """Backs up all secrets to the configured backup secrets store.

    Args:
        ignore_errors: Whether to ignore individual errors during the backup
            process and attempt to backup all secrets.
        delete_secrets: Whether to delete the secrets that have been
            successfully backed up from the primary secrets store. Setting
            this flag effectively moves all secrets from the primary secrets
            store to the backup secrets store.

    # noqa: DAR401
    Raises:
        BackupSecretsStoreNotConfiguredError: if no backup secrets store is
            configured.
    """
    if not self.backup_secrets_store:
        raise BackupSecretsStoreNotConfiguredError(
            "Unable to backup secrets: No backup secrets store is "
            "configured."
        )

    with Session(self.engine) as session:
        secrets_in_db = session.exec(select(SecretSchema)).all()

    for secret in secrets_in_db:
        try:
            values = self._get_secret_values(
                secret_id=secret.id, use_backup=False
            )
        except Exception:
            logger.exception(
                f"Failed to get secret values for secret with ID "
                f"{secret.id}."
            )
            if ignore_errors:
                continue
            raise

        try:
            self._backup_secret_values(secret_id=secret.id, values=values)
        except Exception:
            logger.exception(
                f"Failed to backup secret with ID {secret.id}. "
            )
            if ignore_errors:
                continue
            raise

        if delete_secrets:
            try:
                self._delete_secret_values(
                    secret_id=secret.id, delete_backup=False
                )
            except Exception:
                logger.exception(
                    f"Failed to delete secret with ID {secret.id} from the "
                    f"primary secrets store after backing it up to the "
                    f"backup secrets store."
                )
                if ignore_errors:
                    continue
                raise
cleanup_database_backup(self, strategy=None, location=None)

Delete the database backup.

Parameters:

Name Type Description Default
strategy Optional[zenml.enums.DatabaseBackupStrategy]

Custom backup strategy to use. If not set, the backup strategy from the store configuration will be used.

None
location Optional[Any]

Custom target location to delete the database backup from. If not set, the configured backup location will be used. Depending on the backup strategy, this can be a file path or a database name.

None

Exceptions:

Type Description
ValueError

If the backup database name is not set when the backup database is requested.

Source code in zenml/zen_stores/sql_zen_store.py
def cleanup_database_backup(
    self,
    strategy: Optional[DatabaseBackupStrategy] = None,
    location: Optional[Any] = None,
) -> None:
    """Delete the database backup.

    Args:
        strategy: Custom backup strategy to use. If not set, the backup
            strategy from the store configuration will be used.
        location: Custom target location to delete the database backup
            from. If not set, the configured backup location will be used.
            Depending on the backup strategy, this can be a file path or a
            database name.

    Raises:
        ValueError: If the backup database name is not set when the backup
            database is requested.
    """
    strategy = strategy or self.config.backup_strategy

    if (
        strategy == DatabaseBackupStrategy.DUMP_FILE
        or self.config.driver == SQLDatabaseDriver.SQLITE
    ):
        dump_file = location or self._get_db_backup_file_path()
        if dump_file is not None and os.path.isfile(dump_file):
            try:
                os.remove(dump_file)
            except OSError:
                logger.warning(
                    f"Failed to cleanup database dump file "
                    f"{dump_file}."
                )
            else:
                logger.info(
                    f"Successfully cleaned up database dump file "
                    f"{dump_file}."
                )
    elif strategy == DatabaseBackupStrategy.DATABASE:
        backup_db_name = location or self.config.backup_database

        if not backup_db_name:
            raise ValueError(
                "The backup database name must be set in the store "
                "configuration to use the backup database strategy."
            )
        if self.migration_utils.database_exists(backup_db_name):
            # Drop the backup database
            self.migration_utils.drop_database(
                database=backup_db_name,
            )
            logger.info(
                f"Successfully cleaned up backup database "
                f"{backup_db_name}."
            )
count_pipelines(self, filter_model)

Count all pipelines.

Parameters:

Name Type Description Default
filter_model Optional[zenml.models.v2.core.pipeline.PipelineFilter]

The filter model to use for counting pipelines.

required

Returns:

Type Description
int

The number of pipelines.

Source code in zenml/zen_stores/sql_zen_store.py
def count_pipelines(self, filter_model: Optional[PipelineFilter]) -> int:
    """Count all pipelines.

    Args:
        filter_model: The filter model to use for counting pipelines.

    Returns:
        The number of pipelines.
    """
    return self._count_entity(
        schema=PipelineSchema, filter_model=filter_model
    )
count_runs(self, filter_model)

Count all pipeline runs.

Parameters:

Name Type Description Default
filter_model Optional[zenml.models.v2.core.pipeline_run.PipelineRunFilter]

The filter model to filter the runs.

required

Returns:

Type Description
int

The number of pipeline runs.

Source code in zenml/zen_stores/sql_zen_store.py
def count_runs(self, filter_model: Optional[PipelineRunFilter]) -> int:
    """Count all pipeline runs.

    Args:
        filter_model: The filter model to filter the runs.

    Returns:
        The number of pipeline runs.
    """
    return self._count_entity(
        schema=PipelineRunSchema, filter_model=filter_model
    )
count_stack_components(self, filter_model=None)

Count all components.

Parameters:

Name Type Description Default
filter_model Optional[zenml.models.v2.core.component.ComponentFilter]

The filter model to use for counting components.

None

Returns:

Type Description
int

The number of components.

Source code in zenml/zen_stores/sql_zen_store.py
def count_stack_components(
    self, filter_model: Optional[ComponentFilter] = None
) -> int:
    """Count all components.

    Args:
        filter_model: The filter model to use for counting components.

    Returns:
        The number of components.
    """
    return self._count_entity(
        schema=StackComponentSchema, filter_model=filter_model
    )
count_stacks(self, filter_model)

Count all stacks.

Parameters:

Name Type Description Default
filter_model Optional[zenml.models.v2.core.stack.StackFilter]

The filter model to filter the stacks.

required

Returns:

Type Description
int

The number of stacks.

Source code in zenml/zen_stores/sql_zen_store.py
def count_stacks(self, filter_model: Optional[StackFilter]) -> int:
    """Count all stacks.

    Args:
        filter_model: The filter model to filter the stacks.

    Returns:
        The number of stacks.
    """
    return self._count_entity(
        schema=StackSchema, filter_model=filter_model
    )
create_api_key(self, service_account_id, api_key)

Create a new API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to create the API key.

required
api_key APIKeyRequest

The API key to create.

required

Returns:

Type Description
APIKeyResponse

The created API key.

Exceptions:

Type Description
EntityExistsError

If an API key with the same name is already configured for the same service account.

Source code in zenml/zen_stores/sql_zen_store.py
def create_api_key(
    self, service_account_id: UUID, api_key: APIKeyRequest
) -> APIKeyResponse:
    """Create a new API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            create the API key.
        api_key: The API key to create.

    Returns:
        The created API key.

    Raises:
        EntityExistsError: If an API key with the same name is already
            configured for the same service account.
    """
    with Session(self.engine) as session:
        # Fetch the service account
        service_account = self._get_account_schema(
            service_account_id, session=session, service_account=True
        )

        # Check if a key with the same name already exists for the same
        # service account
        try:
            self._get_api_key(
                service_account_id=service_account.id,
                api_key_name_or_id=api_key.name,
                session=session,
            )
            raise EntityExistsError(
                f"Unable to register API key with name '{api_key.name}': "
                "Found an existing API key with the same name configured "
                f"for the same '{service_account.name}' service account."
            )
        except KeyError:
            pass

        new_api_key, key_value = APIKeySchema.from_request(
            service_account_id=service_account.id,
            request=api_key,
        )
        session.add(new_api_key)
        session.commit()

        api_key_model = new_api_key.to_model(include_metadata=True)
        api_key_model.set_key(key_value)
        return api_key_model
create_artifact(self, artifact)

Creates a new artifact.

Parameters:

Name Type Description Default
artifact ArtifactRequest

The artifact to create.

required

Returns:

Type Description
ArtifactResponse

The newly created artifact.

Exceptions:

Type Description
EntityExistsError

If an artifact with the same name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
def create_artifact(self, artifact: ArtifactRequest) -> ArtifactResponse:
    """Creates a new artifact.

    Args:
        artifact: The artifact to create.

    Returns:
        The newly created artifact.

    Raises:
        EntityExistsError: If an artifact with the same name already exists.
    """
    with Session(self.engine) as session:
        # Check if an artifact with the given name already exists
        existing_artifact = session.exec(
            select(ArtifactSchema).where(
                ArtifactSchema.name == artifact.name
            )
        ).first()
        if existing_artifact is not None:
            raise EntityExistsError(
                f"Unable to create artifact with name '{artifact.name}': "
                "An artifact with the same name already exists."
            )

        # Create the artifact.
        artifact_schema = ArtifactSchema.from_request(artifact)

        # Save tags of the artifact.
        if artifact.tags:
            self._attach_tags_to_resource(
                tag_names=artifact.tags,
                resource_id=artifact_schema.id,
                resource_type=TaggableResourceTypes.ARTIFACT,
            )

        session.add(artifact_schema)
        session.commit()
        return artifact_schema.to_model(include_metadata=True)
create_artifact_version(self, artifact_version)

Creates an artifact version.

Parameters:

Name Type Description Default
artifact_version ArtifactVersionRequest

The artifact version to create.

required

Returns:

Type Description
ArtifactVersionResponse

The created artifact version.

Exceptions:

Type Description
EntityExistsError

if an artifact with the same name and version already exists.

Source code in zenml/zen_stores/sql_zen_store.py
def create_artifact_version(
    self, artifact_version: ArtifactVersionRequest
) -> ArtifactVersionResponse:
    """Creates an artifact version.

    Args:
        artifact_version: The artifact version to create.

    Returns:
        The created artifact version.

    Raises:
        EntityExistsError: if an artifact with the same name and version
            already exists.
    """
    with Session(self.engine) as session:
        # Check if an artifact with the given name and version exists
        def _check(tolerance: int = 0) -> None:
            query = session.exec(
                select(ArtifactVersionSchema)
                .where(
                    ArtifactVersionSchema.artifact_id
                    == artifact_version.artifact_id
                )
                .where(
                    ArtifactVersionSchema.version
                    == artifact_version.version
                )
            )
            existing_artifact = query.fetchmany(tolerance + 1)
            if (
                existing_artifact is not None
                and len(existing_artifact) > tolerance
            ):
                raise EntityExistsError(
                    f"Unable to create artifact with name "
                    f"'{existing_artifact[0].artifact.name}' and version "
                    f"'{artifact_version.version}': An artifact with the same "
                    "name and version already exists."
                )

        _check()
        # Create the artifact version.
        artifact_version_schema = ArtifactVersionSchema.from_request(
            artifact_version
        )
        session.add(artifact_version_schema)

        # Save visualizations of the artifact.
        if artifact_version.visualizations:
            for vis in artifact_version.visualizations:
                vis_schema = ArtifactVisualizationSchema.from_model(
                    artifact_visualization_request=vis,
                    artifact_version_id=artifact_version_schema.id,
                )
                session.add(vis_schema)

        # Save tags of the artifact.
        if artifact_version.tags:
            self._attach_tags_to_resource(
                tag_names=artifact_version.tags,
                resource_id=artifact_version_schema.id,
                resource_type=TaggableResourceTypes.ARTIFACT_VERSION,
            )

        try:
            _check(1)
            session.commit()
        except EntityExistsError as e:
            session.rollback()
            raise e

        return artifact_version_schema.to_model(include_metadata=True)
create_authorized_device(self, device)

Creates a new OAuth 2.0 authorized device.

Parameters:

Name Type Description Default
device OAuthDeviceInternalRequest

The device to be created.

required

Returns:

Type Description
OAuthDeviceInternalResponse

The newly created device.

Exceptions:

Type Description
EntityExistsError

If a device for the same client ID already exists.

Source code in zenml/zen_stores/sql_zen_store.py
def create_authorized_device(
    self, device: OAuthDeviceInternalRequest
) -> OAuthDeviceInternalResponse:
    """Creates a new OAuth 2.0 authorized device.

    Args:
        device: The device to be created.

    Returns:
        The newly created device.

    Raises:
        EntityExistsError: If a device for the same client ID already
            exists.
    """
    with Session(self.engine) as session:
        existing_device = session.exec(
            select(OAuthDeviceSchema).where(
                OAuthDeviceSchema.client_id == device.client_id
            )
        ).first()
        if existing_device is not None:
            raise EntityExistsError(
                f"Unable to create device with client ID "
                f"'{device.client_id}': A device with this client ID "
                "already exists."
            )

        (
            new_device,
            user_code,
            device_code,
        ) = OAuthDeviceSchema.from_request(device)
        session.add(new_device)
        session.commit()
        session.refresh(new_device)

        device_model = new_device.to_internal_model(hydrate=True)
        # Replace the hashed user code with the original user code
        device_model.user_code = user_code
        # Replace the hashed device code with the original device code
        device_model.device_code = device_code

        return device_model
create_build(self, build)

Creates a new build in a workspace.

Parameters:

Name Type Description Default
build PipelineBuildRequest

The build to create.

required

Returns:

Type Description
PipelineBuildResponse

The newly created build.

Source code in zenml/zen_stores/sql_zen_store.py
def create_build(
    self,
    build: PipelineBuildRequest,
) -> PipelineBuildResponse:
    """Creates a new build in a workspace.

    Args:
        build: The build to create.

    Returns:
        The newly created build.
    """
    with Session(self.engine) as session:
        # Create the build
        new_build = PipelineBuildSchema.from_request(build)
        session.add(new_build)
        session.commit()
        session.refresh(new_build)

        return new_build.to_model(include_metadata=True)
create_code_repository(self, code_repository)

Creates a new code repository.

Parameters:

Name Type Description Default
code_repository CodeRepositoryRequest

Code repository to be created.

required

Returns:

Type Description
CodeRepositoryResponse

The newly created code repository.

Exceptions:

Type Description
EntityExistsError

If a code repository with the given name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.REGISTERED_CODE_REPOSITORY)
def create_code_repository(
    self, code_repository: CodeRepositoryRequest
) -> CodeRepositoryResponse:
    """Creates a new code repository.

    Args:
        code_repository: Code repository to be created.

    Returns:
        The newly created code repository.

    Raises:
        EntityExistsError: If a code repository with the given name already
            exists.
    """
    with Session(self.engine) as session:
        existing_repo = session.exec(
            select(CodeRepositorySchema)
            .where(CodeRepositorySchema.name == code_repository.name)
            .where(
                CodeRepositorySchema.workspace_id
                == code_repository.workspace
            )
        ).first()
        if existing_repo is not None:
            raise EntityExistsError(
                f"Unable to create code repository in workspace "
                f"'{code_repository.workspace}': A code repository with "
                "this name already exists."
            )

        new_repo = CodeRepositorySchema.from_request(code_repository)
        session.add(new_repo)
        session.commit()
        session.refresh(new_repo)

        return new_repo.to_model(include_metadata=True)
create_deployment(self, deployment)

Creates a new deployment in a workspace.

Parameters:

Name Type Description Default
deployment PipelineDeploymentRequest

The deployment to create.

required

Returns:

Type Description
PipelineDeploymentResponse

The newly created deployment.

Source code in zenml/zen_stores/sql_zen_store.py
def create_deployment(
    self,
    deployment: PipelineDeploymentRequest,
) -> PipelineDeploymentResponse:
    """Creates a new deployment in a workspace.

    Args:
        deployment: The deployment to create.

    Returns:
        The newly created deployment.
    """
    with Session(self.engine) as session:
        code_reference_id = self._create_or_reuse_code_reference(
            session=session,
            workspace_id=deployment.workspace,
            code_reference=deployment.code_reference,
        )

        new_deployment = PipelineDeploymentSchema.from_request(
            deployment, code_reference_id=code_reference_id
        )
        session.add(new_deployment)
        session.commit()
        session.refresh(new_deployment)

        return new_deployment.to_model(include_metadata=True)
create_event_source(self, event_source)

Create an event_source.

Parameters:

Name Type Description Default
event_source EventSourceRequest

The event_source to create.

required

Returns:

Type Description
EventSourceResponse

The created event_source.

Source code in zenml/zen_stores/sql_zen_store.py
def create_event_source(
    self, event_source: EventSourceRequest
) -> EventSourceResponse:
    """Create an event_source.

    Args:
        event_source: The event_source to create.

    Returns:
        The created event_source.
    """
    with Session(self.engine) as session:
        self._fail_if_event_source_with_name_exists(
            event_source=event_source,
            session=session,
        )
        new_event_source = EventSourceSchema.from_request(event_source)
        session.add(new_event_source)
        session.commit()
        session.refresh(new_event_source)

        return new_event_source.to_model(
            include_metadata=True, include_resources=True
        )
create_flavor(self, flavor)

Creates a new stack component flavor.

Parameters:

Name Type Description Default
flavor FlavorRequest

The stack component flavor to create.

required

Returns:

Type Description
FlavorResponse

The newly created flavor.

Exceptions:

Type Description
EntityExistsError

If a flavor with the same name and type is already owned by this user in this workspace.

ValueError

In case the config_schema string exceeds the max length.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_FLAVOR)
def create_flavor(self, flavor: FlavorRequest) -> FlavorResponse:
    """Creates a new stack component flavor.

    Args:
        flavor: The stack component flavor to create.

    Returns:
        The newly created flavor.

    Raises:
        EntityExistsError: If a flavor with the same name and type
            is already owned by this user in this workspace.
        ValueError: In case the config_schema string exceeds the max length.
    """
    with Session(self.engine) as session:
        # Check if flavor with the same domain key (name, type, workspace,
        # owner) already exists
        existing_flavor = session.exec(
            select(FlavorSchema)
            .where(FlavorSchema.name == flavor.name)
            .where(FlavorSchema.type == flavor.type)
            .where(FlavorSchema.workspace_id == flavor.workspace)
            .where(FlavorSchema.user_id == flavor.user)
        ).first()

        if existing_flavor is not None:
            raise EntityExistsError(
                f"Unable to register '{flavor.type.value}' flavor "
                f"with name '{flavor.name}': Found an existing "
                f"flavor with the same name and type in the same "
                f"'{flavor.workspace}' workspace owned by the same "
                f"'{flavor.user}' user."
            )

        config_schema = json.dumps(flavor.config_schema)

        if len(config_schema) > TEXT_FIELD_MAX_LENGTH:
            raise ValueError(
                "Json representation of configuration schema"
                "exceeds max length."
            )

        else:
            new_flavor = FlavorSchema(
                name=flavor.name,
                type=flavor.type,
                source=flavor.source,
                config_schema=config_schema,
                integration=flavor.integration,
                connector_type=flavor.connector_type,
                connector_resource_type=flavor.connector_resource_type,
                connector_resource_id_attr=flavor.connector_resource_id_attr,
                workspace_id=flavor.workspace,
                user_id=flavor.user,
                logo_url=flavor.logo_url,
                docs_url=flavor.docs_url,
                sdk_docs_url=flavor.sdk_docs_url,
                is_custom=flavor.is_custom,
            )
            session.add(new_flavor)
            session.commit()

            return new_flavor.to_model(include_metadata=True)
create_model(self, model)

Creates a new model.

Parameters:

Name Type Description Default
model ModelRequest

the Model to be created.

required

Returns:

Type Description
ModelResponse

The newly created model.

Exceptions:

Type Description
EntityExistsError

If a workspace with the given name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_MODEL)
def create_model(self, model: ModelRequest) -> ModelResponse:
    """Creates a new model.

    Args:
        model: the Model to be created.

    Returns:
        The newly created model.

    Raises:
        EntityExistsError: If a workspace with the given name already exists.
    """
    with Session(self.engine) as session:
        existing_model = session.exec(
            select(ModelSchema).where(ModelSchema.name == model.name)
        ).first()
        if existing_model is not None:
            raise EntityExistsError(
                f"Unable to create model {model.name}: "
                "A model with this name already exists."
            )

        model_schema = ModelSchema.from_request(model)
        session.add(model_schema)

        if model.tags:
            self._attach_tags_to_resource(
                tag_names=model.tags,
                resource_id=model_schema.id,
                resource_type=TaggableResourceTypes.MODEL,
            )
        session.commit()
        return model_schema.to_model(include_metadata=True)
create_model_version(self, model_version)

Creates a new model version.

Parameters:

Name Type Description Default
model_version ModelVersionRequest

the Model Version to be created.

required

Returns:

Type Description
ModelVersionResponse

The newly created model version.

Exceptions:

Type Description
ValueError

If number is not None during model version creation.

EntityExistsError

If a workspace with the given name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_MODEL_VERSION)
def create_model_version(
    self, model_version: ModelVersionRequest
) -> ModelVersionResponse:
    """Creates a new model version.

    Args:
        model_version: the Model Version to be created.

    Returns:
        The newly created model version.

    Raises:
        ValueError: If `number` is not None during model version creation.
        EntityExistsError: If a workspace with the given name already exists.
    """
    if model_version.number is not None:
        raise ValueError(
            "`number` field  must be None during model version creation."
        )
    with Session(self.engine) as session:
        model_version_ = model_version.copy()
        model = self.get_model(model_version_.model)

        def _check(tolerance: int = 0) -> None:
            query = session.exec(
                select(ModelVersionSchema)
                .where(ModelVersionSchema.model_id == model.id)
                .where(ModelVersionSchema.name == model_version_.name)
            )
            existing_model_version = query.fetchmany(tolerance + 1)
            if (
                existing_model_version is not None
                and len(existing_model_version) > tolerance
            ):
                raise EntityExistsError(
                    f"Unable to create model version {model_version_.name}: "
                    f"A model version with this name already exists in {model.name} model."
                )

        _check()
        all_versions = session.exec(
            select(ModelVersionSchema)
            .where(ModelVersionSchema.model_id == model.id)
            .order_by(ModelVersionSchema.number.desc())  # type: ignore[attr-defined]
        ).first()

        model_version_.number = (
            all_versions.number + 1 if all_versions else 1
        )

        if model_version_.name is None:
            model_version_.name = str(model_version_.number)

        model_version_schema = ModelVersionSchema.from_request(
            model_version_
        )
        session.add(model_version_schema)

        if model_version_.tags:
            self._attach_tags_to_resource(
                tag_names=model_version_.tags,
                resource_id=model_version_schema.id,
                resource_type=TaggableResourceTypes.MODEL_VERSION,
            )
        try:
            _check(1)
            session.commit()
        except EntityExistsError as e:
            session.rollback()
            raise e

        return model_version_schema.to_model(include_metadata=True)

Creates a new model version link.

Parameters:

Name Type Description Default
model_version_artifact_link ModelVersionArtifactRequest

the Model Version to Artifact Link to be created.

required

Returns:

Type Description
ModelVersionArtifactResponse

The newly created model version to artifact link.

Source code in zenml/zen_stores/sql_zen_store.py
def create_model_version_artifact_link(
    self, model_version_artifact_link: ModelVersionArtifactRequest
) -> ModelVersionArtifactResponse:
    """Creates a new model version link.

    Args:
        model_version_artifact_link: the Model Version to Artifact Link
            to be created.

    Returns:
        The newly created model version to artifact link.
    """
    with Session(self.engine) as session:
        # If the link already exists, return it
        existing_model_version_artifact_link = session.exec(
            select(ModelVersionArtifactSchema)
            .where(
                ModelVersionArtifactSchema.model_version_id
                == model_version_artifact_link.model_version
            )
            .where(
                ModelVersionArtifactSchema.artifact_version_id
                == model_version_artifact_link.artifact_version,
            )
        ).first()
        if existing_model_version_artifact_link is not None:
            return existing_model_version_artifact_link.to_model()

        model_version_artifact_link_schema = (
            ModelVersionArtifactSchema.from_request(
                model_version_artifact_request=model_version_artifact_link,
            )
        )
        session.add(model_version_artifact_link_schema)
        session.commit()
        return model_version_artifact_link_schema.to_model(
            include_metadata=True
        )

Creates a new model version to pipeline run link.

Parameters:

Name Type Description Default
model_version_pipeline_run_link ModelVersionPipelineRunRequest

the Model Version to Pipeline Run Link to be created.

required

Returns:

Type Description
ModelVersionPipelineRunResponse
  • If Model Version to Pipeline Run Link already exists - returns the existing link.
  • Otherwise, returns the newly created model version to pipeline run link.
Source code in zenml/zen_stores/sql_zen_store.py
def create_model_version_pipeline_run_link(
    self,
    model_version_pipeline_run_link: ModelVersionPipelineRunRequest,
) -> ModelVersionPipelineRunResponse:
    """Creates a new model version to pipeline run link.

    Args:
        model_version_pipeline_run_link: the Model Version to Pipeline Run
            Link to be created.

    Returns:
        - If Model Version to Pipeline Run Link already exists - returns
            the existing link.
        - Otherwise, returns the newly created model version to pipeline
            run link.
    """
    with Session(self.engine) as session:
        # If the link already exists, return it
        existing_model_version_pipeline_run_link = session.exec(
            select(ModelVersionPipelineRunSchema)
            .where(
                ModelVersionPipelineRunSchema.model_version_id
                == model_version_pipeline_run_link.model_version
            )
            .where(
                ModelVersionPipelineRunSchema.pipeline_run_id
                == model_version_pipeline_run_link.pipeline_run,
            )
        ).first()
        if existing_model_version_pipeline_run_link is not None:
            return existing_model_version_pipeline_run_link.to_model()

        # Otherwise, create a new link
        model_version_pipeline_run_link_schema = (
            ModelVersionPipelineRunSchema.from_request(
                model_version_pipeline_run_link
            )
        )
        session.add(model_version_pipeline_run_link_schema)
        session.commit()
        return model_version_pipeline_run_link_schema.to_model(
            include_metadata=True
        )
create_pipeline(self, pipeline)

Creates a new pipeline in a workspace.

Parameters:

Name Type Description Default
pipeline PipelineRequest

The pipeline to create.

required

Returns:

Type Description
PipelineResponse

The newly created pipeline.

Exceptions:

Type Description
EntityExistsError

If an identical pipeline already exists.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATE_PIPELINE)
def create_pipeline(
    self,
    pipeline: PipelineRequest,
) -> PipelineResponse:
    """Creates a new pipeline in a workspace.

    Args:
        pipeline: The pipeline to create.

    Returns:
        The newly created pipeline.

    Raises:
        EntityExistsError: If an identical pipeline already exists.
    """
    with Session(self.engine) as session:
        # Check if pipeline with the given name already exists
        existing_pipeline = session.exec(
            select(PipelineSchema)
            .where(PipelineSchema.name == pipeline.name)
            .where(PipelineSchema.version_hash == pipeline.version_hash)
            .where(PipelineSchema.workspace_id == pipeline.workspace)
        ).first()
        if existing_pipeline is not None:
            raise EntityExistsError(
                f"Unable to create pipeline in workspace "
                f"'{pipeline.workspace}': A pipeline with this name and "
                f"version already exists."
            )

        # Create the pipeline
        new_pipeline = PipelineSchema.from_request(pipeline)
        session.add(new_pipeline)
        session.commit()
        session.refresh(new_pipeline)

        return new_pipeline.to_model(include_metadata=True)
create_run(self, pipeline_run)

Creates a pipeline run.

Parameters:

Name Type Description Default
pipeline_run PipelineRunRequest

The pipeline run to create.

required

Returns:

Type Description
PipelineRunResponse

The created pipeline run.

Exceptions:

Type Description
EntityExistsError

If a run with the same name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
def create_run(
    self, pipeline_run: PipelineRunRequest
) -> PipelineRunResponse:
    """Creates a pipeline run.

    Args:
        pipeline_run: The pipeline run to create.

    Returns:
        The created pipeline run.

    Raises:
        EntityExistsError: If a run with the same name already exists.
    """
    with Session(self.engine) as session:
        # Check if pipeline run with same name already exists.
        existing_domain_run = session.exec(
            select(PipelineRunSchema).where(
                PipelineRunSchema.name == pipeline_run.name
            )
        ).first()
        if existing_domain_run is not None:
            raise EntityExistsError(
                f"Unable to create pipeline run: A pipeline run with name "
                f"'{pipeline_run.name}' already exists."
            )

        # Create the pipeline run
        new_run = PipelineRunSchema.from_request(pipeline_run)
        session.add(new_run)
        session.commit()

        return new_run.to_model(include_metadata=True)
create_run_metadata(self, run_metadata)

Creates run metadata.

Parameters:

Name Type Description Default
run_metadata RunMetadataRequest

The run metadata to create.

required

Returns:

Type Description
List[zenml.models.v2.core.run_metadata.RunMetadataResponse]

The created run metadata.

Source code in zenml/zen_stores/sql_zen_store.py
def create_run_metadata(
    self, run_metadata: RunMetadataRequest
) -> List[RunMetadataResponse]:
    """Creates run metadata.

    Args:
        run_metadata: The run metadata to create.

    Returns:
        The created run metadata.
    """
    return_value: List[RunMetadataResponse] = []
    with Session(self.engine) as session:
        for key, value in run_metadata.values.items():
            type_ = run_metadata.types[key]
            run_metadata_schema = RunMetadataSchema(
                workspace_id=run_metadata.workspace,
                user_id=run_metadata.user,
                resource_id=run_metadata.resource_id,
                resource_type=run_metadata.resource_type.value,
                stack_component_id=run_metadata.stack_component_id,
                key=key,
                value=json.dumps(value),
                type=type_,
            )
            session.add(run_metadata_schema)
            session.commit()
            return_value.append(
                run_metadata_schema.to_model(include_metadata=True)
            )
    return return_value
create_run_step(self, step_run)

Creates a step run.

Parameters:

Name Type Description Default
step_run StepRunRequest

The step run to create.

required

Returns:

Type Description
StepRunResponse

The created step run.

Exceptions:

Type Description
EntityExistsError

if the step run already exists.

KeyError

if the pipeline run doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def create_run_step(self, step_run: StepRunRequest) -> StepRunResponse:
    """Creates a step run.

    Args:
        step_run: The step run to create.

    Returns:
        The created step run.

    Raises:
        EntityExistsError: if the step run already exists.
        KeyError: if the pipeline run doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if the pipeline run exists
        run = session.exec(
            select(PipelineRunSchema).where(
                PipelineRunSchema.id == step_run.pipeline_run_id
            )
        ).first()
        if run is None:
            raise KeyError(
                f"Unable to create step '{step_run.name}': No pipeline run "
                f"with ID '{step_run.pipeline_run_id}' found."
            )

        # Check if the step name already exists in the pipeline run
        existing_step_run = session.exec(
            select(StepRunSchema)
            .where(StepRunSchema.name == step_run.name)
            .where(
                StepRunSchema.pipeline_run_id == step_run.pipeline_run_id
            )
        ).first()
        if existing_step_run is not None:
            raise EntityExistsError(
                f"Unable to create step '{step_run.name}': A step with "
                f"this name already exists in the pipeline run with ID "
                f"'{step_run.pipeline_run_id}'."
            )

        # Create the step
        step_schema = StepRunSchema.from_request(step_run)
        session.add(step_schema)

        # Add logs entry for the step if exists
        if step_run.logs is not None:
            log_entry = LogsSchema(
                uri=step_run.logs.uri,
                step_run_id=step_schema.id,
                artifact_store_id=step_run.logs.artifact_store_id,
            )
            session.add(log_entry)

        # Save parent step IDs into the database.
        for parent_step_id in step_run.parent_step_ids:
            self._set_run_step_parent_step(
                child_id=step_schema.id,
                parent_id=parent_step_id,
                session=session,
            )

        # Save input artifact IDs into the database.
        for input_name, artifact_version_id in step_run.inputs.items():
            self._set_run_step_input_artifact(
                run_step_id=step_schema.id,
                artifact_version_id=artifact_version_id,
                name=input_name,
                input_type=StepRunInputArtifactType.DEFAULT,
                session=session,
            )

        # Save output artifact IDs into the database.
        for output_name, artifact_version_id in step_run.outputs.items():
            self._set_run_step_output_artifact(
                step_run_id=step_schema.id,
                artifact_version_id=artifact_version_id,
                name=output_name,
                output_type=StepRunOutputArtifactType.DEFAULT,
                session=session,
            )

        if step_run.status != ExecutionStatus.RUNNING:
            self._update_pipeline_run_status(
                pipeline_run_id=step_run.pipeline_run_id, session=session
            )

        session.commit()

        return step_schema.to_model(include_metadata=True)
create_schedule(self, schedule)

Creates a new schedule.

Parameters:

Name Type Description Default
schedule ScheduleRequest

The schedule to create.

required

Returns:

Type Description
ScheduleResponse

The newly created schedule.

Source code in zenml/zen_stores/sql_zen_store.py
def create_schedule(self, schedule: ScheduleRequest) -> ScheduleResponse:
    """Creates a new schedule.

    Args:
        schedule: The schedule to create.

    Returns:
        The newly created schedule.
    """
    with Session(self.engine) as session:
        new_schedule = ScheduleSchema.from_request(schedule)
        session.add(new_schedule)
        session.commit()
        return new_schedule.to_model(include_metadata=True)
create_secret(self, secret)

Creates a new secret.

The new secret is also validated against the scoping rules enforced in the secrets store:

  • only one workspace-scoped secret with the given name can exist in the target workspace.
  • only one user-scoped secret with the given name can exist in the target workspace for the target user.

Parameters:

Name Type Description Default
secret SecretRequest

The secret to create.

required

Returns:

Type Description
SecretResponse

The newly created secret.

Exceptions:

Type Description
EntityExistsError

If a secret with the same name already exists in the same scope.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_SECRET)
def create_secret(self, secret: SecretRequest) -> SecretResponse:
    """Creates a new secret.

    The new secret is also validated against the scoping rules enforced in
    the secrets store:

      - only one workspace-scoped secret with the given name can exist
        in the target workspace.
      - only one user-scoped secret with the given name can exist in the
        target workspace for the target user.

    Args:
        secret: The secret to create.

    Returns:
        The newly created secret.

    Raises:
        EntityExistsError: If a secret with the same name already exists in
            the same scope.
    """
    with Session(self.engine) as session:
        # Check if a secret with the same name already exists in the same
        # scope.
        secret_exists, msg = self._check_sql_secret_scope(
            session=session,
            secret_name=secret.name,
            scope=secret.scope,
            workspace=secret.workspace,
            user=secret.user,
        )
        if secret_exists:
            raise EntityExistsError(msg)

        new_secret = SecretSchema.from_request(
            secret,
        )
        session.add(new_secret)
        session.commit()

        secret_model = new_secret.to_model(include_metadata=True)

    try:
        # Set the secret values in the configured secrets store
        self._set_secret_values(
            secret_id=new_secret.id, values=secret.secret_values
        )
    except:
        # If setting the secret values fails, delete the secret from the
        # database.
        with Session(self.engine) as session:
            session.delete(new_secret)
            session.commit()
        raise

    secret_model.set_secrets(secret.secret_values)
    return secret_model
create_service_account(self, service_account)

Creates a new service account.

Parameters:

Name Type Description Default
service_account ServiceAccountRequest

Service account to be created.

required

Returns:

Type Description
ServiceAccountResponse

The newly created service account.

Exceptions:

Type Description
EntityExistsError

If a user or service account with the given name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_SERVICE_ACCOUNT)
def create_service_account(
    self, service_account: ServiceAccountRequest
) -> ServiceAccountResponse:
    """Creates a new service account.

    Args:
        service_account: Service account to be created.

    Returns:
        The newly created service account.

    Raises:
        EntityExistsError: If a user or service account with the given name
            already exists.
    """
    with Session(self.engine) as session:
        # Check if a service account with the given name already
        # exists
        err_msg = (
            f"Unable to create service account with name "
            f"'{service_account.name}': Found existing service "
            "account with this name."
        )
        try:
            self._get_account_schema(
                service_account.name, session=session, service_account=True
            )
            raise EntityExistsError(err_msg)
        except KeyError:
            pass

        # Create the service account
        new_account = UserSchema.from_service_account_request(
            service_account
        )
        session.add(new_account)
        # on commit an IntegrityError may arise we let it bubble up
        session.commit()

        return new_account.to_service_account_model(include_metadata=True)
create_service_connector(self, service_connector)

Creates a new service connector.

Parameters:

Name Type Description Default
service_connector ServiceConnectorRequest

Service connector to be created.

required

Returns:

Type Description
ServiceConnectorResponse

The newly created service connector.

Exceptions:

Type Description
Exception

If anything goes wrong during the creation of the service connector.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_SERVICE_CONNECTOR)
def create_service_connector(
    self, service_connector: ServiceConnectorRequest
) -> ServiceConnectorResponse:
    """Creates a new service connector.

    Args:
        service_connector: Service connector to be created.

    Returns:
        The newly created service connector.

    Raises:
        Exception: If anything goes wrong during the creation of the
            service connector.
    """
    # If the connector type is locally available, we validate the request
    # against the connector type schema before storing it in the database
    if service_connector_registry.is_registered(service_connector.type):
        connector_type = (
            service_connector_registry.get_service_connector_type(
                service_connector.type
            )
        )
        service_connector.validate_and_configure_resources(
            connector_type=connector_type,
            resource_types=service_connector.resource_types,
            resource_id=service_connector.resource_id,
            configuration=service_connector.configuration,
            secrets=service_connector.secrets,
        )

    with Session(self.engine) as session:
        self._fail_if_service_connector_with_name_exists(
            name=service_connector.name,
            workspace_id=service_connector.workspace,
            session=session,
        )

        # Create the secret
        secret_id = self._create_connector_secret(
            connector_name=service_connector.name,
            user=service_connector.user,
            workspace=service_connector.workspace,
            secrets=service_connector.secrets,
        )
        try:
            # Create the service connector
            new_service_connector = ServiceConnectorSchema.from_request(
                service_connector,
                secret_id=secret_id,
            )

            session.add(new_service_connector)
            session.commit()

            session.refresh(new_service_connector)
        except Exception:
            # Delete the secret if it was created
            if secret_id:
                try:
                    self.delete_secret(secret_id)
                except Exception:
                    # Ignore any errors that occur while deleting the
                    # secret
                    pass

            raise

        connector = new_service_connector.to_model(include_metadata=True)
        self._populate_connector_type(connector)
        return connector
create_stack(self, stack)

Register a new stack.

Parameters:

Name Type Description Default
stack StackRequest

The stack to register.

required

Returns:

Type Description
StackResponse

The registered stack.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.REGISTERED_STACK)
def create_stack(self, stack: StackRequest) -> StackResponse:
    """Register a new stack.

    Args:
        stack: The stack to register.

    Returns:
        The registered stack.
    """
    with Session(self.engine) as session:
        self._fail_if_stack_with_name_exists(stack=stack, session=session)

        # Get the Schemas of all components mentioned
        component_ids = (
            [
                component_id
                for list_of_component_ids in stack.components.values()
                for component_id in list_of_component_ids
            ]
            if stack.components is not None
            else []
        )
        filters = [
            (StackComponentSchema.id == component_id)
            for component_id in component_ids
        ]

        defined_components = session.exec(
            select(StackComponentSchema).where(or_(*filters))
        ).all()

        new_stack_schema = StackSchema(
            workspace_id=stack.workspace,
            user_id=stack.user,
            stack_spec_path=stack.stack_spec_path,
            name=stack.name,
            description=stack.description,
            components=defined_components,
        )

        session.add(new_stack_schema)
        session.commit()
        session.refresh(new_stack_schema)

        return new_stack_schema.to_model(include_metadata=True)
create_stack_component(self, component)

Create a stack component.

Parameters:

Name Type Description Default
component ComponentRequest

The stack component to create.

required

Returns:

Type Description
ComponentResponse

The created stack component.

Exceptions:

Type Description
KeyError

if the stack component references a non-existent connector.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.REGISTERED_STACK_COMPONENT)
def create_stack_component(
    self,
    component: ComponentRequest,
) -> ComponentResponse:
    """Create a stack component.

    Args:
        component: The stack component to create.

    Returns:
        The created stack component.

    Raises:
        KeyError: if the stack component references a non-existent
            connector.
    """
    with Session(self.engine) as session:
        self._fail_if_component_with_name_type_exists(
            name=component.name,
            component_type=component.type,
            workspace_id=component.workspace,
            session=session,
        )

        service_connector: Optional[ServiceConnectorSchema] = None
        if component.connector:
            service_connector = session.exec(
                select(ServiceConnectorSchema).where(
                    ServiceConnectorSchema.id == component.connector
                )
            ).first()

            if service_connector is None:
                raise KeyError(
                    f"Service connector with ID {component.connector} not "
                    "found."
                )

        # Create the component
        new_component = StackComponentSchema(
            name=component.name,
            workspace_id=component.workspace,
            user_id=component.user,
            component_spec_path=component.component_spec_path,
            type=component.type,
            flavor=component.flavor,
            configuration=base64.b64encode(
                json.dumps(component.configuration).encode("utf-8")
            ),
            labels=base64.b64encode(
                json.dumps(component.labels).encode("utf-8")
            ),
            connector=service_connector,
            connector_resource_id=component.connector_resource_id,
        )

        session.add(new_component)
        session.commit()

        session.refresh(new_component)

        return new_component.to_model(include_metadata=True)
create_tag(self, tag)

Creates a new tag.

Parameters:

Name Type Description Default
tag TagRequest

the tag to be created.

required

Returns:

Type Description
TagResponse

The newly created tag.

Exceptions:

Type Description
EntityExistsError

If a tag with the given name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_TAG)
def create_tag(self, tag: TagRequest) -> TagResponse:
    """Creates a new tag.

    Args:
        tag: the tag to be created.

    Returns:
        The newly created tag.

    Raises:
        EntityExistsError: If a tag with the given name already exists.
    """
    with Session(self.engine) as session:
        existing_tag = session.exec(
            select(TagSchema).where(TagSchema.name == tag.name)
        ).first()
        if existing_tag is not None:
            raise EntityExistsError(
                f"Unable to create tag {tag.name}: "
                "A tag with this name already exists."
            )

        tag_schema = TagSchema.from_request(tag)
        session.add(tag_schema)

        session.commit()
        return tag_schema.to_model(include_metadata=True)
create_tag_resource(self, tag_resource)

Creates a new tag resource relationship.

Parameters:

Name Type Description Default
tag_resource TagResourceRequest

the tag resource relationship to be created.

required

Returns:

Type Description
TagResourceResponse

The newly created tag resource relationship.

Exceptions:

Type Description
EntityExistsError

If a tag resource relationship with the given configuration already exists.

Source code in zenml/zen_stores/sql_zen_store.py
def create_tag_resource(
    self, tag_resource: TagResourceRequest
) -> TagResourceResponse:
    """Creates a new tag resource relationship.

    Args:
        tag_resource: the tag resource relationship to be created.

    Returns:
        The newly created tag resource relationship.

    Raises:
        EntityExistsError: If a tag resource relationship with the given
            configuration already exists.
    """
    with Session(self.engine) as session:
        existing_tag_resource = session.exec(
            select(TagResourceSchema).where(
                TagResourceSchema.tag_id == tag_resource.tag_id,
                TagResourceSchema.resource_id == tag_resource.resource_id,
                TagResourceSchema.resource_type
                == tag_resource.resource_type.value,
            )
        ).first()
        if existing_tag_resource is not None:
            raise EntityExistsError(
                f"Unable to create a tag "
                f"{tag_resource.resource_type.name.lower()} "
                f"relationship with IDs "
                f"`{tag_resource.tag_id}`|`{tag_resource.resource_id}`. "
                "This relationship already exists."
            )

        tag_resource_schema = TagResourceSchema.from_request(tag_resource)
        session.add(tag_resource_schema)

        session.commit()
        return tag_resource_schema.to_model(include_metadata=True)
create_trigger(self, trigger)

Creates a new trigger.

Parameters:

Name Type Description Default
trigger TriggerRequest

Trigger to be created.

required

Returns:

Type Description
TriggerResponse

The newly created trigger.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_TRIGGER)
def create_trigger(self, trigger: TriggerRequest) -> TriggerResponse:
    """Creates a new trigger.

    Args:
        trigger: Trigger to be created.

    Returns:
        The newly created trigger.
    """
    with Session(self.engine) as session:
        # Verify that the given event_source exists
        self._get_event_source(
            event_source_id=trigger.event_source_id, session=session
        )

        # Verify that the given service account exists
        self._get_account_schema(
            account_name_or_id=trigger.service_account_id,
            session=session,
            service_account=True,
        )

        # Verify that the trigger won't validate Unique
        self._fail_if_trigger_with_name_exists(
            trigger_name=trigger.name,
            workspace_id=trigger.workspace,
            session=session,
        )

        new_trigger = TriggerSchema.from_request(trigger)
        session.add(new_trigger)
        session.commit()
        session.refresh(new_trigger)

        return new_trigger.to_model(
            include_metadata=True, include_resources=True
        )
create_trigger_execution(self, trigger_execution)

Create a trigger execution.

Parameters:

Name Type Description Default
trigger_execution TriggerExecutionRequest

The trigger execution to create.

required

Returns:

Type Description
TriggerExecutionResponse

The created trigger execution.

Source code in zenml/zen_stores/sql_zen_store.py
def create_trigger_execution(
    self, trigger_execution: TriggerExecutionRequest
) -> TriggerExecutionResponse:
    """Create a trigger execution.

    Args:
        trigger_execution: The trigger execution to create.

    Returns:
        The created trigger execution.
    """
    with Session(self.engine) as session:
        # TODO: Verify that the given trigger exists
        new_execution = TriggerExecutionSchema.from_request(
            trigger_execution
        )
        session.add(new_execution)
        session.commit()
        session.refresh(new_execution)

        return new_execution.to_model(
            include_metadata=True, include_resources=True
        )
create_user(self, user)

Creates a new user.

Parameters:

Name Type Description Default
user UserRequest

User to be created.

required

Returns:

Type Description
UserResponse

The newly created user.

Exceptions:

Type Description
EntityExistsError

If a user or service account with the given name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
def create_user(self, user: UserRequest) -> UserResponse:
    """Creates a new user.

    Args:
        user: User to be created.

    Returns:
        The newly created user.

    Raises:
        EntityExistsError: If a user or service account with the given name
            already exists.
    """
    with Session(self.engine) as session:
        # Check if a user account with the given name already exists
        err_msg = (
            f"Unable to create user with name '{user.name}': "
            f"Found an existing user account with this name."
        )
        try:
            self._get_account_schema(
                user.name,
                session=session,
                # Filter out service accounts
                service_account=False,
            )
            raise EntityExistsError(err_msg)
        except KeyError:
            pass

        # Create the user
        new_user = UserSchema.from_user_request(user)
        session.add(new_user)
        # on commit an IntegrityError may arise we let it bubble up
        session.commit()
        return new_user.to_model(include_metadata=True)
create_workspace(self, workspace)

Creates a new workspace.

Parameters:

Name Type Description Default
workspace WorkspaceRequest

The workspace to create.

required

Returns:

Type Description
WorkspaceResponse

The newly created workspace.

Exceptions:

Type Description
EntityExistsError

If a workspace with the given name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.CREATED_WORKSPACE)
def create_workspace(
    self, workspace: WorkspaceRequest
) -> WorkspaceResponse:
    """Creates a new workspace.

    Args:
        workspace: The workspace to create.

    Returns:
        The newly created workspace.

    Raises:
        EntityExistsError: If a workspace with the given name already exists.
    """
    with Session(self.engine) as session:
        # Check if workspace with the given name already exists
        existing_workspace = session.exec(
            select(WorkspaceSchema).where(
                WorkspaceSchema.name == workspace.name
            )
        ).first()
        if existing_workspace is not None:
            raise EntityExistsError(
                f"Unable to create workspace {workspace.name}: "
                "A workspace with this name already exists."
            )

        # Create the workspace
        new_workspace = WorkspaceSchema.from_request(workspace)
        session.add(new_workspace)
        session.commit()

        # Explicitly refresh the new_workspace schema
        session.refresh(new_workspace)

        workspace_model = new_workspace.to_model(include_metadata=True)

    self._get_or_create_default_stack(workspace=workspace_model)
    return workspace_model

Deletes all model version to artifact links.

Parameters:

Name Type Description Default
model_version_id UUID

ID of the model version containing the link.

required
only_links bool

Whether to only delete the link to the artifact.

True
Source code in zenml/zen_stores/sql_zen_store.py
def delete_all_model_version_artifact_links(
    self,
    model_version_id: UUID,
    only_links: bool = True,
) -> None:
    """Deletes all model version to artifact links.

    Args:
        model_version_id: ID of the model version containing the link.
        only_links: Whether to only delete the link to the artifact.
    """
    with Session(self.engine) as session:
        if not only_links:
            artifact_version_ids = session.execute(
                select(
                    ModelVersionArtifactSchema.artifact_version_id
                ).where(
                    ModelVersionArtifactSchema.model_version_id
                    == model_version_id
                )
            ).fetchall()
            session.execute(
                delete(ArtifactVersionSchema).where(
                    col(ArtifactVersionSchema.id).in_(
                        [a[0] for a in artifact_version_ids]
                    )
                ),
            )
        session.execute(
            delete(ModelVersionArtifactSchema).where(
                ModelVersionArtifactSchema.model_version_id
                == model_version_id
            )
        )

        session.commit()
delete_api_key(self, service_account_id, api_key_name_or_id)

Delete an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to delete the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to delete.

required
Source code in zenml/zen_stores/sql_zen_store.py
def delete_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
) -> None:
    """Delete an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            delete the API key.
        api_key_name_or_id: The name or ID of the API key to delete.
    """
    with Session(self.engine) as session:
        api_key = self._get_api_key(
            service_account_id=service_account_id,
            api_key_name_or_id=api_key_name_or_id,
            session=session,
        )

        session.delete(api_key)
        session.commit()
delete_artifact(self, artifact_id)

Deletes an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to delete.

required

Exceptions:

Type Description
KeyError

if the artifact doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_artifact(self, artifact_id: UUID) -> None:
    """Deletes an artifact.

    Args:
        artifact_id: The ID of the artifact to delete.

    Raises:
        KeyError: if the artifact doesn't exist.
    """
    with Session(self.engine) as session:
        existing_artifact = session.exec(
            select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
        ).first()
        if not existing_artifact:
            raise KeyError(f"Artifact with ID {artifact_id} not found.")
        session.delete(existing_artifact)
        session.commit()
delete_artifact_version(self, artifact_version_id)

Deletes an artifact version.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to delete.

required

Exceptions:

Type Description
KeyError

if the artifact version doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_artifact_version(self, artifact_version_id: UUID) -> None:
    """Deletes an artifact version.

    Args:
        artifact_version_id: The ID of the artifact version to delete.

    Raises:
        KeyError: if the artifact version doesn't exist.
    """
    with Session(self.engine) as session:
        artifact_version = session.exec(
            select(ArtifactVersionSchema).where(
                ArtifactVersionSchema.id == artifact_version_id
            )
        ).first()
        if artifact_version is None:
            raise KeyError(
                f"Unable to delete artifact version with ID "
                f"{artifact_version_id}: No artifact version with this ID "
                "found."
            )
        session.delete(artifact_version)
        session.commit()
delete_authorized_device(self, device_id)

Deletes an OAuth 2.0 authorized device.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to delete.

required

Exceptions:

Type Description
KeyError

If no device with the given ID exists.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_authorized_device(self, device_id: UUID) -> None:
    """Deletes an OAuth 2.0 authorized device.

    Args:
        device_id: The ID of the device to delete.

    Raises:
        KeyError: If no device with the given ID exists.
    """
    with Session(self.engine) as session:
        existing_device = session.exec(
            select(OAuthDeviceSchema).where(
                OAuthDeviceSchema.id == device_id
            )
        ).first()
        if existing_device is None:
            raise KeyError(
                f"Unable to delete device with ID {device_id}: No device "
                "with this ID found."
            )

        session.delete(existing_device)
        session.commit()
delete_build(self, build_id)

Deletes a build.

Parameters:

Name Type Description Default
build_id UUID

The ID of the build to delete.

required

Exceptions:

Type Description
KeyError

if the build doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_build(self, build_id: UUID) -> None:
    """Deletes a build.

    Args:
        build_id: The ID of the build to delete.

    Raises:
        KeyError: if the build doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if build with the given ID exists
        build = session.exec(
            select(PipelineBuildSchema).where(
                PipelineBuildSchema.id == build_id
            )
        ).first()
        if build is None:
            raise KeyError(
                f"Unable to delete build with ID {build_id}: "
                f"No build with this ID found."
            )

        session.delete(build)
        session.commit()
delete_code_repository(self, code_repository_id)

Deletes a code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to delete.

required

Exceptions:

Type Description
KeyError

If no code repository with the given ID exists.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_code_repository(self, code_repository_id: UUID) -> None:
    """Deletes a code repository.

    Args:
        code_repository_id: The ID of the code repository to delete.

    Raises:
        KeyError: If no code repository with the given ID exists.
    """
    with Session(self.engine) as session:
        existing_repo = session.exec(
            select(CodeRepositorySchema).where(
                CodeRepositorySchema.id == code_repository_id
            )
        ).first()
        if existing_repo is None:
            raise KeyError(
                f"Unable to delete code repository with ID "
                f"{code_repository_id}: No code repository with this ID "
                "found."
            )

        session.delete(existing_repo)
        session.commit()
delete_deployment(self, deployment_id)

Deletes a deployment.

Parameters:

Name Type Description Default
deployment_id UUID

The ID of the deployment to delete.

required

Exceptions:

Type Description
KeyError

If the deployment doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_deployment(self, deployment_id: UUID) -> None:
    """Deletes a deployment.

    Args:
        deployment_id: The ID of the deployment to delete.

    Raises:
        KeyError: If the deployment doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if build with the given ID exists
        deployment = session.exec(
            select(PipelineDeploymentSchema).where(
                PipelineDeploymentSchema.id == deployment_id
            )
        ).first()
        if deployment is None:
            raise KeyError(
                f"Unable to delete deployment with ID {deployment_id}: "
                f"No deployment with this ID found."
            )

        session.delete(deployment)
        session.commit()
delete_event_source(self, event_source_id)

Delete an event_source.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to delete.

required

Exceptions:

Type Description
KeyError

if the event_source doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_event_source(self, event_source_id: UUID) -> None:
    """Delete an event_source.

    Args:
        event_source_id: The ID of the event_source to delete.

    Raises:
        KeyError: if the event_source doesn't exist.
    """
    with Session(self.engine) as session:
        event_source = self._get_event_source(
            event_source_id=event_source_id, session=session
        )
        if event_source is None:
            raise KeyError(
                f"Unable to delete event_source with ID `{event_source_id}`: "
                f"No event_source with this ID found."
            )
        session.delete(event_source)
        session.commit()

        # TODO: catch and throw proper error if it can't be deleted due to
        #  not-null constraints on triggers
delete_expired_authorized_devices(self)

Deletes all expired OAuth 2.0 authorized devices.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_expired_authorized_devices(self) -> None:
    """Deletes all expired OAuth 2.0 authorized devices."""
    with Session(self.engine) as session:
        expired_devices = session.exec(
            select(OAuthDeviceSchema).where(OAuthDeviceSchema.user is None)
        ).all()
        for device in expired_devices:
            # Delete devices that have expired
            if (
                device.expires is not None
                and device.expires < datetime.now()
                and device.user_id is None
            ):
                session.delete(device)
        session.commit()
delete_flavor(self, flavor_id)

Delete a flavor.

Parameters:

Name Type Description Default
flavor_id UUID

The id of the flavor to delete.

required

Exceptions:

Type Description
KeyError

if the flavor doesn't exist.

IllegalOperationError

if the flavor is used by a stack component.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_flavor(self, flavor_id: UUID) -> None:
    """Delete a flavor.

    Args:
        flavor_id: The id of the flavor to delete.

    Raises:
        KeyError: if the flavor doesn't exist.
        IllegalOperationError: if the flavor is used by a stack component.
    """
    with Session(self.engine) as session:
        try:
            flavor_in_db = session.exec(
                select(FlavorSchema).where(FlavorSchema.id == flavor_id)
            ).one()

            if flavor_in_db is None:
                raise KeyError(f"Flavor with ID {flavor_id} not found.")
            components_of_flavor = session.exec(
                select(StackComponentSchema).where(
                    StackComponentSchema.flavor == flavor_in_db.name
                )
            ).all()
            if len(components_of_flavor) > 0:
                raise IllegalOperationError(
                    f"Stack Component `{flavor_in_db.name}` of type "
                    f"`{flavor_in_db.type} cannot be "
                    f"deleted as it is used by "
                    f"{len(components_of_flavor)} "
                    f"components. Before deleting this "
                    f"flavor, make sure to delete all "
                    f"associated components."
                )
            else:
                session.delete(flavor_in_db)
                session.commit()
        except NoResultFound as error:
            raise KeyError from error
delete_model(self, model_name_or_id)

Deletes a model.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model to be deleted.

required

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_model(self, model_name_or_id: Union[str, UUID]) -> None:
    """Deletes a model.

    Args:
        model_name_or_id: name or id of the model to be deleted.

    Raises:
        KeyError: specified ID or name not found.
    """
    with Session(self.engine) as session:
        model = self._get_model_schema(
            model_name_or_id=model_name_or_id, session=session
        )
        if model is None:
            raise KeyError(
                f"Unable to delete model with ID `{model_name_or_id}`: "
                f"No model with this ID found."
            )
        session.delete(model)
        session.commit()
delete_model_version(self, model_version_id)

Deletes a model version.

Parameters:

Name Type Description Default
model_version_id UUID

name or id of the model version to be deleted.

required

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_model_version(
    self,
    model_version_id: UUID,
) -> None:
    """Deletes a model version.

    Args:
        model_version_id: name or id of the model version to be deleted.

    Raises:
        KeyError: specified ID or name not found.
    """
    with Session(self.engine) as session:
        query = select(ModelVersionSchema).where(
            ModelVersionSchema.id == model_version_id
        )
        model_version = session.exec(query).first()
        if model_version is None:
            raise KeyError(
                "Unable to delete model version with id "
                f"`{model_version_id}`: "
                "No model version with this id found."
            )
        session.delete(model_version)
        session.commit()

Deletes a model version to artifact link.

Parameters:

Name Type Description Default
model_version_id UUID

ID of the model version containing the link.

required
model_version_artifact_link_name_or_id Union[str, uuid.UUID]

name or ID of the model version to artifact link to be deleted.

required

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_model_version_artifact_link(
    self,
    model_version_id: UUID,
    model_version_artifact_link_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a model version to artifact link.

    Args:
        model_version_id: ID of the model version containing the link.
        model_version_artifact_link_name_or_id: name or ID of the model
            version to artifact link to be deleted.

    Raises:
        KeyError: specified ID or name not found.
    """
    with Session(self.engine) as session:
        model_version = self.get_model_version(model_version_id)
        query = select(ModelVersionArtifactSchema).where(
            ModelVersionArtifactSchema.model_version_id == model_version.id
        )
        try:
            UUID(str(model_version_artifact_link_name_or_id))
            query = query.where(
                ModelVersionArtifactSchema.id
                == model_version_artifact_link_name_or_id
            )
        except ValueError:
            query = (
                query.where(
                    ModelVersionArtifactSchema.artifact_version_id
                    == ArtifactVersionSchema.id
                )
                .where(
                    ArtifactVersionSchema.artifact_id == ArtifactSchema.id
                )
                .where(
                    ArtifactSchema.name
                    == model_version_artifact_link_name_or_id
                )
            )

        model_version_artifact_link = session.exec(query).first()
        if model_version_artifact_link is None:
            raise KeyError(
                f"Unable to delete model version link with name or ID "
                f"`{model_version_artifact_link_name_or_id}`: "
                f"No model version link with this name found."
            )

        session.delete(model_version_artifact_link)
        session.commit()

Deletes a model version to pipeline run link.

Parameters:

Name Type Description Default
model_version_id UUID

name or ID of the model version containing the link.

required
model_version_pipeline_run_link_name_or_id Union[str, uuid.UUID]

name or ID of the model version to pipeline run link to be deleted.

required

Exceptions:

Type Description
KeyError

specified ID not found.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_model_version_pipeline_run_link(
    self,
    model_version_id: UUID,
    model_version_pipeline_run_link_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a model version to pipeline run link.

    Args:
        model_version_id: name or ID of the model version containing the
            link.
        model_version_pipeline_run_link_name_or_id: name or ID of the model
            version to pipeline run link to be deleted.

    Raises:
        KeyError: specified ID not found.
    """
    with Session(self.engine) as session:
        model_version = self.get_model_version(
            model_version_id=model_version_id
        )
        query = select(ModelVersionPipelineRunSchema).where(
            ModelVersionPipelineRunSchema.model_version_id
            == model_version.id
        )
        try:
            UUID(str(model_version_pipeline_run_link_name_or_id))
            query = query.where(
                ModelVersionPipelineRunSchema.id
                == model_version_pipeline_run_link_name_or_id
            )
        except ValueError:
            query = query.where(
                ModelVersionPipelineRunSchema.pipeline_run_id
                == PipelineRunSchema.id
            ).where(
                PipelineRunSchema.name
                == model_version_pipeline_run_link_name_or_id
            )

        model_version_pipeline_run_link = session.exec(query).first()
        if model_version_pipeline_run_link is None:
            raise KeyError(
                f"Unable to delete model version link with name "
                f"`{model_version_pipeline_run_link_name_or_id}`: "
                f"No model version link with this name found."
            )

        session.delete(model_version_pipeline_run_link)
        session.commit()
delete_pipeline(self, pipeline_id)

Deletes a pipeline.

Parameters:

Name Type Description Default
pipeline_id UUID

The ID of the pipeline to delete.

required

Exceptions:

Type Description
KeyError

if the pipeline doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_pipeline(self, pipeline_id: UUID) -> None:
    """Deletes a pipeline.

    Args:
        pipeline_id: The ID of the pipeline to delete.

    Raises:
        KeyError: if the pipeline doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if pipeline with the given ID exists
        pipeline = session.exec(
            select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
        ).first()
        if pipeline is None:
            raise KeyError(
                f"Unable to delete pipeline with ID {pipeline_id}: "
                f"No pipeline with this ID found."
            )

        session.delete(pipeline)
        session.commit()
delete_run(self, run_id)

Deletes a pipeline run.

Parameters:

Name Type Description Default
run_id UUID

The ID of the pipeline run to delete.

required

Exceptions:

Type Description
KeyError

if the pipeline run doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_run(self, run_id: UUID) -> None:
    """Deletes a pipeline run.

    Args:
        run_id: The ID of the pipeline run to delete.

    Raises:
        KeyError: if the pipeline run doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if pipeline run with the given ID exists
        existing_run = session.exec(
            select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
        ).first()
        if existing_run is None:
            raise KeyError(
                f"Unable to delete pipeline run with ID {run_id}: "
                f"No pipeline run with this ID found."
            )

        # Delete the pipeline run
        session.delete(existing_run)
        session.commit()
delete_schedule(self, schedule_id)

Deletes a schedule.

Parameters:

Name Type Description Default
schedule_id UUID

The ID of the schedule to delete.

required

Exceptions:

Type Description
KeyError

if the schedule doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_schedule(self, schedule_id: UUID) -> None:
    """Deletes a schedule.

    Args:
        schedule_id: The ID of the schedule to delete.

    Raises:
        KeyError: if the schedule doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if schedule with the given ID exists
        schedule = session.exec(
            select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
        ).first()
        if schedule is None:
            raise KeyError(
                f"Unable to delete schedule with ID {schedule_id}: "
                f"No schedule with this ID found."
            )

        # Delete the schedule
        session.delete(schedule)
        session.commit()
delete_secret(self, secret_id)

Delete a secret.

Parameters:

Name Type Description Default
secret_id UUID

The id of the secret to delete.

required

Exceptions:

Type Description
KeyError

if the secret doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_secret(self, secret_id: UUID) -> None:
    """Delete a secret.

    Args:
        secret_id: The id of the secret to delete.

    Raises:
        KeyError: if the secret doesn't exist.
    """
    # Delete the secret values in the configured secrets store
    try:
        self._delete_secret_values(secret_id=secret_id)
    except KeyError:
        # If the secret values don't exist in the secrets store, we don't
        # need to raise an error.
        pass

    with Session(self.engine) as session:
        try:
            secret_in_db = session.exec(
                select(SecretSchema).where(SecretSchema.id == secret_id)
            ).one()
            session.delete(secret_in_db)
            session.commit()
        except NoResultFound:
            raise KeyError(f"Secret with ID {secret_id} not found.")
delete_service_account(self, service_account_name_or_id)

Delete a service account.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or the ID of the service account to delete.

required

Exceptions:

Type Description
IllegalOperationError

if the service account has already been used to create other resources.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
) -> None:
    """Delete a service account.

    Args:
        service_account_name_or_id: The name or the ID of the service
            account to delete.

    Raises:
        IllegalOperationError: if the service account has already been used
            to create other resources.
    """
    with Session(self.engine) as session:
        service_account = self._get_account_schema(
            service_account_name_or_id,
            session=session,
            service_account=True,
        )
        # Check if the service account has any resources associated with it
        # and raise an error if it does.
        if self._account_owns_resources(service_account, session=session):
            raise IllegalOperationError(
                "The service account has already been used to create "
                "other resources that it now owns and therefore cannot be "
                "deleted. Please delete all resources owned by the service "
                "account or consider deactivating it instead."
            )

        session.delete(service_account)
        session.commit()
delete_service_connector(self, service_connector_id)

Deletes a service connector.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to delete.

required

Exceptions:

Type Description
KeyError

If no service connector with the given ID exists.

IllegalOperationError

If the service connector is still referenced by one or more stack components.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_service_connector(self, service_connector_id: UUID) -> None:
    """Deletes a service connector.

    Args:
        service_connector_id: The ID of the service connector to delete.

    Raises:
        KeyError: If no service connector with the given ID exists.
        IllegalOperationError: If the service connector is still referenced
            by one or more stack components.
    """
    with Session(self.engine) as session:
        try:
            service_connector = session.exec(
                select(ServiceConnectorSchema).where(
                    ServiceConnectorSchema.id == service_connector_id
                )
            ).one()

            if service_connector is None:
                raise KeyError(
                    f"Service connector with ID {service_connector_id} not "
                    "found."
                )

            if len(service_connector.components) > 0:
                raise IllegalOperationError(
                    f"Service connector with ID {service_connector_id} "
                    f"cannot be deleted as it is still referenced by "
                    f"{len(service_connector.components)} "
                    "stack components. Before deleting this service "
                    "connector, make sure to remove it from all stack "
                    "components."
                )
            else:
                session.delete(service_connector)

            if service_connector.secret_id:
                try:
                    self.delete_secret(service_connector.secret_id)
                except KeyError:
                    # If the secret doesn't exist anymore, we can ignore
                    # this error
                    pass
        except NoResultFound as error:
            raise KeyError from error

        session.commit()
delete_stack(self, stack_id)

Delete a stack.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack to delete.

required

Exceptions:

Type Description
KeyError

if the stack doesn't exist.

IllegalOperationError

if the stack is a default stack.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_stack(self, stack_id: UUID) -> None:
    """Delete a stack.

    Args:
        stack_id: The ID of the stack to delete.

    Raises:
        KeyError: if the stack doesn't exist.
        IllegalOperationError: if the stack is a default stack.
    """
    with Session(self.engine) as session:
        try:
            stack = session.exec(
                select(StackSchema).where(StackSchema.id == stack_id)
            ).one()

            if stack is None:
                raise KeyError(f"Stack with ID {stack_id} not found.")
            if stack.name == DEFAULT_STACK_AND_COMPONENT_NAME:
                raise IllegalOperationError(
                    "The default stack cannot be deleted."
                )
            session.delete(stack)
        except NoResultFound as error:
            raise KeyError from error

        session.commit()
delete_stack_component(self, component_id)

Delete a stack component.

Parameters:

Name Type Description Default
component_id UUID

The id of the stack component to delete.

required

Exceptions:

Type Description
KeyError

if the stack component doesn't exist.

IllegalOperationError

if the stack component is part of one or more stacks, or if it's a default stack component.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_stack_component(self, component_id: UUID) -> None:
    """Delete a stack component.

    Args:
        component_id: The id of the stack component to delete.

    Raises:
        KeyError: if the stack component doesn't exist.
        IllegalOperationError: if the stack component is part of one or
            more stacks, or if it's a default stack component.
    """
    with Session(self.engine) as session:
        try:
            stack_component = session.exec(
                select(StackComponentSchema).where(
                    StackComponentSchema.id == component_id
                )
            ).one()

            if stack_component is None:
                raise KeyError(f"Stack with ID {component_id} not found.")
            if (
                stack_component.name == DEFAULT_STACK_AND_COMPONENT_NAME
                and stack_component.type
                in [
                    StackComponentType.ORCHESTRATOR,
                    StackComponentType.ARTIFACT_STORE,
                ]
            ):
                raise IllegalOperationError(
                    f"The default {stack_component.type} cannot be deleted."
                )

            if len(stack_component.stacks) > 0:
                raise IllegalOperationError(
                    f"Stack Component `{stack_component.name}` of type "
                    f"`{stack_component.type} cannot be "
                    f"deleted as it is part of "
                    f"{len(stack_component.stacks)} stacks. "
                    f"Before deleting this stack "
                    f"component, make sure to remove it "
                    f"from all stacks."
                )
            else:
                session.delete(stack_component)
        except NoResultFound as error:
            raise KeyError from error

        session.commit()
delete_tag(self, tag_name_or_id)

Deletes a tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to delete.

required

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_tag(
    self,
    tag_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a tag.

    Args:
        tag_name_or_id: name or id of the tag to delete.

    Raises:
        KeyError: specified ID or name not found.
    """
    with Session(self.engine) as session:
        tag = self._get_tag_schema(
            tag_name_or_id=tag_name_or_id, session=session
        )
        if tag is None:
            raise KeyError(
                f"Unable to delete tag with ID `{tag_name_or_id}`: "
                f"No tag with this ID found."
            )
        session.delete(tag)
        session.commit()
delete_tag_resource(self, tag_id, resource_id, resource_type)

Deletes a tag resource relationship.

Parameters:

Name Type Description Default
tag_id UUID

The ID of the tag to delete.

required
resource_id UUID

The ID of the resource to delete.

required
resource_type TaggableResourceTypes

The type of the resource to delete.

required

Exceptions:

Type Description
KeyError

specified ID not found.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_tag_resource(
    self,
    tag_id: UUID,
    resource_id: UUID,
    resource_type: TaggableResourceTypes,
) -> None:
    """Deletes a tag resource relationship.

    Args:
        tag_id: The ID of the tag to delete.
        resource_id: The ID of the resource to delete.
        resource_type: The type of the resource to delete.

    Raises:
        KeyError: specified ID not found.
    """
    with Session(self.engine) as session:
        tag_model = self._get_tag_model_schema(
            tag_id=tag_id,
            resource_id=resource_id,
            resource_type=resource_type,
            session=session,
        )
        if tag_model is None:
            raise KeyError(
                f"Unable to delete tag<>resource with IDs: "
                f"`tag_id`='{tag_id}' and `resource_id`='{resource_id}' "
                f"and `resource_type`='{resource_type.value}': No "
                "tag<>resource with these IDs found."
            )
        session.delete(tag_model)
        session.commit()
delete_trigger(self, trigger_id)

Delete a trigger.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger to delete.

required

Exceptions:

Type Description
KeyError

if the trigger doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_trigger(self, trigger_id: UUID) -> None:
    """Delete a trigger.

    Args:
        trigger_id: The ID of the trigger to delete.

    Raises:
        KeyError: if the trigger doesn't exist.
    """
    with Session(self.engine) as session:
        try:
            trigger = session.exec(
                select(TriggerSchema).where(TriggerSchema.id == trigger_id)
            ).one()

            if trigger is None:
                raise KeyError(f"Trigger with ID {trigger_id} not found.")
            session.delete(trigger)
        except NoResultFound as error:
            raise KeyError from error

        session.commit()
delete_trigger_execution(self, trigger_execution_id)

Delete a trigger execution.

Parameters:

Name Type Description Default
trigger_execution_id UUID

The ID of the trigger execution to delete.

required

Exceptions:

Type Description
KeyError

If the trigger execution doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_trigger_execution(self, trigger_execution_id: UUID) -> None:
    """Delete a trigger execution.

    Args:
        trigger_execution_id: The ID of the trigger execution to delete.

    Raises:
        KeyError: If the trigger execution doesn't exist.
    """
    with Session(self.engine) as session:
        try:
            execution = session.exec(
                select(TriggerExecutionSchema).where(
                    TriggerExecutionSchema.id == trigger_execution_id
                )
            ).one()

            session.delete(execution)
            session.commit()
        except NoResultFound:
            raise KeyError(
                f"Execution with ID {trigger_execution_id} not found."
            )
delete_user(self, user_name_or_id)

Deletes a user.

Parameters:

Name Type Description Default
user_name_or_id Union[str, uuid.UUID]

The name or the ID of the user to delete.

required

Exceptions:

Type Description
IllegalOperationError

If the user is the default user account or if the user already owns resources.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
    """Deletes a user.

    Args:
        user_name_or_id: The name or the ID of the user to delete.

    Raises:
        IllegalOperationError: If the user is the default user account or
            if the user already owns resources.
    """
    with Session(self.engine) as session:
        user = self._get_account_schema(
            user_name_or_id, session=session, service_account=False
        )
        if user.name == self._default_user_name:
            raise IllegalOperationError(
                "The default user account cannot be deleted."
            )
        if self._account_owns_resources(user, session=session):
            raise IllegalOperationError(
                "The user account has already been used to create "
                "other resources that it now owns and therefore cannot be "
                "deleted. Please delete all resources owned by the user "
                "account or consider deactivating it instead."
            )

        session.delete(user)
        session.commit()
delete_workspace(self, workspace_name_or_id)

Deletes a workspace.

Parameters:

Name Type Description Default
workspace_name_or_id Union[str, uuid.UUID]

Name or ID of the workspace to delete.

required

Exceptions:

Type Description
IllegalOperationError

If the workspace is the default workspace.

Source code in zenml/zen_stores/sql_zen_store.py
def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
    """Deletes a workspace.

    Args:
        workspace_name_or_id: Name or ID of the workspace to delete.

    Raises:
        IllegalOperationError: If the workspace is the default workspace.
    """
    with Session(self.engine) as session:
        # Check if workspace with the given name exists
        workspace = self._get_workspace_schema(
            workspace_name_or_id, session=session
        )
        if workspace.name == self._default_workspace_name:
            raise IllegalOperationError(
                "The default workspace cannot be deleted."
            )

        session.delete(workspace)
        session.commit()
entity_exists(self, entity_id, schema_class)

Check whether an entity exists in the database.

Parameters:

Name Type Description Default
entity_id UUID

The ID of the entity to check.

required
schema_class Type[~AnySchema]

The schema class.

required

Returns:

Type Description
bool

If the entity exists.

Source code in zenml/zen_stores/sql_zen_store.py
def entity_exists(
    self, entity_id: UUID, schema_class: Type[AnySchema]
) -> bool:
    """Check whether an entity exists in the database.

    Args:
        entity_id: The ID of the entity to check.
        schema_class: The schema class.

    Returns:
        If the entity exists.
    """
    with Session(self.engine) as session:
        schema = session.exec(
            select(schema_class.id).where(schema_class.id == entity_id)
        ).first()

        return False if schema is None else True
filter_and_paginate(session, query, table, filter_model, custom_schema_to_model_conversion=None, custom_fetch=None, hydrate=False) classmethod

Given a query, return a Page instance with a list of filtered Models.

Parameters:

Name Type Description Default
session Session

The SQLModel Session

required
query Union[sqlmodel.sql.expression.Select, sqlmodel.sql.expression.SelectOfScalar]

The query to execute

required
table Type[~AnySchema]

The table to select from

required
filter_model BaseFilter

The filter to use, including pagination and sorting

required
custom_schema_to_model_conversion Optional[Callable[..., ~AnyResponse]]

Callable to convert the schema into a model. This is used if the Model contains additional data that is not explicitly stored as a field or relationship on the model.

None
custom_fetch Optional[Callable[[sqlmodel.orm.session.Session, Union[sqlmodel.sql.expression.Select, sqlmodel.sql.expression.SelectOfScalar], zenml.models.v2.base.filter.BaseFilter], List[Any]]]

Custom callable to use to fetch items from the database for a given query. This is used if the items fetched from the database need to be processed differently (e.g. to perform additional filtering). The callable should take a Session, a Select query and a BaseFilterModel filter as arguments and return a List of items.

None
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[AnyResponse]

The Domain Model representation of the DB resource

Exceptions:

Type Description
ValueError

if the filtered page number is out of bounds.

RuntimeError

if the schema does not have a to_model method.

Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def filter_and_paginate(
    cls,
    session: Session,
    query: Union[Select[Any], SelectOfScalar[Any]],
    table: Type[AnySchema],
    filter_model: BaseFilter,
    custom_schema_to_model_conversion: Optional[
        Callable[..., AnyResponse]
    ] = None,
    custom_fetch: Optional[
        Callable[
            [
                Session,
                Union[Select[Any], SelectOfScalar[Any]],
                BaseFilter,
            ],
            List[Any],
        ]
    ] = None,
    hydrate: bool = False,
) -> Page[AnyResponse]:
    """Given a query, return a Page instance with a list of filtered Models.

    Args:
        session: The SQLModel Session
        query: The query to execute
        table: The table to select from
        filter_model: The filter to use, including pagination and sorting
        custom_schema_to_model_conversion: Callable to convert the schema
            into a model. This is used if the Model contains additional
            data that is not explicitly stored as a field or relationship
            on the model.
        custom_fetch: Custom callable to use to fetch items from the
            database for a given query. This is used if the items fetched
            from the database need to be processed differently (e.g. to
            perform additional filtering). The callable should take a
            `Session`, a `Select` query and a `BaseFilterModel` filter as
            arguments and return a `List` of items.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The Domain Model representation of the DB resource

    Raises:
        ValueError: if the filtered page number is out of bounds.
        RuntimeError: if the schema does not have a `to_model` method.
    """
    query = filter_model.apply_filter(query=query, table=table)

    # Get the total amount of items in the database for a given query
    custom_fetch_result: Optional[List[Any]] = None
    if custom_fetch:
        custom_fetch_result = custom_fetch(session, query, filter_model)
        total = len(custom_fetch_result)
    else:
        total = session.scalar(
            select([func.count("*")]).select_from(
                query.options(noload("*")).subquery()
            )
        )

    # Sorting
    column, operand = filter_model.sorting_params
    if operand == SorterOps.DESCENDING:
        query = query.order_by(desc(getattr(table, column)))
    else:
        query = query.order_by(asc(getattr(table, column)))

    # Get the total amount of pages in the database for a given query
    if total == 0:
        total_pages = 1
    else:
        total_pages = math.ceil(total / filter_model.size)

    if filter_model.page > total_pages:
        raise ValueError(
            f"Invalid page {filter_model.page}. The requested page size is "
            f"{filter_model.size} and there are a total of {total} items "
            f"for this query. The maximum page value therefore is "
            f"{total_pages}."
        )

    # Get a page of the actual data
    item_schemas: List[AnySchema]
    if custom_fetch:
        assert custom_fetch_result is not None
        item_schemas = custom_fetch_result
        # select the items in the current page
        item_schemas = item_schemas[
            filter_model.offset : filter_model.offset + filter_model.size
        ]
    else:
        item_schemas = (
            session.exec(
                query.limit(filter_model.size).offset(filter_model.offset)
            )
            .unique()
            .all()
        )

    # Convert this page of items from schemas to models.
    items: List[AnyResponse] = []
    for schema in item_schemas:
        # If a custom conversion function is provided, use it.
        if custom_schema_to_model_conversion:
            items.append(custom_schema_to_model_conversion(schema))
            continue
        # Otherwise, try to use the `to_model` method of the schema.
        to_model = getattr(schema, "to_model", None)
        if callable(to_model):
            items.append(to_model(include_metadata=hydrate))
            continue
        # If neither of the above work, raise an error.
        raise RuntimeError(
            f"Cannot convert schema `{schema.__class__.__name__}` to model "
            "since it does not have a `to_model` method."
        )

    return Page[Any](
        total=total,
        total_pages=total_pages,
        items=items,
        index=filter_model.page,
        max_size=filter_model.size,
    )
get_api_key(self, service_account_id, api_key_name_or_id, hydrate=True)

Get an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to fetch the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
APIKeyResponse

The API key with the given ID.

Source code in zenml/zen_stores/sql_zen_store.py
def get_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    hydrate: bool = True,
) -> APIKeyResponse:
    """Get an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to fetch
            the API key.
        api_key_name_or_id: The name or ID of the API key to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The API key with the given ID.
    """
    with Session(self.engine) as session:
        api_key = self._get_api_key(
            service_account_id=service_account_id,
            api_key_name_or_id=api_key_name_or_id,
            session=session,
        )
        return api_key.to_model(include_metadata=hydrate)
get_artifact(self, artifact_id, hydrate=True)

Gets an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactResponse

The artifact.

Exceptions:

Type Description
KeyError

if the artifact doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_artifact(
    self, artifact_id: UUID, hydrate: bool = True
) -> ArtifactResponse:
    """Gets an artifact.

    Args:
        artifact_id: The ID of the artifact to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact.

    Raises:
        KeyError: if the artifact doesn't exist.
    """
    with Session(self.engine) as session:
        artifact = session.exec(
            select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
        ).first()
        if artifact is None:
            raise KeyError(
                f"Unable to get artifact with ID {artifact_id}: No "
                "artifact with this ID found."
            )
        return artifact.to_model(include_metadata=hydrate)
get_artifact_version(self, artifact_version_id, hydrate=True)

Gets an artifact version.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactVersionResponse

The artifact version.

Exceptions:

Type Description
KeyError

if the artifact version doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_artifact_version(
    self, artifact_version_id: UUID, hydrate: bool = True
) -> ArtifactVersionResponse:
    """Gets an artifact version.

    Args:
        artifact_version_id: The ID of the artifact version to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact version.

    Raises:
        KeyError: if the artifact version doesn't exist.
    """
    with Session(self.engine) as session:
        artifact_version = session.exec(
            select(ArtifactVersionSchema).where(
                ArtifactVersionSchema.id == artifact_version_id
            )
        ).first()
        if artifact_version is None:
            raise KeyError(
                f"Unable to get artifact version with ID "
                f"{artifact_version_id}: No artifact version with this ID "
                f"found."
            )
        return artifact_version.to_model(include_metadata=hydrate)
get_artifact_visualization(self, artifact_visualization_id, hydrate=True)

Gets an artifact visualization.

Parameters:

Name Type Description Default
artifact_visualization_id UUID

The ID of the artifact visualization to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactVisualizationResponse

The artifact visualization.

Exceptions:

Type Description
KeyError

if the code reference doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_artifact_visualization(
    self, artifact_visualization_id: UUID, hydrate: bool = True
) -> ArtifactVisualizationResponse:
    """Gets an artifact visualization.

    Args:
        artifact_visualization_id: The ID of the artifact visualization to
            get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact visualization.

    Raises:
        KeyError: if the code reference doesn't exist.
    """
    with Session(self.engine) as session:
        artifact_visualization = session.exec(
            select(ArtifactVisualizationSchema).where(
                ArtifactVisualizationSchema.id == artifact_visualization_id
            )
        ).first()
        if artifact_visualization is None:
            raise KeyError(
                f"Unable to get artifact visualization with ID "
                f"{artifact_visualization_id}: "
                f"No artifact visualization with this ID found."
            )
        return artifact_visualization.to_model(include_metadata=hydrate)
get_auth_user(self, user_name_or_id)

Gets the auth model to a specific user.

Parameters:

Name Type Description Default
user_name_or_id Union[str, uuid.UUID]

The name or ID of the user to get.

required

Returns:

Type Description
UserAuthModel

The requested user, if it was found.

Source code in zenml/zen_stores/sql_zen_store.py
def get_auth_user(
    self, user_name_or_id: Union[str, UUID]
) -> UserAuthModel:
    """Gets the auth model to a specific user.

    Args:
        user_name_or_id: The name or ID of the user to get.

    Returns:
        The requested user, if it was found.
    """
    with Session(self.engine) as session:
        user = self._get_account_schema(
            user_name_or_id, session=session, service_account=False
        )
        return UserAuthModel(
            id=user.id,
            name=user.name,
            full_name=user.full_name,
            email_opted_in=user.email_opted_in,
            active=user.active,
            created=user.created,
            updated=user.updated,
            password=user.password,
            activation_token=user.activation_token,
            is_service_account=False,
        )
get_authorized_device(self, device_id, hydrate=True)

Gets a specific OAuth 2.0 authorized device.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
OAuthDeviceResponse

The requested device, if it was found.

Exceptions:

Type Description
KeyError

If no device with the given ID exists.

Source code in zenml/zen_stores/sql_zen_store.py
def get_authorized_device(
    self, device_id: UUID, hydrate: bool = True
) -> OAuthDeviceResponse:
    """Gets a specific OAuth 2.0 authorized device.

    Args:
        device_id: The ID of the device to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested device, if it was found.

    Raises:
        KeyError: If no device with the given ID exists.
    """
    with Session(self.engine) as session:
        device = session.exec(
            select(OAuthDeviceSchema).where(
                OAuthDeviceSchema.id == device_id
            )
        ).first()
        if device is None:
            raise KeyError(
                f"Unable to get device with ID {device_id}: No device with "
                "this ID found."
            )

        return device.to_model(include_metadata=hydrate)
get_build(self, build_id, hydrate=True)

Get a build with a given ID.

Parameters:

Name Type Description Default
build_id UUID

ID of the build.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineBuildResponse

The build.

Exceptions:

Type Description
KeyError

If the build does not exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_build(
    self, build_id: UUID, hydrate: bool = True
) -> PipelineBuildResponse:
    """Get a build with a given ID.

    Args:
        build_id: ID of the build.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The build.

    Raises:
        KeyError: If the build does not exist.
    """
    with Session(self.engine) as session:
        # Check if build with the given ID exists
        build = session.exec(
            select(PipelineBuildSchema).where(
                PipelineBuildSchema.id == build_id
            )
        ).first()
        if build is None:
            raise KeyError(
                f"Unable to get build with ID '{build_id}': "
                "No build with this ID found."
            )

        return build.to_model(include_metadata=hydrate)
get_code_reference(self, code_reference_id, hydrate=True)

Gets a code reference.

Parameters:

Name Type Description Default
code_reference_id UUID

The ID of the code reference to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
CodeReferenceResponse

The code reference.

Exceptions:

Type Description
KeyError

if the code reference doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_code_reference(
    self, code_reference_id: UUID, hydrate: bool = True
) -> CodeReferenceResponse:
    """Gets a code reference.

    Args:
        code_reference_id: The ID of the code reference to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The code reference.

    Raises:
        KeyError: if the code reference doesn't exist.
    """
    with Session(self.engine) as session:
        code_reference = session.exec(
            select(CodeReferenceSchema).where(
                CodeRepositorySchema.id == code_reference_id
            )
        ).first()
        if code_reference is None:
            raise KeyError(
                f"Unable to get code reference with ID "
                f"{code_reference_id}: "
                f"No code reference with this ID found."
            )
        return code_reference.to_model(include_metadata=hydrate)
get_code_repository(self, code_repository_id, hydrate=True)

Gets a specific code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
CodeRepositoryResponse

The requested code repository, if it was found.

Exceptions:

Type Description
KeyError

If no code repository with the given ID exists.

Source code in zenml/zen_stores/sql_zen_store.py
def get_code_repository(
    self, code_repository_id: UUID, hydrate: bool = True
) -> CodeRepositoryResponse:
    """Gets a specific code repository.

    Args:
        code_repository_id: The ID of the code repository to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested code repository, if it was found.

    Raises:
        KeyError: If no code repository with the given ID exists.
    """
    with Session(self.engine) as session:
        repo = session.exec(
            select(CodeRepositorySchema).where(
                CodeRepositorySchema.id == code_repository_id
            )
        ).first()
        if repo is None:
            raise KeyError(
                f"Unable to get code repository with ID "
                f"'{code_repository_id}': No code repository with this "
                "ID found."
            )

        return repo.to_model(include_metadata=hydrate)
get_deployment(self, deployment_id, hydrate=True)

Get a deployment with a given ID.

Parameters:

Name Type Description Default
deployment_id UUID

ID of the deployment.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineDeploymentResponse

The deployment.

Exceptions:

Type Description
KeyError

If the deployment does not exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_deployment(
    self, deployment_id: UUID, hydrate: bool = True
) -> PipelineDeploymentResponse:
    """Get a deployment with a given ID.

    Args:
        deployment_id: ID of the deployment.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The deployment.

    Raises:
        KeyError: If the deployment does not exist.
    """
    with Session(self.engine) as session:
        # Check if deployment with the given ID exists
        deployment = session.exec(
            select(PipelineDeploymentSchema).where(
                PipelineDeploymentSchema.id == deployment_id
            )
        ).first()
        if deployment is None:
            raise KeyError(
                f"Unable to get deployment with ID '{deployment_id}': "
                "No deployment with this ID found."
            )

        return deployment.to_model(include_metadata=hydrate)
get_deployment_id(self)

Get the ID of the deployment.

Returns:

Type Description
UUID

The ID of the deployment.

Exceptions:

Type Description
KeyError

If the deployment ID could not be loaded from the database.

Source code in zenml/zen_stores/sql_zen_store.py
def get_deployment_id(self) -> UUID:
    """Get the ID of the deployment.

    Returns:
        The ID of the deployment.

    Raises:
        KeyError: If the deployment ID could not be loaded from the
            database.
    """
    # Fetch the deployment ID from the database
    with Session(self.engine) as session:
        identity = session.exec(select(IdentitySchema)).first()

        if identity is None:
            raise KeyError(
                "The deployment ID could not be loaded from the database."
            )
        return identity.id
get_entity_by_id(self, entity_id, schema_class)

Get an entity by ID.

Parameters:

Name Type Description Default
entity_id UUID

The ID of the entity to get.

required
schema_class Type[~AnySchema]

The schema class.

required

Exceptions:

Type Description
RuntimeError

If the schema to model conversion failed.

Returns:

Type Description
Optional[~AnyIdentifiedResponse]

The entity if it exists, None otherwise

Source code in zenml/zen_stores/sql_zen_store.py
def get_entity_by_id(
    self, entity_id: UUID, schema_class: Type[AnySchema]
) -> Optional[AnyIdentifiedResponse]:
    """Get an entity by ID.

    Args:
        entity_id: The ID of the entity to get.
        schema_class: The schema class.

    Raises:
        RuntimeError: If the schema to model conversion failed.

    Returns:
        The entity if it exists, None otherwise
    """
    with Session(self.engine) as session:
        schema = session.exec(
            select(schema_class).where(schema_class.id == entity_id)
        ).first()

        if not schema:
            return None

        to_model = getattr(schema, "to_model", None)
        if callable(to_model):
            return cast(AnyIdentifiedResponse, to_model(hydrate=True))
        else:
            raise RuntimeError("Unable to convert schema to model.")
get_event_source(self, event_source_id, hydrate=True)

Get an event_source by ID.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
EventSourceResponse

The event_source.

Source code in zenml/zen_stores/sql_zen_store.py
def get_event_source(
    self,
    event_source_id: UUID,
    hydrate: bool = True,
) -> EventSourceResponse:
    """Get an event_source by ID.

    Args:
        event_source_id: The ID of the event_source to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The event_source.
    """
    with Session(self.engine) as session:
        return self._get_event_source(
            event_source_id=event_source_id, session=session
        ).to_model(include_metadata=hydrate, include_resources=True)
get_flavor(self, flavor_id, hydrate=True)

Get a flavor by ID.

Parameters:

Name Type Description Default
flavor_id UUID

The ID of the flavor to fetch.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
FlavorResponse

The stack component flavor.

Exceptions:

Type Description
KeyError

if the stack component flavor doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_flavor(
    self, flavor_id: UUID, hydrate: bool = True
) -> FlavorResponse:
    """Get a flavor by ID.

    Args:
        flavor_id: The ID of the flavor to fetch.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack component flavor.

    Raises:
        KeyError: if the stack component flavor doesn't exist.
    """
    with Session(self.engine) as session:
        flavor_in_db = session.exec(
            select(FlavorSchema).where(FlavorSchema.id == flavor_id)
        ).first()
        if flavor_in_db is None:
            raise KeyError(f"Flavor with ID {flavor_id} not found.")
        return flavor_in_db.to_model(include_metadata=hydrate)
get_internal_api_key(self, api_key_id, hydrate=True)

Get internal details for an API key by its unique ID.

Parameters:

Name Type Description Default
api_key_id UUID

The ID of the API key to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
APIKeyInternalResponse

The internal details for the API key with the given ID.

Exceptions:

Type Description
KeyError

if the API key doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_internal_api_key(
    self, api_key_id: UUID, hydrate: bool = True
) -> APIKeyInternalResponse:
    """Get internal details for an API key by its unique ID.

    Args:
        api_key_id: The ID of the API key to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The internal details for the API key with the given ID.

    Raises:
        KeyError: if the API key doesn't exist.
    """
    with Session(self.engine) as session:
        api_key = session.exec(
            select(APIKeySchema).where(APIKeySchema.id == api_key_id)
        ).first()
        if api_key is None:
            raise KeyError(f"API key with ID {api_key_id} not found.")
        return api_key.to_internal_model(hydrate=hydrate)
get_internal_authorized_device(self, device_id=None, client_id=None, hydrate=True)

Gets a specific OAuth 2.0 authorized device for internal use.

Parameters:

Name Type Description Default
client_id Optional[uuid.UUID]

The client ID of the device to get.

None
device_id Optional[uuid.UUID]

The ID of the device to get.

None
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
OAuthDeviceInternalResponse

The requested device, if it was found.

Exceptions:

Type Description
KeyError

If no device with the given client ID exists.

ValueError

If neither device ID nor client ID are provided.

Source code in zenml/zen_stores/sql_zen_store.py
def get_internal_authorized_device(
    self,
    device_id: Optional[UUID] = None,
    client_id: Optional[UUID] = None,
    hydrate: bool = True,
) -> OAuthDeviceInternalResponse:
    """Gets a specific OAuth 2.0 authorized device for internal use.

    Args:
        client_id: The client ID of the device to get.
        device_id: The ID of the device to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested device, if it was found.

    Raises:
        KeyError: If no device with the given client ID exists.
        ValueError: If neither device ID nor client ID are provided.
    """
    with Session(self.engine) as session:
        if device_id is not None:
            device = session.exec(
                select(OAuthDeviceSchema).where(
                    OAuthDeviceSchema.id == device_id
                )
            ).first()
        elif client_id is not None:
            device = session.exec(
                select(OAuthDeviceSchema).where(
                    OAuthDeviceSchema.client_id == client_id
                )
            ).first()
        else:
            raise ValueError(
                "Either device ID or client ID must be provided."
            )
        if device is None:
            raise KeyError(
                f"Unable to get device with client ID {client_id}: No "
                "device with this client ID found."
            )

        return device.to_internal_model(hydrate=hydrate)
get_logs(self, logs_id, hydrate=True)

Gets logs with the given ID.

Parameters:

Name Type Description Default
logs_id UUID

The ID of the logs to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
LogsResponse

The logs.

Exceptions:

Type Description
KeyError

if the logs doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_logs(self, logs_id: UUID, hydrate: bool = True) -> LogsResponse:
    """Gets logs with the given ID.

    Args:
        logs_id: The ID of the logs to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The logs.

    Raises:
        KeyError: if the logs doesn't exist.
    """
    with Session(self.engine) as session:
        logs = session.exec(
            select(LogsSchema).where(LogsSchema.id == logs_id)
        ).first()
        if logs is None:
            raise KeyError(
                f"Unable to get logs with ID "
                f"{logs_id}: "
                f"No logs with this ID found."
            )
        return logs.to_model(include_metadata=hydrate)
get_model(self, model_name_or_id, hydrate=True)

Get an existing model.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model to be retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Exceptions:

Type Description
KeyError

specified ID or name not found.

Returns:

Type Description
ModelResponse

The model of interest.

Source code in zenml/zen_stores/sql_zen_store.py
def get_model(
    self,
    model_name_or_id: Union[str, UUID],
    hydrate: bool = True,
) -> ModelResponse:
    """Get an existing model.

    Args:
        model_name_or_id: name or id of the model to be retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Raises:
        KeyError: specified ID or name not found.

    Returns:
        The model of interest.
    """
    with Session(self.engine) as session:
        model = self._get_model_schema(
            model_name_or_id=model_name_or_id, session=session
        )
        if model is None:
            raise KeyError(
                f"Unable to get model with ID `{model_name_or_id}`: "
                f"No model with this ID found."
            )
        return model.to_model(include_metadata=hydrate)
get_model_version(self, model_version_id, hydrate=True)

Get an existing model version.

Parameters:

Name Type Description Default
model_version_id UUID

name, id, stage or number of the model version to be retrieved. If skipped - latest is retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ModelVersionResponse

The model version of interest.

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/sql_zen_store.py
def get_model_version(
    self, model_version_id: UUID, hydrate: bool = True
) -> ModelVersionResponse:
    """Get an existing model version.

    Args:
        model_version_id: name, id, stage or number of the model version to
            be retrieved. If skipped - latest is retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The model version of interest.

    Raises:
        KeyError: specified ID or name not found.
    """
    with Session(self.engine) as session:
        model_version = self._get_schema_by_name_or_id(
            object_name_or_id=model_version_id,
            schema_class=ModelVersionSchema,
            schema_name="model_version",
            session=session,
        )
        if model_version is None:
            raise KeyError(
                f"Unable to get model version with ID "
                f"`{model_version_id}`: No model version with this "
                f"ID found."
            )
        return model_version.to_model(include_metadata=hydrate)
get_or_create_run(self, pipeline_run)

Gets or creates a pipeline run.

If a run with the same ID or name already exists, it is returned. Otherwise, a new run is created.

Parameters:

Name Type Description Default
pipeline_run PipelineRunRequest

The pipeline run to get or create.

required
noqa: DAR401

Exceptions:

Type Description
ValueError

If the request does not contain an orchestrator run ID.

EntityExistsError

If a run with the same name already exists.

RuntimeError

If the run fetching failed unexpectedly.

Returns:

Type Description
Tuple[zenml.models.v2.core.pipeline_run.PipelineRunResponse, bool]

The pipeline run, and a boolean indicating whether the run was created or not.

Source code in zenml/zen_stores/sql_zen_store.py
def get_or_create_run(
    self, pipeline_run: PipelineRunRequest
) -> Tuple[PipelineRunResponse, bool]:
    """Gets or creates a pipeline run.

    If a run with the same ID or name already exists, it is returned.
    Otherwise, a new run is created.

    Args:
        pipeline_run: The pipeline run to get or create.

    # noqa: DAR401
    Raises:
        ValueError: If the request does not contain an orchestrator run ID.
        EntityExistsError: If a run with the same name already exists.
        RuntimeError: If the run fetching failed unexpectedly.

    Returns:
        The pipeline run, and a boolean indicating whether the run was
        created or not.
    """
    if not pipeline_run.orchestrator_run_id:
        raise ValueError(
            "Unable to get or create run for request with missing "
            "orchestrator run ID."
        )

    try:
        return (
            self._replace_placeholder_run(pipeline_run=pipeline_run),
            True,
        )
    except KeyError:
        # We were not able to find/replace a placeholder run. This could be
        # due to one of the following three reasons:
        # (1) There never was a placeholder run for the deployment. This is
        #     the case if the user ran the pipeline on a schedule.
        # (2) There was a placeholder run, but a previous pipeline run
        #     already used it. This is the case if users rerun a pipeline
        #     run e.g. from the orchestrator UI, as they will use the same
        #     deployment_id with a new orchestrator_run_id.
        # (3) A step of the same pipeline run already replaced the
        #     placeholder run.
        pass

    try:
        # We now try to create a new run. The following will happen in the
        # three cases described above:
        # (1) The behavior depends on whether we're the first step of the
        #     pipeline run that's trying to create the run. If yes, the
        #     `self.create_run(...)` will succeed. If no, a run with the
        #     same deployment_id and orchestrator_run_id already exists and
        #     the `self.create_run(...)` call will fail due to the unique
        #     constraint on those columns.
        # (2) Same as (1).
        # (3) A step of the same pipeline run replaced the placeholder
        #     run, which now contains the deployment_id and
        #     orchestrator_run_id of the run that we're trying to create.
        #     -> The `self.create_run(...) call will fail due to the unique
        #     constraint on those columns.
        return self.create_run(pipeline_run), True
    except (EntityExistsError, IntegrityError) as create_error:
        # Creating the run failed with an
        # - IntegrityError: This happens when we violated a unique
        #   constraint, which in turn means a run with the same
        #   deployment_id and orchestrator_run_id exists. We now fetch and
        #   return that run.
        # - EntityExistsError: This happens when a run with the same name
        #   already exists. This could be either a different run (in which
        #   case we want to fail) or a run created by a step of the same
        #   pipeline run (in which case we want to return it).
        # Note: The IntegrityError might also be raised when other unique
        # constraints get violated. The only other such constraint is the
        # primary key constraint on the run ID, which means we randomly
        # generated an existing UUID. In this case the call below will fail,
        # but the chance of that happening is so low we don't handle it.
        try:
            return (
                self._get_run_by_orchestrator_run_id(
                    orchestrator_run_id=pipeline_run.orchestrator_run_id,
                    deployment_id=pipeline_run.deployment,
                ),
                False,
            )
        except KeyError:
            if isinstance(create_error, EntityExistsError):
                # There was a run with the same name which does not share
                # the deployment_id and orchestrator_run_id -> We fail with
                # the error that run names must be unique.
                raise create_error from None

            # This should never happen as the run creation failed with an
            # IntegrityError which means a run with the deployment_id and
            # orchestrator_run_id exists.
            raise RuntimeError(
                f"Failed to get or create run: {create_error}"
            )
get_pipeline(self, pipeline_id, hydrate=True)

Get a pipeline with a given ID.

Parameters:

Name Type Description Default
pipeline_id UUID

ID of the pipeline.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineResponse

The pipeline.

Exceptions:

Type Description
KeyError

if the pipeline does not exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_pipeline(
    self, pipeline_id: UUID, hydrate: bool = True
) -> PipelineResponse:
    """Get a pipeline with a given ID.

    Args:
        pipeline_id: ID of the pipeline.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The pipeline.

    Raises:
        KeyError: if the pipeline does not exist.
    """
    with Session(self.engine) as session:
        # Check if pipeline with the given ID exists
        pipeline = session.exec(
            select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
        ).first()
        if pipeline is None:
            raise KeyError(
                f"Unable to get pipeline with ID '{pipeline_id}': "
                "No pipeline with this ID found."
            )

        return pipeline.to_model(include_metadata=hydrate)
get_run(self, run_name_or_id, hydrate=True)

Gets a pipeline run.

Parameters:

Name Type Description Default
run_name_or_id Union[str, uuid.UUID]

The name or ID of the pipeline run to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineRunResponse

The pipeline run.

Source code in zenml/zen_stores/sql_zen_store.py
def get_run(
    self, run_name_or_id: Union[str, UUID], hydrate: bool = True
) -> PipelineRunResponse:
    """Gets a pipeline run.

    Args:
        run_name_or_id: The name or ID of the pipeline run to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The pipeline run.
    """
    with Session(self.engine) as session:
        return self._get_run_schema(
            run_name_or_id, session=session
        ).to_model(include_metadata=hydrate)
get_run_metadata(self, run_metadata_id, hydrate=True)

Gets run metadata with the given ID.

Parameters:

Name Type Description Default
run_metadata_id UUID

The ID of the run metadata to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
RunMetadataResponse

The run metadata.

Exceptions:

Type Description
KeyError

if the run metadata doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_run_metadata(
    self, run_metadata_id: UUID, hydrate: bool = True
) -> RunMetadataResponse:
    """Gets run metadata with the given ID.

    Args:
        run_metadata_id: The ID of the run metadata to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The run metadata.

    Raises:
        KeyError: if the run metadata doesn't exist.
    """
    with Session(self.engine) as session:
        run_metadata = session.exec(
            select(RunMetadataSchema).where(
                RunMetadataSchema.id == run_metadata_id
            )
        ).first()
        if run_metadata is None:
            raise KeyError(
                f"Unable to get run metadata with ID "
                f"{run_metadata_id}: "
                f"No run metadata with this ID found."
            )
        return run_metadata.to_model(include_metadata=hydrate)
get_run_step(self, step_run_id, hydrate=True)

Get a step run by ID.

Parameters:

Name Type Description Default
step_run_id UUID

The ID of the step run to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
StepRunResponse

The step run.

Exceptions:

Type Description
KeyError

if the step run doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_run_step(
    self, step_run_id: UUID, hydrate: bool = True
) -> StepRunResponse:
    """Get a step run by ID.

    Args:
        step_run_id: The ID of the step run to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The step run.

    Raises:
        KeyError: if the step run doesn't exist.
    """
    with Session(self.engine) as session:
        step_run = session.exec(
            select(StepRunSchema).where(StepRunSchema.id == step_run_id)
        ).first()
        if step_run is None:
            raise KeyError(
                f"Unable to get step run with ID {step_run_id}: No step "
                "run with this ID found."
            )
        return step_run.to_model(include_metadata=hydrate)
get_schedule(self, schedule_id, hydrate=True)

Get a schedule with a given ID.

Parameters:

Name Type Description Default
schedule_id UUID

ID of the schedule.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ScheduleResponse

The schedule.

Exceptions:

Type Description
KeyError

if the schedule does not exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_schedule(
    self, schedule_id: UUID, hydrate: bool = True
) -> ScheduleResponse:
    """Get a schedule with a given ID.

    Args:
        schedule_id: ID of the schedule.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The schedule.

    Raises:
        KeyError: if the schedule does not exist.
    """
    with Session(self.engine) as session:
        # Check if schedule with the given ID exists
        schedule = session.exec(
            select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
        ).first()
        if schedule is None:
            raise KeyError(
                f"Unable to get schedule with ID '{schedule_id}': "
                "No schedule with this ID found."
            )
        return schedule.to_model(include_metadata=hydrate)
get_secret(self, secret_id, hydrate=True)

Get a secret by ID.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to fetch.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
SecretResponse

The secret.

Exceptions:

Type Description
KeyError

if the secret doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_secret(
    self, secret_id: UUID, hydrate: bool = True
) -> SecretResponse:
    """Get a secret by ID.

    Args:
        secret_id: The ID of the secret to fetch.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The secret.

    Raises:
        KeyError: if the secret doesn't exist.
    """
    with Session(self.engine) as session:
        secret_in_db = session.exec(
            select(SecretSchema).where(SecretSchema.id == secret_id)
        ).first()
        if secret_in_db is None:
            raise KeyError(f"Secret with ID {secret_id} not found.")
        secret_model = secret_in_db.to_model(include_metadata=hydrate)

    secret_model.set_secrets(self._get_secret_values(secret_id=secret_id))

    return secret_model
get_service_account(self, service_account_name_or_id, hydrate=True)

Gets a specific service account.

Raises a KeyError in case a service account with that id does not exist.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or ID of the service account to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ServiceAccountResponse

The requested service account, if it was found.

Source code in zenml/zen_stores/sql_zen_store.py
def get_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
    hydrate: bool = True,
) -> ServiceAccountResponse:
    """Gets a specific service account.

    Raises a KeyError in case a service account with that id does not exist.

    Args:
        service_account_name_or_id: The name or ID of the service account to
            get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested service account, if it was found.
    """
    with Session(self.engine) as session:
        account = self._get_account_schema(
            service_account_name_or_id,
            session=session,
            service_account=True,
        )

        return account.to_service_account_model(include_metadata=hydrate)
get_service_connector(self, service_connector_id, hydrate=True)

Gets a specific service connector.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ServiceConnectorResponse

The requested service connector, if it was found.

Exceptions:

Type Description
KeyError

If no service connector with the given ID exists.

Source code in zenml/zen_stores/sql_zen_store.py
def get_service_connector(
    self, service_connector_id: UUID, hydrate: bool = True
) -> ServiceConnectorResponse:
    """Gets a specific service connector.

    Args:
        service_connector_id: The ID of the service connector to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested service connector, if it was found.

    Raises:
        KeyError: If no service connector with the given ID exists.
    """
    with Session(self.engine) as session:
        service_connector = session.exec(
            select(ServiceConnectorSchema).where(
                ServiceConnectorSchema.id == service_connector_id
            )
        ).first()

        if service_connector is None:
            raise KeyError(
                f"Service connector with ID {service_connector_id} not "
                "found."
            )

        connector = service_connector.to_model(include_metadata=hydrate)
        self._populate_connector_type(connector)
        return connector
get_service_connector_client(self, service_connector_id, resource_type=None, resource_id=None)

Get a service connector client for a service connector and given resource.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the base service connector to use.

required
resource_type Optional[str]

The type of resource to get a client for.

None
resource_id Optional[str]

The ID of the resource to get a client for.

None

Returns:

Type Description
ServiceConnectorResponse

A service connector client that can be used to access the given resource.

Source code in zenml/zen_stores/sql_zen_store.py
def get_service_connector_client(
    self,
    service_connector_id: UUID,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
) -> ServiceConnectorResponse:
    """Get a service connector client for a service connector and given resource.

    Args:
        service_connector_id: The ID of the base service connector to use.
        resource_type: The type of resource to get a client for.
        resource_id: The ID of the resource to get a client for.

    Returns:
        A service connector client that can be used to access the given
        resource.
    """
    connector = self.get_service_connector(service_connector_id)

    connector_instance = service_connector_registry.instantiate_connector(
        model=connector
    )

    # Fetch the connector client
    connector_client = connector_instance.get_connector_client(
        resource_type=resource_type,
        resource_id=resource_id,
    )

    # Return the model for the connector client
    connector = connector_client.to_response_model(
        user=connector.user,
        workspace=connector.workspace,
        description=connector.description,
        labels=connector.labels,
    )

    self._populate_connector_type(connector)

    return connector
get_service_connector_type(self, connector_type)

Returns the requested service connector type.

Parameters:

Name Type Description Default
connector_type str

the service connector type identifier.

required

Returns:

Type Description
ServiceConnectorTypeModel

The requested service connector type.

Source code in zenml/zen_stores/sql_zen_store.py
def get_service_connector_type(
    self,
    connector_type: str,
) -> ServiceConnectorTypeModel:
    """Returns the requested service connector type.

    Args:
        connector_type: the service connector type identifier.

    Returns:
        The requested service connector type.
    """
    return service_connector_registry.get_service_connector_type(
        connector_type
    )
get_stack(self, stack_id, hydrate=True)

Get a stack by its unique ID.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
StackResponse

The stack with the given ID.

Exceptions:

Type Description
KeyError

if the stack doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_stack(self, stack_id: UUID, hydrate: bool = True) -> StackResponse:
    """Get a stack by its unique ID.

    Args:
        stack_id: The ID of the stack to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack with the given ID.

    Raises:
        KeyError: if the stack doesn't exist.
    """
    with Session(self.engine) as session:
        stack = session.exec(
            select(StackSchema).where(StackSchema.id == stack_id)
        ).first()

        if stack is None:
            raise KeyError(f"Stack with ID {stack_id} not found.")
        return stack.to_model(include_metadata=hydrate)
get_stack_component(self, component_id, hydrate=True)

Get a stack component by ID.

Parameters:

Name Type Description Default
component_id UUID

The ID of the stack component to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ComponentResponse

The stack component.

Exceptions:

Type Description
KeyError

if the stack component doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_stack_component(
    self, component_id: UUID, hydrate: bool = True
) -> ComponentResponse:
    """Get a stack component by ID.

    Args:
        component_id: The ID of the stack component to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack component.

    Raises:
        KeyError: if the stack component doesn't exist.
    """
    with Session(self.engine) as session:
        stack_component = session.exec(
            select(StackComponentSchema).where(
                StackComponentSchema.id == component_id
            )
        ).first()

        if stack_component is None:
            raise KeyError(
                f"Stack component with ID {component_id} not found."
            )

        return stack_component.to_model(include_metadata=hydrate)
get_store_info(self)

Get information about the store.

Returns:

Type Description
ServerModel

Information about the store.

Source code in zenml/zen_stores/sql_zen_store.py
def get_store_info(self) -> ServerModel:
    """Get information about the store.

    Returns:
        Information about the store.
    """
    model = super().get_store_info()
    sql_url = make_url(self.config.url)
    model.database_type = ServerDatabaseType(sql_url.drivername)
    # Fetch the deployment ID from the database and use it to replace
    # the one fetched from the global configuration
    model.id = self.get_deployment_id()
    return model
get_tag(self, tag_name_or_id, hydrate=True)

Get an existing tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to be retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TagResponse

The tag of interest.

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/sql_zen_store.py
def get_tag(
    self, tag_name_or_id: Union[str, UUID], hydrate: bool = True
) -> TagResponse:
    """Get an existing tag.

    Args:
        tag_name_or_id: name or id of the tag to be retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The tag of interest.

    Raises:
        KeyError: specified ID or name not found.
    """
    with Session(self.engine) as session:
        tag = self._get_tag_schema(
            tag_name_or_id=tag_name_or_id, session=session
        )
        if tag is None:
            raise KeyError(
                f"Unable to get tag with ID `{tag_name_or_id}`: "
                f"No tag with this ID found."
            )
        return tag.to_model(include_metadata=hydrate)
get_trigger(self, trigger_id, hydrate=True)

Get a trigger by its unique ID.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TriggerResponse

The trigger with the given ID.

Exceptions:

Type Description
KeyError

if the trigger doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_trigger(
    self, trigger_id: UUID, hydrate: bool = True
) -> TriggerResponse:
    """Get a trigger by its unique ID.

    Args:
        trigger_id: The ID of the trigger to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The trigger with the given ID.

    Raises:
        KeyError: if the trigger doesn't exist.
    """
    with Session(self.engine) as session:
        trigger = session.exec(
            select(TriggerSchema).where(TriggerSchema.id == trigger_id)
        ).first()

        if trigger is None:
            raise KeyError(f"Trigger with ID {trigger_id} not found.")
        return trigger.to_model(
            include_metadata=hydrate, include_resources=hydrate
        )
get_trigger_execution(self, trigger_execution_id, hydrate=True)

Get an trigger execution by ID.

Parameters:

Name Type Description Default
trigger_execution_id UUID

The ID of the trigger execution to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TriggerExecutionResponse

The trigger execution.

Exceptions:

Type Description
KeyError

If the trigger execution doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_trigger_execution(
    self,
    trigger_execution_id: UUID,
    hydrate: bool = True,
) -> TriggerExecutionResponse:
    """Get an trigger execution by ID.

    Args:
        trigger_execution_id: The ID of the trigger execution to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The trigger execution.

    Raises:
        KeyError: If the trigger execution doesn't exist.
    """
    with Session(self.engine) as session:
        execution = session.exec(
            select(TriggerExecutionSchema).where(
                TriggerExecutionSchema.id == trigger_execution_id
            )
        ).first()

        if execution is None:
            raise KeyError(
                f"Trigger execution with ID {trigger_execution_id} not found."
            )
        return execution.to_model(
            include_metadata=hydrate, include_resources=True
        )
get_user(self, user_name_or_id=None, include_private=False, hydrate=True)

Gets a specific user, when no id is specified the active user is returned.

noqa: DAR401
noqa: DAR402

Raises a KeyError in case a user with that name or id does not exist.

For backwards-compatibility reasons, this method can also be called to fetch service accounts by their ID.

Parameters:

Name Type Description Default
user_name_or_id Union[str, uuid.UUID]

The name or ID of the user to get.

None
include_private bool

Whether to include private user information

False
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
UserResponse

The requested user, if it was found.

Exceptions:

Type Description
KeyError

If the user does not exist.

Source code in zenml/zen_stores/sql_zen_store.py
def get_user(
    self,
    user_name_or_id: Optional[Union[str, UUID]] = None,
    include_private: bool = False,
    hydrate: bool = True,
) -> UserResponse:
    """Gets a specific user, when no id is specified the active user is returned.

    # noqa: DAR401
    # noqa: DAR402

    Raises a KeyError in case a user with that name or id does not exist.

    For backwards-compatibility reasons, this method can also be called
    to fetch service accounts by their ID.

    Args:
        user_name_or_id: The name or ID of the user to get.
        include_private: Whether to include private user information
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested user, if it was found.

    Raises:
        KeyError: If the user does not exist.
    """
    if not user_name_or_id:
        user_name_or_id = self._default_user_name

    with Session(self.engine) as session:
        # If a UUID is passed, we also allow fetching service accounts
        # with that ID.
        service_account: Optional[bool] = False
        if uuid_utils.is_valid_uuid(user_name_or_id):
            service_account = None
        user = self._get_account_schema(
            user_name_or_id,
            session=session,
            service_account=service_account,
        )

        return user.to_model(
            include_private=include_private, include_metadata=hydrate
        )
get_workspace(self, workspace_name_or_id, hydrate=True)

Get an existing workspace by name or ID.

Parameters:

Name Type Description Default
workspace_name_or_id Union[str, uuid.UUID]

Name or ID of the workspace to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
WorkspaceResponse

The requested workspace if one was found.

Source code in zenml/zen_stores/sql_zen_store.py
def get_workspace(
    self, workspace_name_or_id: Union[str, UUID], hydrate: bool = True
) -> WorkspaceResponse:
    """Get an existing workspace by name or ID.

    Args:
        workspace_name_or_id: Name or ID of the workspace to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested workspace if one was found.
    """
    with Session(self.engine) as session:
        workspace = self._get_workspace_schema(
            workspace_name_or_id, session=session
        )
    return workspace.to_model(include_metadata=hydrate)
list_api_keys(self, service_account_id, filter_model, hydrate=False)

List all API keys for a service account matching the given filter criteria.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to list the API keys.

required
filter_model APIKeyFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[APIKeyResponse]

A list of all API keys matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_api_keys(
    self,
    service_account_id: UUID,
    filter_model: APIKeyFilter,
    hydrate: bool = False,
) -> Page[APIKeyResponse]:
    """List all API keys for a service account matching the given filter criteria.

    Args:
        service_account_id: The ID of the service account for which to list
            the API keys.
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all API keys matching the filter criteria.
    """
    with Session(self.engine) as session:
        # Fetch the service account
        service_account = self._get_account_schema(
            service_account_id, session=session, service_account=True
        )

        filter_model.set_service_account(service_account.id)
        query = select(APIKeySchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=APIKeySchema,
            filter_model=filter_model,
            hydrate=hydrate,
        )
list_artifact_versions(self, artifact_version_filter_model, hydrate=False)

List all artifact versions matching the given filter criteria.

Parameters:

Name Type Description Default
artifact_version_filter_model ArtifactVersionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ArtifactVersionResponse]

A list of all artifact versions matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_artifact_versions(
    self,
    artifact_version_filter_model: ArtifactVersionFilter,
    hydrate: bool = False,
) -> Page[ArtifactVersionResponse]:
    """List all artifact versions matching the given filter criteria.

    Args:
        artifact_version_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all artifact versions matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(ArtifactVersionSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=ArtifactVersionSchema,
            filter_model=artifact_version_filter_model,
            hydrate=hydrate,
        )
list_artifacts(self, filter_model, hydrate=False)

List all artifacts matching the given filter criteria.

Parameters:

Name Type Description Default
filter_model ArtifactFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ArtifactResponse]

A list of all artifacts matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_artifacts(
    self, filter_model: ArtifactFilter, hydrate: bool = False
) -> Page[ArtifactResponse]:
    """List all artifacts matching the given filter criteria.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all artifacts matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(ArtifactSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=ArtifactSchema,
            filter_model=filter_model,
            hydrate=hydrate,
        )
list_authorized_devices(self, filter_model, hydrate=False)

List all OAuth 2.0 authorized devices for a user.

Parameters:

Name Type Description Default
filter_model OAuthDeviceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[OAuthDeviceResponse]

A page of all matching OAuth 2.0 authorized devices.

Source code in zenml/zen_stores/sql_zen_store.py
def list_authorized_devices(
    self,
    filter_model: OAuthDeviceFilter,
    hydrate: bool = False,
) -> Page[OAuthDeviceResponse]:
    """List all OAuth 2.0 authorized devices for a user.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all matching OAuth 2.0 authorized devices.
    """
    with Session(self.engine) as session:
        query = select(OAuthDeviceSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=OAuthDeviceSchema,
            filter_model=filter_model,
            hydrate=hydrate,
        )
list_builds(self, build_filter_model, hydrate=False)

List all builds matching the given filter criteria.

Parameters:

Name Type Description Default
build_filter_model PipelineBuildFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineBuildResponse]

A page of all builds matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_builds(
    self,
    build_filter_model: PipelineBuildFilter,
    hydrate: bool = False,
) -> Page[PipelineBuildResponse]:
    """List all builds matching the given filter criteria.

    Args:
        build_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all builds matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(PipelineBuildSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=PipelineBuildSchema,
            filter_model=build_filter_model,
            hydrate=hydrate,
        )
list_code_repositories(self, filter_model, hydrate=False)

List all code repositories.

Parameters:

Name Type Description Default
filter_model CodeRepositoryFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[CodeRepositoryResponse]

A page of all code repositories.

Source code in zenml/zen_stores/sql_zen_store.py
def list_code_repositories(
    self,
    filter_model: CodeRepositoryFilter,
    hydrate: bool = False,
) -> Page[CodeRepositoryResponse]:
    """List all code repositories.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all code repositories.
    """
    with Session(self.engine) as session:
        query = select(CodeRepositorySchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=CodeRepositorySchema,
            filter_model=filter_model,
            hydrate=hydrate,
        )
list_deployments(self, deployment_filter_model, hydrate=False)

List all deployments matching the given filter criteria.

Parameters:

Name Type Description Default
deployment_filter_model PipelineDeploymentFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineDeploymentResponse]

A page of all deployments matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_deployments(
    self,
    deployment_filter_model: PipelineDeploymentFilter,
    hydrate: bool = False,
) -> Page[PipelineDeploymentResponse]:
    """List all deployments matching the given filter criteria.

    Args:
        deployment_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all deployments matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(PipelineDeploymentSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=PipelineDeploymentSchema,
            filter_model=deployment_filter_model,
            hydrate=hydrate,
        )
list_event_sources(self, event_source_filter_model, hydrate=False)

List all event_sources matching the given filter criteria.

Parameters:

Name Type Description Default
event_source_filter_model EventSourceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[EventSourceResponse]

A list of all event_sources matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_event_sources(
    self,
    event_source_filter_model: EventSourceFilter,
    hydrate: bool = False,
) -> Page[EventSourceResponse]:
    """List all event_sources matching the given filter criteria.

    Args:
        event_source_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all event_sources matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(EventSourceSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=EventSourceSchema,
            filter_model=event_source_filter_model,
            hydrate=hydrate,
        )
list_flavors(self, flavor_filter_model, hydrate=False)

List all stack component flavors matching the given filter criteria.

Parameters:

Name Type Description Default
flavor_filter_model FlavorFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[FlavorResponse]

List of all the stack component flavors matching the given criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_flavors(
    self,
    flavor_filter_model: FlavorFilter,
    hydrate: bool = False,
) -> Page[FlavorResponse]:
    """List all stack component flavors matching the given filter criteria.

    Args:
        flavor_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        List of all the stack component flavors matching the given criteria.
    """
    with Session(self.engine) as session:
        query = select(FlavorSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=FlavorSchema,
            filter_model=flavor_filter_model,
            hydrate=hydrate,
        )

Get all model version to artifact links by filter.

Parameters:

Name Type Description Default
model_version_artifact_link_filter_model ModelVersionArtifactFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionArtifactResponse]

A page of all model version to artifact links.

Source code in zenml/zen_stores/sql_zen_store.py
def list_model_version_artifact_links(
    self,
    model_version_artifact_link_filter_model: ModelVersionArtifactFilter,
    hydrate: bool = False,
) -> Page[ModelVersionArtifactResponse]:
    """Get all model version to artifact links by filter.

    Args:
        model_version_artifact_link_filter_model: All filter parameters
            including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model version to artifact links.
    """
    with Session(self.engine) as session:
        query = select(ModelVersionArtifactSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=ModelVersionArtifactSchema,
            filter_model=model_version_artifact_link_filter_model,
            hydrate=hydrate,
        )

Get all model version to pipeline run links by filter.

Parameters:

Name Type Description Default
model_version_pipeline_run_link_filter_model ModelVersionPipelineRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionPipelineRunResponse]

A page of all model version to pipeline run links.

Source code in zenml/zen_stores/sql_zen_store.py
def list_model_version_pipeline_run_links(
    self,
    model_version_pipeline_run_link_filter_model: ModelVersionPipelineRunFilter,
    hydrate: bool = False,
) -> Page[ModelVersionPipelineRunResponse]:
    """Get all model version to pipeline run links by filter.

    Args:
        model_version_pipeline_run_link_filter_model: All filter parameters
            including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model version to pipeline run links.
    """
    query = select(ModelVersionPipelineRunSchema)
    with Session(self.engine) as session:
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=ModelVersionPipelineRunSchema,
            filter_model=model_version_pipeline_run_link_filter_model,
            hydrate=hydrate,
        )
list_model_versions(self, model_version_filter_model, model_name_or_id=None, hydrate=False)

Get all model versions by filter.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model containing the model versions.

None
model_version_filter_model ModelVersionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionResponse]

A page of all model versions.

Source code in zenml/zen_stores/sql_zen_store.py
def list_model_versions(
    self,
    model_version_filter_model: ModelVersionFilter,
    model_name_or_id: Optional[Union[str, UUID]] = None,
    hydrate: bool = False,
) -> Page[ModelVersionResponse]:
    """Get all model versions by filter.

    Args:
        model_name_or_id: name or id of the model containing the model
            versions.
        model_version_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model versions.
    """
    with Session(self.engine) as session:
        if model_name_or_id:
            model = self.get_model(model_name_or_id)
            model_version_filter_model.set_scope_model(model.id)

        query = select(ModelVersionSchema)

        return self.filter_and_paginate(
            session=session,
            query=query,
            table=ModelVersionSchema,
            filter_model=model_version_filter_model,
            hydrate=hydrate,
        )
list_models(self, model_filter_model, hydrate=False)

Get all models by filter.

Parameters:

Name Type Description Default
model_filter_model ModelFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelResponse]

A page of all models.

Source code in zenml/zen_stores/sql_zen_store.py
def list_models(
    self,
    model_filter_model: ModelFilter,
    hydrate: bool = False,
) -> Page[ModelResponse]:
    """Get all models by filter.

    Args:
        model_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all models.
    """
    with Session(self.engine) as session:
        query = select(ModelSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=ModelSchema,
            filter_model=model_filter_model,
            hydrate=hydrate,
        )
list_pipeline_namespaces(self, filter_model, hydrate=False)

List all pipeline namespaces matching the given filter criteria.

Parameters:

Name Type Description Default
filter_model PipelineNamespaceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineNamespaceResponse]

A list of all pipeline namespaces matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_pipeline_namespaces(
    self,
    filter_model: PipelineNamespaceFilter,
    hydrate: bool = False,
) -> Page[PipelineNamespaceResponse]:
    """List all pipeline namespaces matching the given filter criteria.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all pipeline namespaces matching the filter criteria.
    """

    def _custom_conversion(
        row: Tuple[str, UUID, str],
    ) -> PipelineNamespaceResponse:
        name, latest_run_id, latest_run_status = row

        body = PipelineNamespaceResponseBody(
            latest_run_id=latest_run_id,
            latest_run_status=latest_run_status,
        )

        return PipelineNamespaceResponse(name=name, body=body)

    def _custom_fetch(
        session: Session,
        query: Union[Select[Any], SelectOfScalar[Any]],
        filter: BaseFilter,
    ) -> List[Any]:
        return session.exec(query).unique().all()

    with Session(self.engine) as session:
        max_date_subquery = (
            select(  # type: ignore[call-overload]
                PipelineSchema.name,
                func.max(PipelineRunSchema.created).label("max_created"),
            )
            .outerjoin(
                PipelineRunSchema,
                PipelineSchema.id == PipelineRunSchema.pipeline_id,
            )
            .group_by(PipelineSchema.name)
            .subquery()
        )

        query = (
            select(
                max_date_subquery.c.name,
                PipelineRunSchema.id,
                PipelineRunSchema.status,
            )
            .outerjoin(
                PipelineSchema,
                PipelineSchema.name == max_date_subquery.c.name,
            )
            .outerjoin(
                PipelineRunSchema,
                PipelineRunSchema.created
                == max_date_subquery.c.max_created,
            )
        )

        return self.filter_and_paginate(
            session=session,
            query=query,
            table=PipelineSchema,
            filter_model=filter_model,
            hydrate=hydrate,
            custom_fetch=_custom_fetch,
            custom_schema_to_model_conversion=_custom_conversion,
        )
list_pipelines(self, pipeline_filter_model, hydrate=False)

List all pipelines matching the given filter criteria.

Parameters:

Name Type Description Default
pipeline_filter_model PipelineFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineResponse]

A list of all pipelines matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_pipelines(
    self,
    pipeline_filter_model: PipelineFilter,
    hydrate: bool = False,
) -> Page[PipelineResponse]:
    """List all pipelines matching the given filter criteria.

    Args:
        pipeline_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all pipelines matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(PipelineSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=PipelineSchema,
            filter_model=pipeline_filter_model,
            hydrate=hydrate,
        )
list_run_metadata(self, run_metadata_filter_model, hydrate=False)

List run metadata.

Parameters:

Name Type Description Default
run_metadata_filter_model RunMetadataFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[RunMetadataResponse]

The run metadata.

Source code in zenml/zen_stores/sql_zen_store.py
def list_run_metadata(
    self,
    run_metadata_filter_model: RunMetadataFilter,
    hydrate: bool = False,
) -> Page[RunMetadataResponse]:
    """List run metadata.

    Args:
        run_metadata_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The run metadata.
    """
    with Session(self.engine) as session:
        query = select(RunMetadataSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=RunMetadataSchema,
            filter_model=run_metadata_filter_model,
            hydrate=hydrate,
        )
list_run_steps(self, step_run_filter_model, hydrate=False)

List all step runs matching the given filter criteria.

Parameters:

Name Type Description Default
step_run_filter_model StepRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[StepRunResponse]

A list of all step runs matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_run_steps(
    self,
    step_run_filter_model: StepRunFilter,
    hydrate: bool = False,
) -> Page[StepRunResponse]:
    """List all step runs matching the given filter criteria.

    Args:
        step_run_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all step runs matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(StepRunSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=StepRunSchema,
            filter_model=step_run_filter_model,
            hydrate=hydrate,
        )
list_runs(self, runs_filter_model, hydrate=False)

List all pipeline runs matching the given filter criteria.

Parameters:

Name Type Description Default
runs_filter_model PipelineRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineRunResponse]

A list of all pipeline runs matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_runs(
    self,
    runs_filter_model: PipelineRunFilter,
    hydrate: bool = False,
) -> Page[PipelineRunResponse]:
    """List all pipeline runs matching the given filter criteria.

    Args:
        runs_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all pipeline runs matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(PipelineRunSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=PipelineRunSchema,
            filter_model=runs_filter_model,
            hydrate=hydrate,
        )
list_schedules(self, schedule_filter_model, hydrate=False)

List all schedules in the workspace.

Parameters:

Name Type Description Default
schedule_filter_model ScheduleFilter

All filter parameters including pagination params

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ScheduleResponse]

A list of schedules.

Source code in zenml/zen_stores/sql_zen_store.py
def list_schedules(
    self,
    schedule_filter_model: ScheduleFilter,
    hydrate: bool = False,
) -> Page[ScheduleResponse]:
    """List all schedules in the workspace.

    Args:
        schedule_filter_model: All filter parameters including pagination
            params
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of schedules.
    """
    with Session(self.engine) as session:
        query = select(ScheduleSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=ScheduleSchema,
            filter_model=schedule_filter_model,
            hydrate=hydrate,
        )
list_secrets(self, secret_filter_model, hydrate=False)

List all secrets matching the given filter criteria.

Note that returned secrets do not include any secret values. To fetch the secret values, use get_secret.

Parameters:

Name Type Description Default
secret_filter_model SecretFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[SecretResponse]

A list of all secrets matching the filter criteria, with pagination information and sorted according to the filter criteria. The returned secrets do not include any secret values, only metadata. To fetch the secret values, use get_secret individually with each secret.

Source code in zenml/zen_stores/sql_zen_store.py
def list_secrets(
    self, secret_filter_model: SecretFilter, hydrate: bool = False
) -> Page[SecretResponse]:
    """List all secrets matching the given filter criteria.

    Note that returned secrets do not include any secret values. To fetch
    the secret values, use `get_secret`.

    Args:
        secret_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all secrets matching the filter criteria, with pagination
        information and sorted according to the filter criteria. The
        returned secrets do not include any secret values, only metadata. To
        fetch the secret values, use `get_secret` individually with each
        secret.
    """
    with Session(self.engine) as session:
        query = select(SecretSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=SecretSchema,
            filter_model=secret_filter_model,
            hydrate=hydrate,
        )
list_service_accounts(self, filter_model, hydrate=False)

List all service accounts.

Parameters:

Name Type Description Default
filter_model ServiceAccountFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ServiceAccountResponse]

A list of filtered service accounts.

Source code in zenml/zen_stores/sql_zen_store.py
def list_service_accounts(
    self,
    filter_model: ServiceAccountFilter,
    hydrate: bool = False,
) -> Page[ServiceAccountResponse]:
    """List all service accounts.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of filtered service accounts.
    """
    with Session(self.engine) as session:
        query = select(UserSchema)
        paged_service_accounts: Page[ServiceAccountResponse] = (
            self.filter_and_paginate(
                session=session,
                query=query,
                table=UserSchema,
                filter_model=filter_model,
                custom_schema_to_model_conversion=lambda user: user.to_service_account_model(
                    include_metadata=hydrate
                ),
                hydrate=hydrate,
            )
        )
        return paged_service_accounts
list_service_connector_resources(self, workspace_name_or_id, connector_type=None, resource_type=None, resource_id=None, filter_model=None)

List resources that can be accessed by service connectors.

Parameters:

Name Type Description Default
workspace_name_or_id Union[str, uuid.UUID]

The name or ID of the workspace to scope to.

required
connector_type Optional[str]

The type of service connector to scope to.

None
resource_type Optional[str]

The type of resource to scope to.

None
resource_id Optional[str]

The ID of the resource to scope to.

None
filter_model Optional[zenml.models.v2.core.service_connector.ServiceConnectorFilter]

Optional filter model to use when fetching service connectors.

None

Returns:

Type Description
List[zenml.models.v2.misc.service_connector_type.ServiceConnectorResourcesModel]

The matching list of resources that available service connectors have access to.

Source code in zenml/zen_stores/sql_zen_store.py
def list_service_connector_resources(
    self,
    workspace_name_or_id: Union[str, UUID],
    connector_type: Optional[str] = None,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
    filter_model: Optional[ServiceConnectorFilter] = None,
) -> List[ServiceConnectorResourcesModel]:
    """List resources that can be accessed by service connectors.

    Args:
        workspace_name_or_id: The name or ID of the workspace to scope to.
        connector_type: The type of service connector to scope to.
        resource_type: The type of resource to scope to.
        resource_id: The ID of the resource to scope to.
        filter_model: Optional filter model to use when fetching service
            connectors.

    Returns:
        The matching list of resources that available service
        connectors have access to.
    """
    workspace = self.get_workspace(workspace_name_or_id)

    if not filter_model:
        filter_model = ServiceConnectorFilter(
            connector_type=connector_type,
            resource_type=resource_type,
            workspace_id=workspace.id,
        )

    service_connectors = self.list_service_connectors(
        filter_model=filter_model
    ).items

    resource_list: List[ServiceConnectorResourcesModel] = []

    for connector in service_connectors:
        if not service_connector_registry.is_registered(connector.type):
            # For connectors that we can instantiate, i.e. those that have a
            # connector type available locally, we return complete
            # information about the resources that they have access to.
            #
            # For those that are not locally available, we only return
            # rudimentary information extracted from the connector model
            # without actively trying to discover the resources that they
            # have access to.

            if resource_id and connector.resource_id != resource_id:
                # If an explicit resource ID is required, the connector
                # has to be configured with it.
                continue

            resources = (
                ServiceConnectorResourcesModel.from_connector_model(
                    connector,
                    resource_type=resource_type,
                )
            )
            for r in resources.resources:
                if not r.resource_ids:
                    r.error = (
                        f"The service '{connector.type}' connector type is "
                        "not available."
                    )

        else:
            try:
                connector_instance = (
                    service_connector_registry.instantiate_connector(
                        model=connector
                    )
                )

                resources = connector_instance.verify(
                    resource_type=resource_type,
                    resource_id=resource_id,
                    list_resources=True,
                )
            except (ValueError, AuthorizationException) as e:
                error = (
                    f'Failed to fetch {resource_type or "available"} '
                    f"resources from service connector {connector.name}/"
                    f"{connector.id}: {e}"
                )
                # Log an exception if debug logging is enabled
                if logger.isEnabledFor(logging.DEBUG):
                    logger.exception(error)
                else:
                    logger.error(error)
                continue

        resource_list.append(resources)

    return resource_list
list_service_connector_types(self, connector_type=None, resource_type=None, auth_method=None)

Get a list of service connector types.

Parameters:

Name Type Description Default
connector_type Optional[str]

Filter by connector type.

None
resource_type Optional[str]

Filter by resource type.

None
auth_method Optional[str]

Filter by authentication method.

None

Returns:

Type Description
List[zenml.models.v2.misc.service_connector_type.ServiceConnectorTypeModel]

List of service connector types.

Source code in zenml/zen_stores/sql_zen_store.py
def list_service_connector_types(
    self,
    connector_type: Optional[str] = None,
    resource_type: Optional[str] = None,
    auth_method: Optional[str] = None,
) -> List[ServiceConnectorTypeModel]:
    """Get a list of service connector types.

    Args:
        connector_type: Filter by connector type.
        resource_type: Filter by resource type.
        auth_method: Filter by authentication method.

    Returns:
        List of service connector types.
    """
    return service_connector_registry.list_service_connector_types(
        connector_type=connector_type,
        resource_type=resource_type,
        auth_method=auth_method,
    )
list_service_connectors(self, filter_model, hydrate=False)

List all service connectors.

Parameters:

Name Type Description Default
filter_model ServiceConnectorFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ServiceConnectorResponse]

A page of all service connectors.

Source code in zenml/zen_stores/sql_zen_store.py
def list_service_connectors(
    self,
    filter_model: ServiceConnectorFilter,
    hydrate: bool = False,
) -> Page[ServiceConnectorResponse]:
    """List all service connectors.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all service connectors.
    """

    def fetch_connectors(
        session: Session,
        query: Union[
            Select[ServiceConnectorSchema],
            SelectOfScalar[ServiceConnectorSchema],
        ],
        filter_model: BaseFilter,
    ) -> List[ServiceConnectorSchema]:
        """Custom fetch function for connector filtering and pagination.

        Applies resource type and label filters to the query.

        Args:
            session: The database session.
            query: The query to filter.
            filter_model: The filter model.

        Returns:
            The filtered and paginated results.
        """
        assert isinstance(filter_model, ServiceConnectorFilter)
        items = self._list_filtered_service_connectors(
            session=session, query=query, filter_model=filter_model
        )

        return items

    with Session(self.engine) as session:
        query = select(ServiceConnectorSchema)
        paged_connectors: Page[ServiceConnectorResponse] = (
            self.filter_and_paginate(
                session=session,
                query=query,
                table=ServiceConnectorSchema,
                filter_model=filter_model,
                custom_fetch=fetch_connectors,
                hydrate=hydrate,
            )
        )

        self._populate_connector_type(*paged_connectors.items)
        return paged_connectors
list_stack_components(self, component_filter_model, hydrate=False)

List all stack components matching the given filter criteria.

Parameters:

Name Type Description Default
component_filter_model ComponentFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ComponentResponse]

A list of all stack components matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_stack_components(
    self,
    component_filter_model: ComponentFilter,
    hydrate: bool = False,
) -> Page[ComponentResponse]:
    """List all stack components matching the given filter criteria.

    Args:
        component_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all stack components matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(StackComponentSchema)
        paged_components: Page[ComponentResponse] = (
            self.filter_and_paginate(
                session=session,
                query=query,
                table=StackComponentSchema,
                filter_model=component_filter_model,
                hydrate=hydrate,
            )
        )
        return paged_components
list_stacks(self, stack_filter_model, hydrate=False)

List all stacks matching the given filter criteria.

Parameters:

Name Type Description Default
stack_filter_model StackFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[StackResponse]

A list of all stacks matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_stacks(
    self,
    stack_filter_model: StackFilter,
    hydrate: bool = False,
) -> Page[StackResponse]:
    """List all stacks matching the given filter criteria.

    Args:
        stack_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all stacks matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(StackSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=StackSchema,
            filter_model=stack_filter_model,
            hydrate=hydrate,
        )
list_tags(self, tag_filter_model, hydrate=False)

Get all tags by filter.

Parameters:

Name Type Description Default
tag_filter_model TagFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TagResponse]

A page of all tags.

Source code in zenml/zen_stores/sql_zen_store.py
def list_tags(
    self,
    tag_filter_model: TagFilter,
    hydrate: bool = False,
) -> Page[TagResponse]:
    """Get all tags by filter.

    Args:
        tag_filter_model: All filter parameters including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all tags.
    """
    with Session(self.engine) as session:
        query = select(TagSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=TagSchema,
            filter_model=tag_filter_model,
            hydrate=hydrate,
        )
list_trigger_executions(self, trigger_execution_filter_model, hydrate=False)

List all trigger executions matching the given filter criteria.

Parameters:

Name Type Description Default
trigger_execution_filter_model TriggerExecutionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TriggerExecutionResponse]

A list of all trigger executions matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_trigger_executions(
    self,
    trigger_execution_filter_model: TriggerExecutionFilter,
    hydrate: bool = False,
) -> Page[TriggerExecutionResponse]:
    """List all trigger executions matching the given filter criteria.

    Args:
        trigger_execution_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all trigger executions matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(TriggerExecutionSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=TriggerExecutionSchema,
            filter_model=trigger_execution_filter_model,
            hydrate=hydrate,
        )
list_triggers(self, trigger_filter_model, hydrate=False)

List all trigger matching the given filter criteria.

Parameters:

Name Type Description Default
trigger_filter_model TriggerFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TriggerResponse]

A list of all triggers matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_triggers(
    self,
    trigger_filter_model: TriggerFilter,
    hydrate: bool = False,
) -> Page[TriggerResponse]:
    """List all trigger matching the given filter criteria.

    Args:
        trigger_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all triggers matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(TriggerSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=TriggerSchema,
            filter_model=trigger_filter_model,
            hydrate=hydrate,
        )
list_users(self, user_filter_model, hydrate=False)

List all users.

Parameters:

Name Type Description Default
user_filter_model UserFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[UserResponse]

A list of all users.

Source code in zenml/zen_stores/sql_zen_store.py
def list_users(
    self,
    user_filter_model: UserFilter,
    hydrate: bool = False,
) -> Page[UserResponse]:
    """List all users.

    Args:
        user_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all users.
    """
    with Session(self.engine) as session:
        query = select(UserSchema)
        paged_user: Page[UserResponse] = self.filter_and_paginate(
            session=session,
            query=query,
            table=UserSchema,
            filter_model=user_filter_model,
            hydrate=hydrate,
        )
        return paged_user
list_workspaces(self, workspace_filter_model, hydrate=False)

List all workspace matching the given filter criteria.

Parameters:

Name Type Description Default
workspace_filter_model WorkspaceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[WorkspaceResponse]

A list of all workspace matching the filter criteria.

Source code in zenml/zen_stores/sql_zen_store.py
def list_workspaces(
    self,
    workspace_filter_model: WorkspaceFilter,
    hydrate: bool = False,
) -> Page[WorkspaceResponse]:
    """List all workspace matching the given filter criteria.

    Args:
        workspace_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all workspace matching the filter criteria.
    """
    with Session(self.engine) as session:
        query = select(WorkspaceSchema)
        return self.filter_and_paginate(
            session=session,
            query=query,
            table=WorkspaceSchema,
            filter_model=workspace_filter_model,
            hydrate=hydrate,
        )
migrate_database(self)

Migrate the database to the head as defined by the python package.

Exceptions:

Type Description
RuntimeError

If the database exists and is not empty but has never been migrated with alembic before.

Source code in zenml/zen_stores/sql_zen_store.py
def migrate_database(self) -> None:
    """Migrate the database to the head as defined by the python package.

    Raises:
        RuntimeError: If the database exists and is not empty but has never
            been migrated with alembic before.
    """
    alembic_logger = logging.getLogger("alembic")

    # remove all existing handlers
    while len(alembic_logger.handlers):
        alembic_logger.removeHandler(alembic_logger.handlers[0])

    logging_level = get_logging_level()

    # suppress alembic info logging if the zenml logging level is not debug
    if logging_level == LoggingLevels.DEBUG:
        alembic_logger.setLevel(logging.DEBUG)
    else:
        alembic_logger.setLevel(logging.WARNING)

    alembic_logger.addHandler(get_console_handler())

    # We need to account for 3 distinct cases here:
    # 1. the database is completely empty (not initialized)
    # 2. the database is not empty and has been migrated with alembic before
    # 3. the database is not empty, but has never been migrated with alembic
    #   before (i.e. was created with SQLModel back when alembic wasn't
    #   used). We don't support this direct upgrade case anymore.
    current_revisions = self.alembic.current_revisions()
    head_revisions = self.alembic.head_revisions()
    if len(current_revisions) >= 1:
        # Case 2: the database has been migrated with alembic before. Just
        # upgrade to the latest revision.
        if len(current_revisions) > 1:
            logger.warning(
                "The ZenML database has more than one migration head "
                "revision. This is not expected and might indicate a "
                "database migration problem. Please raise an issue on "
                "GitHub if you encounter this."
            )

        logger.debug("Current revisions: %s", current_revisions)
        logger.debug("Head revisions: %s", head_revisions)

        # If the current revision and head revision don't match, a database
        # migration that changes the database structure or contents may
        # actually be performed, in which case we enable the backup
        # functionality. We only enable the backup functionality if the
        # database will actually be changed, to avoid the overhead for
        # unnecessary backups.
        backup_enabled = (
            self.config.backup_strategy != DatabaseBackupStrategy.DISABLED
            and set(current_revisions) != set(head_revisions)
        )
        backup_location: Optional[Any] = None
        backup_location_msg: Optional[str] = None

        if backup_enabled:
            try:
                logger.info("Backing up the database before migration.")
                (
                    backup_location_msg,
                    backup_location,
                ) = self.backup_database(overwrite=True)
            except Exception as e:
                raise RuntimeError(
                    f"Failed to backup the database: {str(e)}. "
                    "Please check the logs for more details."
                    "If you would like to disable the database backup "
                    "functionality, set the `backup_strategy` attribute "
                    "of the store configuration to `disabled`."
                ) from e
            else:
                if backup_location is not None:
                    logger.info(
                        "Database successfully backed up to "
                        f"{backup_location_msg}. If something goes wrong "
                        "with the upgrade, ZenML will attempt to restore "
                        "the database from this backup automatically."
                    )

        try:
            self.alembic.upgrade()
        except Exception as e:
            if backup_enabled and backup_location:
                logger.exception(
                    "Failed to migrate the database. Attempting to restore "
                    f"the database from {backup_location_msg}."
                )
                try:
                    self.restore_database(location=backup_location)
                except Exception:
                    logger.exception(
                        "Failed to restore the database from "
                        f"{backup_location_msg}. Please "
                        "check the logs for more details. You might need "
                        "to restore the database manually."
                    )
                else:
                    raise RuntimeError(
                        "The database migration failed, but the database "
                        "was successfully restored from the backup. "
                        "You can safely retry the upgrade or revert to "
                        "the previous version of ZenML. Please check the "
                        "logs for more details."
                    ) from e
            raise RuntimeError(
                f"The database migration failed: {str(e)}"
            ) from e

        else:
            # We always remove the backup after a successful upgrade,
            # not just to avoid cluttering the disk, but also to avoid
            # reusing an outdated database from the backup in case of
            # future upgrade failures.
            try:
                self.cleanup_database_backup()
            except Exception:
                logger.exception("Failed to cleanup the database backup.")

    elif self.alembic.db_is_empty():
        # Case 1: the database is empty. We can just create the
        # tables from scratch with from SQLModel. After tables are
        # created we put an alembic revision to latest and populate
        # identity table with needed info.
        logger.info("Creating database tables")
        with self.engine.begin() as conn:
            conn.run_callable(
                SQLModel.metadata.create_all  # type: ignore[arg-type]
            )
        with Session(self.engine) as session:
            session.add(
                IdentitySchema(
                    id=str(GlobalConfiguration().user_id).replace("-", "")
                )
            )
            session.commit()
        self.alembic.stamp("head")
    else:
        # Case 3: the database is not empty, but has never been
        # migrated with alembic before. We don't support this direct
        # upgrade case anymore. The user needs to run a two-step
        # upgrade.
        raise RuntimeError(
            "The ZenML database has never been migrated with alembic "
            "before. This can happen if you are performing a direct "
            "upgrade from a really old version of ZenML. This direct "
            "upgrade path is not supported anymore. Please upgrade "
            "your ZenML installation first to 0.54.0 or an earlier "
            "version and then to the latest version."
        )

    # If an alembic migration took place, all non-custom flavors are purged
    #  and the FlavorRegistry recreates all in-built and integration
    #  flavors in the db.
    revisions_afterwards = self.alembic.current_revisions()

    if current_revisions != revisions_afterwards:
        self._sync_flavors()
prune_artifact_versions(self, only_versions=True)

Prunes unused artifact versions and their artifacts.

Parameters:

Name Type Description Default
only_versions bool

Only delete artifact versions, keeping artifacts

True
Source code in zenml/zen_stores/sql_zen_store.py
def prune_artifact_versions(
    self,
    only_versions: bool = True,
) -> None:
    """Prunes unused artifact versions and their artifacts.

    Args:
        only_versions: Only delete artifact versions, keeping artifacts
    """
    with Session(self.engine) as session:
        unused_artifact_versions = [
            a[0]
            for a in session.execute(
                select(ArtifactVersionSchema.id).where(
                    and_(
                        col(ArtifactVersionSchema.id).notin_(
                            select(StepRunOutputArtifactSchema.artifact_id)
                        ),
                        col(ArtifactVersionSchema.id).notin_(
                            select(StepRunInputArtifactSchema.artifact_id)
                        ),
                    )
                )
            ).fetchall()
        ]
        session.execute(
            delete(ArtifactVersionSchema).where(
                col(ArtifactVersionSchema.id).in_(
                    unused_artifact_versions
                ),
            )
        )
        if not only_versions:
            unused_artifacts = [
                a[0]
                for a in session.execute(
                    select(ArtifactSchema.id).where(
                        col(ArtifactSchema.id).notin_(
                            select(ArtifactVersionSchema.artifact_id)
                        )
                    )
                ).fetchall()
            ]
            session.execute(
                delete(ArtifactSchema).where(
                    col(ArtifactSchema.id).in_(unused_artifacts)
                )
            )
        session.commit()
restore_database(self, strategy=None, location=None, cleanup=False)

Restore the database.

Parameters:

Name Type Description Default
strategy Optional[zenml.enums.DatabaseBackupStrategy]

Custom backup strategy to use. If not set, the backup strategy from the store configuration will be used.

None
location Optional[Any]

Custom target location to restore the database from. If not set, the configured backup location will be used. Depending on the backup strategy, this can be a file path, a database name or an in-memory database representation.

None
cleanup bool

Whether to cleanup the backup after restoring the database.

False

Exceptions:

Type Description
ValueError

If the backup database name is not set when the backup database is requested or if the backup strategy is invalid.

Source code in zenml/zen_stores/sql_zen_store.py
def restore_database(
    self,
    strategy: Optional[DatabaseBackupStrategy] = None,
    location: Optional[Any] = None,
    cleanup: bool = False,
) -> None:
    """Restore the database.

    Args:
        strategy: Custom backup strategy to use. If not set, the backup
            strategy from the store configuration will be used.
        location: Custom target location to restore the database from. If
            not set, the configured backup location will be used. Depending
            on the backup strategy, this can be a file path, a database
            name or an in-memory database representation.
        cleanup: Whether to cleanup the backup after restoring the database.

    Raises:
        ValueError: If the backup database name is not set when the backup
            database is requested or if the backup strategy is invalid.
    """
    strategy = strategy or self.config.backup_strategy

    if (
        strategy == DatabaseBackupStrategy.DUMP_FILE
        or self.config.driver == SQLDatabaseDriver.SQLITE
    ):
        dump_file = location or self._get_db_backup_file_path()
        self.migration_utils.restore_database_from_file(
            dump_file=dump_file
        )
    elif strategy == DatabaseBackupStrategy.DATABASE:
        backup_db_name = location or self.config.backup_database
        if not backup_db_name:
            raise ValueError(
                "The backup database name must be set in the store "
                "configuration to use the backup database strategy."
            )

        self.migration_utils.restore_database_from_db(
            backup_db_name=backup_db_name
        )
    elif strategy == DatabaseBackupStrategy.IN_MEMORY:
        if location is None or not isinstance(location, list):
            raise ValueError(
                "The in-memory database representation must be provided "
                "to restore the database from an in-memory backup."
            )
        self.migration_utils.restore_database_from_memory(db_dump=location)

    else:
        raise ValueError(f"Invalid backup strategy: {strategy}.")

    if cleanup:
        self.cleanup_database_backup()
restore_secrets(self, ignore_errors=False, delete_secrets=False)

Restore all secrets from the configured backup secrets store.

Parameters:

Name Type Description Default
ignore_errors bool

Whether to ignore individual errors during the restore process and attempt to restore all secrets.

False
delete_secrets bool

Whether to delete the secrets that have been successfully restored from the backup secrets store. Setting this flag effectively moves all secrets from the backup secrets store to the primary secrets store.

False
noqa: DAR401

Exceptions:

Type Description
BackupSecretsStoreNotConfiguredError

if no backup secrets store is configured.

Source code in zenml/zen_stores/sql_zen_store.py
def restore_secrets(
    self, ignore_errors: bool = False, delete_secrets: bool = False
) -> None:
    """Restore all secrets from the configured backup secrets store.

    Args:
        ignore_errors: Whether to ignore individual errors during the
            restore process and attempt to restore all secrets.
        delete_secrets: Whether to delete the secrets that have been
            successfully restored from the backup secrets store. Setting
            this flag effectively moves all secrets from the backup secrets
            store to the primary secrets store.

    # noqa: DAR401
    Raises:
        BackupSecretsStoreNotConfiguredError: if no backup secrets store is
            configured.
    """
    if not self.backup_secrets_store:
        raise BackupSecretsStoreNotConfiguredError(
            "Unable to restore secrets: No backup secrets store is "
            "configured."
        )

    with Session(self.engine) as session:
        secrets_in_db = session.exec(select(SecretSchema)).all()

    for secret in secrets_in_db:
        try:
            values = self._get_backup_secret_values(secret_id=secret.id)
        except Exception:
            logger.exception(
                f"Failed to get backup secret values for secret with ID "
                f"{secret.id}."
            )
            if ignore_errors:
                continue
            raise

        try:
            self._update_secret_values(
                secret_id=secret.id,
                values=cast(Dict[str, Optional[str]], values),
                overwrite=True,
                backup=False,
            )
        except Exception:
            logger.exception(
                f"Failed to restore secret with ID {secret.id}. "
            )
            if ignore_errors:
                continue
            raise

        if delete_secrets:
            try:
                self._delete_backup_secret_values(secret_id=secret.id)
            except Exception:
                logger.exception(
                    f"Failed to delete backup secret with ID {secret.id} "
                    f"from the backup secrets store after restoring it to "
                    f"the primary secrets store."
                )
                if ignore_errors:
                    continue
                raise
rotate_api_key(self, service_account_id, api_key_name_or_id, rotate_request)

Rotate an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to rotate the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to rotate.

required
rotate_request APIKeyRotateRequest

The rotate request on the API key.

required

Returns:

Type Description
APIKeyResponse

The updated API key.

Source code in zenml/zen_stores/sql_zen_store.py
def rotate_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    rotate_request: APIKeyRotateRequest,
) -> APIKeyResponse:
    """Rotate an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            rotate the API key.
        api_key_name_or_id: The name or ID of the API key to rotate.
        rotate_request: The rotate request on the API key.

    Returns:
        The updated API key.
    """
    with Session(self.engine) as session:
        api_key = self._get_api_key(
            service_account_id=service_account_id,
            api_key_name_or_id=api_key_name_or_id,
            session=session,
        )

        _, new_key = api_key.rotate(rotate_request)
        session.add(api_key)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(api_key)
        api_key_model = api_key.to_model()
        api_key_model.set_key(new_key)

        return api_key_model
update_api_key(self, service_account_id, api_key_name_or_id, api_key_update)

Update an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to update the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to update.

required
api_key_update APIKeyUpdate

The update request on the API key.

required

Returns:

Type Description
APIKeyResponse

The updated API key.

Exceptions:

Type Description
EntityExistsError

if the API key update would result in a name conflict with an existing API key for the same service account.

Source code in zenml/zen_stores/sql_zen_store.py
def update_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    api_key_update: APIKeyUpdate,
) -> APIKeyResponse:
    """Update an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to update
            the API key.
        api_key_name_or_id: The name or ID of the API key to update.
        api_key_update: The update request on the API key.

    Returns:
        The updated API key.

    Raises:
        EntityExistsError: if the API key update would result in a name
            conflict with an existing API key for the same service account.
    """
    with Session(self.engine) as session:
        api_key = self._get_api_key(
            service_account_id=service_account_id,
            api_key_name_or_id=api_key_name_or_id,
            session=session,
        )

        if api_key_update.name and api_key.name != api_key_update.name:
            # Check if a key with the new name already exists for the same
            # service account
            try:
                self._get_api_key(
                    service_account_id=service_account_id,
                    api_key_name_or_id=api_key_update.name,
                    session=session,
                )

                raise EntityExistsError(
                    f"Unable to update API key with name "
                    f"'{api_key_update.name}': Found an existing API key "
                    "with the same name configured for the same "
                    f"'{api_key.service_account.name}' service account."
                )
            except KeyError:
                pass

        api_key.update(update=api_key_update)
        session.add(api_key)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(api_key)
        return api_key.to_model(include_metadata=True)
update_artifact(self, artifact_id, artifact_update)

Updates an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to update.

required
artifact_update ArtifactUpdate

The update to be applied to the artifact.

required

Returns:

Type Description
ArtifactResponse

The updated artifact.

Exceptions:

Type Description
KeyError

if the artifact doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def update_artifact(
    self, artifact_id: UUID, artifact_update: ArtifactUpdate
) -> ArtifactResponse:
    """Updates an artifact.

    Args:
        artifact_id: The ID of the artifact to update.
        artifact_update: The update to be applied to the artifact.

    Returns:
        The updated artifact.

    Raises:
        KeyError: if the artifact doesn't exist.
    """
    with Session(self.engine) as session:
        existing_artifact = session.exec(
            select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
        ).first()
        if not existing_artifact:
            raise KeyError(f"Artifact with ID {artifact_id} not found.")

        # Handle tag updates.
        if artifact_update.add_tags:
            self._attach_tags_to_resource(
                tag_names=artifact_update.add_tags,
                resource_id=existing_artifact.id,
                resource_type=TaggableResourceTypes.ARTIFACT,
            )
        if artifact_update.remove_tags:
            self._detach_tags_from_resource(
                tag_names=artifact_update.remove_tags,
                resource_id=existing_artifact.id,
                resource_type=TaggableResourceTypes.ARTIFACT,
            )

        # Update the schema itself.
        existing_artifact.update(artifact_update=artifact_update)
        session.add(existing_artifact)
        session.commit()
        session.refresh(existing_artifact)
        return existing_artifact.to_model(include_metadata=True)
update_artifact_version(self, artifact_version_id, artifact_version_update)

Updates an artifact version.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to update.

required
artifact_version_update ArtifactVersionUpdate

The update to be applied to the artifact version.

required

Returns:

Type Description
ArtifactVersionResponse

The updated artifact version.

Exceptions:

Type Description
KeyError

if the artifact version doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def update_artifact_version(
    self,
    artifact_version_id: UUID,
    artifact_version_update: ArtifactVersionUpdate,
) -> ArtifactVersionResponse:
    """Updates an artifact version.

    Args:
        artifact_version_id: The ID of the artifact version to update.
        artifact_version_update: The update to be applied to the artifact
            version.

    Returns:
        The updated artifact version.

    Raises:
        KeyError: if the artifact version doesn't exist.
    """
    with Session(self.engine) as session:
        existing_artifact_version = session.exec(
            select(ArtifactVersionSchema).where(
                ArtifactVersionSchema.id == artifact_version_id
            )
        ).first()
        if not existing_artifact_version:
            raise KeyError(
                f"Artifact version with ID {artifact_version_id} not found."
            )

        # Handle tag updates.
        if artifact_version_update.add_tags:
            self._attach_tags_to_resource(
                tag_names=artifact_version_update.add_tags,
                resource_id=existing_artifact_version.id,
                resource_type=TaggableResourceTypes.ARTIFACT_VERSION,
            )
        if artifact_version_update.remove_tags:
            self._detach_tags_from_resource(
                tag_names=artifact_version_update.remove_tags,
                resource_id=existing_artifact_version.id,
                resource_type=TaggableResourceTypes.ARTIFACT_VERSION,
            )

        # Update the schema itself.
        existing_artifact_version.update(
            artifact_version_update=artifact_version_update
        )
        session.add(existing_artifact_version)
        session.commit()
        session.refresh(existing_artifact_version)
        return existing_artifact_version.to_model(include_metadata=True)
update_authorized_device(self, device_id, update)

Updates an existing OAuth 2.0 authorized device for internal use.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to update.

required
update OAuthDeviceUpdate

The update to be applied to the device.

required

Returns:

Type Description
OAuthDeviceResponse

The updated OAuth 2.0 authorized device.

Exceptions:

Type Description
KeyError

If no device with the given ID exists.

Source code in zenml/zen_stores/sql_zen_store.py
def update_authorized_device(
    self, device_id: UUID, update: OAuthDeviceUpdate
) -> OAuthDeviceResponse:
    """Updates an existing OAuth 2.0 authorized device for internal use.

    Args:
        device_id: The ID of the device to update.
        update: The update to be applied to the device.

    Returns:
        The updated OAuth 2.0 authorized device.

    Raises:
        KeyError: If no device with the given ID exists.
    """
    with Session(self.engine) as session:
        existing_device = session.exec(
            select(OAuthDeviceSchema).where(
                OAuthDeviceSchema.id == device_id
            )
        ).first()
        if existing_device is None:
            raise KeyError(
                f"Unable to update device with ID {device_id}: No "
                "device with this ID found."
            )

        existing_device.update(update)

        session.add(existing_device)
        session.commit()

        return existing_device.to_model(include_metadata=True)
update_code_repository(self, code_repository_id, update)

Updates an existing code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to update.

required
update CodeRepositoryUpdate

The update to be applied to the code repository.

required

Returns:

Type Description
CodeRepositoryResponse

The updated code repository.

Exceptions:

Type Description
KeyError

If no code repository with the given name exists.

Source code in zenml/zen_stores/sql_zen_store.py
def update_code_repository(
    self, code_repository_id: UUID, update: CodeRepositoryUpdate
) -> CodeRepositoryResponse:
    """Updates an existing code repository.

    Args:
        code_repository_id: The ID of the code repository to update.
        update: The update to be applied to the code repository.

    Returns:
        The updated code repository.

    Raises:
        KeyError: If no code repository with the given name exists.
    """
    with Session(self.engine) as session:
        existing_repo = session.exec(
            select(CodeRepositorySchema).where(
                CodeRepositorySchema.id == code_repository_id
            )
        ).first()
        if existing_repo is None:
            raise KeyError(
                f"Unable to update code repository with ID "
                f"{code_repository_id}: No code repository with this ID "
                "found."
            )

        existing_repo.update(update)

        session.add(existing_repo)
        session.commit()

        return existing_repo.to_model(include_metadata=True)
update_event_source(self, event_source_id, event_source_update)

Update an existing event_source.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to update.

required
event_source_update EventSourceUpdate

The update to be applied to the event_source.

required

Returns:

Type Description
EventSourceResponse

The updated event_source.

Source code in zenml/zen_stores/sql_zen_store.py
def update_event_source(
    self,
    event_source_id: UUID,
    event_source_update: EventSourceUpdate,
) -> EventSourceResponse:
    """Update an existing event_source.

    Args:
        event_source_id: The ID of the event_source to update.
        event_source_update: The update to be applied to the event_source.

    Returns:
        The updated event_source.
    """
    with Session(self.engine) as session:
        event_source = self._get_event_source(
            session=session, event_source_id=event_source_id
        )
        event_source.update(update=event_source_update)
        session.add(event_source)
        session.commit()

        # Refresh the event_source that was just created
        session.refresh(event_source)
        return event_source.to_model(
            include_metadata=True, include_resources=True
        )
update_flavor(self, flavor_id, flavor_update)

Updates an existing user.

Parameters:

Name Type Description Default
flavor_id UUID

The id of the flavor to update.

required
flavor_update FlavorUpdate

The update to be applied to the flavor.

required

Returns:

Type Description
FlavorResponse

The updated flavor.

Exceptions:

Type Description
KeyError

If no flavor with the given id exists.

Source code in zenml/zen_stores/sql_zen_store.py
def update_flavor(
    self, flavor_id: UUID, flavor_update: FlavorUpdate
) -> FlavorResponse:
    """Updates an existing user.

    Args:
        flavor_id: The id of the flavor to update.
        flavor_update: The update to be applied to the flavor.

    Returns:
        The updated flavor.

    Raises:
        KeyError: If no flavor with the given id exists.
    """
    with Session(self.engine) as session:
        existing_flavor = session.exec(
            select(FlavorSchema).where(FlavorSchema.id == flavor_id)
        ).first()

        if not existing_flavor:
            raise KeyError(f"Flavor with ID {flavor_id} not found.")

        existing_flavor.update(flavor_update=flavor_update)
        session.add(existing_flavor)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(existing_flavor)
        return existing_flavor.to_model(include_metadata=True)
update_internal_api_key(self, api_key_id, api_key_update)

Update an API key with internal details.

Parameters:

Name Type Description Default
api_key_id UUID

The ID of the API key.

required
api_key_update APIKeyInternalUpdate

The update request on the API key.

required

Returns:

Type Description
APIKeyResponse

The updated API key.

Exceptions:

Type Description
KeyError

if the API key doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def update_internal_api_key(
    self, api_key_id: UUID, api_key_update: APIKeyInternalUpdate
) -> APIKeyResponse:
    """Update an API key with internal details.

    Args:
        api_key_id: The ID of the API key.
        api_key_update: The update request on the API key.

    Returns:
        The updated API key.

    Raises:
        KeyError: if the API key doesn't exist.
    """
    with Session(self.engine) as session:
        api_key = session.exec(
            select(APIKeySchema).where(APIKeySchema.id == api_key_id)
        ).first()

        if not api_key:
            raise KeyError(f"API key with ID {api_key_id} not found.")

        api_key.internal_update(update=api_key_update)
        session.add(api_key)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(api_key)
        return api_key.to_model(include_metadata=True)
update_internal_authorized_device(self, device_id, update)

Updates an existing OAuth 2.0 authorized device.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to update.

required
update OAuthDeviceInternalUpdate

The update to be applied to the device.

required

Returns:

Type Description
OAuthDeviceInternalResponse

The updated OAuth 2.0 authorized device.

Exceptions:

Type Description
KeyError

If no device with the given ID exists.

Source code in zenml/zen_stores/sql_zen_store.py
def update_internal_authorized_device(
    self, device_id: UUID, update: OAuthDeviceInternalUpdate
) -> OAuthDeviceInternalResponse:
    """Updates an existing OAuth 2.0 authorized device.

    Args:
        device_id: The ID of the device to update.
        update: The update to be applied to the device.

    Returns:
        The updated OAuth 2.0 authorized device.

    Raises:
        KeyError: If no device with the given ID exists.
    """
    with Session(self.engine) as session:
        existing_device = session.exec(
            select(OAuthDeviceSchema).where(
                OAuthDeviceSchema.id == device_id
            )
        ).first()
        if existing_device is None:
            raise KeyError(
                f"Unable to update device with ID {device_id}: No device "
                "with this ID found."
            )

        (
            _,
            user_code,
            device_code,
        ) = existing_device.internal_update(update)

        session.add(existing_device)
        session.commit()

        device_model = existing_device.to_internal_model(hydrate=True)
        if user_code:
            # Replace the hashed user code with the original user code
            device_model.user_code = user_code

        if device_code:
            # Replace the hashed device code with the original device code
            device_model.device_code = device_code

        return device_model
update_model(self, model_id, model_update)

Updates an existing model.

Parameters:

Name Type Description Default
model_id UUID

UUID of the model to be updated.

required
model_update ModelUpdate

the Model to be updated.

required

Exceptions:

Type Description
KeyError

specified ID not found.

Returns:

Type Description
ModelResponse

The updated model.

Source code in zenml/zen_stores/sql_zen_store.py
def update_model(
    self,
    model_id: UUID,
    model_update: ModelUpdate,
) -> ModelResponse:
    """Updates an existing model.

    Args:
        model_id: UUID of the model to be updated.
        model_update: the Model to be updated.

    Raises:
        KeyError: specified ID not found.

    Returns:
        The updated model.
    """
    with Session(self.engine) as session:
        existing_model = session.exec(
            select(ModelSchema).where(ModelSchema.id == model_id)
        ).first()

        if not existing_model:
            raise KeyError(f"Model with ID {model_id} not found.")

        if model_update.add_tags:
            self._attach_tags_to_resource(
                tag_names=model_update.add_tags,
                resource_id=existing_model.id,
                resource_type=TaggableResourceTypes.MODEL,
            )
        model_update.add_tags = None
        if model_update.remove_tags:
            self._detach_tags_from_resource(
                tag_names=model_update.remove_tags,
                resource_id=existing_model.id,
                resource_type=TaggableResourceTypes.MODEL,
            )
        model_update.remove_tags = None

        existing_model.update(model_update=model_update)

        session.add(existing_model)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(existing_model)
        return existing_model.to_model(include_metadata=True)
update_model_version(self, model_version_id, model_version_update_model)

Get all model versions by filter.

Parameters:

Name Type Description Default
model_version_id UUID

The ID of model version to be updated.

required
model_version_update_model ModelVersionUpdate

The model version to be updated.

required

Returns:

Type Description
ModelVersionResponse

An updated model version.

Exceptions:

Type Description
KeyError

If the model version not found

RuntimeError

If there is a model version with target stage, but force flag is off

Source code in zenml/zen_stores/sql_zen_store.py
def update_model_version(
    self,
    model_version_id: UUID,
    model_version_update_model: ModelVersionUpdate,
) -> ModelVersionResponse:
    """Get all model versions by filter.

    Args:
        model_version_id: The ID of model version to be updated.
        model_version_update_model: The model version to be updated.

    Returns:
        An updated model version.

    Raises:
        KeyError: If the model version not found
        RuntimeError: If there is a model version with target stage,
            but `force` flag is off
    """
    with Session(self.engine) as session:
        existing_model_version = session.exec(
            select(ModelVersionSchema)
            .where(
                ModelVersionSchema.model_id
                == model_version_update_model.model
            )
            .where(ModelVersionSchema.id == model_version_id)
        ).first()

        if not existing_model_version:
            raise KeyError(f"Model version {model_version_id} not found.")

        stage = None
        if (stage_ := model_version_update_model.stage) is not None:
            stage = getattr(stage_, "value", stage_)

            existing_model_version_in_target_stage = session.exec(
                select(ModelVersionSchema)
                .where(
                    ModelVersionSchema.model_id
                    == model_version_update_model.model
                )
                .where(ModelVersionSchema.stage == stage)
            ).first()

            if (
                existing_model_version_in_target_stage is not None
                and existing_model_version_in_target_stage.id
                != existing_model_version.id
            ):
                if not model_version_update_model.force:
                    raise RuntimeError(
                        f"Model version {existing_model_version_in_target_stage.name} is "
                        f"in {stage}, but `force` flag is False."
                    )
                else:
                    existing_model_version_in_target_stage.update(
                        target_stage=ModelStages.ARCHIVED.value
                    )
                    session.add(existing_model_version_in_target_stage)

                    logger.info(
                        f"Model version {existing_model_version_in_target_stage.name} has been set to {ModelStages.ARCHIVED.value}."
                    )

        if model_version_update_model.add_tags:
            self._attach_tags_to_resource(
                tag_names=model_version_update_model.add_tags,
                resource_id=existing_model_version.id,
                resource_type=TaggableResourceTypes.MODEL_VERSION,
            )
        if model_version_update_model.remove_tags:
            self._detach_tags_from_resource(
                tag_names=model_version_update_model.remove_tags,
                resource_id=existing_model_version.id,
                resource_type=TaggableResourceTypes.MODEL_VERSION,
            )

        existing_model_version.update(
            target_stage=stage,
            target_name=model_version_update_model.name,
            target_description=model_version_update_model.description,
        )
        session.add(existing_model_version)
        session.commit()
        session.refresh(existing_model_version)

        return existing_model_version.to_model(include_metadata=True)
update_pipeline(self, pipeline_id, pipeline_update)

Updates a pipeline.

Parameters:

Name Type Description Default
pipeline_id UUID

The ID of the pipeline to be updated.

required
pipeline_update PipelineUpdate

The update to be applied.

required

Returns:

Type Description
PipelineResponse

The updated pipeline.

Exceptions:

Type Description
KeyError

if the pipeline doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def update_pipeline(
    self,
    pipeline_id: UUID,
    pipeline_update: PipelineUpdate,
) -> PipelineResponse:
    """Updates a pipeline.

    Args:
        pipeline_id: The ID of the pipeline to be updated.
        pipeline_update: The update to be applied.

    Returns:
        The updated pipeline.

    Raises:
        KeyError: if the pipeline doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if pipeline with the given ID exists
        existing_pipeline = session.exec(
            select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
        ).first()
        if existing_pipeline is None:
            raise KeyError(
                f"Unable to update pipeline with ID {pipeline_id}: "
                f"No pipeline with this ID found."
            )

        # Update the pipeline
        existing_pipeline.update(pipeline_update)

        session.add(existing_pipeline)
        session.commit()

        return existing_pipeline.to_model(include_metadata=True)
update_run(self, run_id, run_update)

Updates a pipeline run.

Parameters:

Name Type Description Default
run_id UUID

The ID of the pipeline run to update.

required
run_update PipelineRunUpdate

The update to be applied to the pipeline run.

required

Returns:

Type Description
PipelineRunResponse

The updated pipeline run.

Exceptions:

Type Description
KeyError

if the pipeline run doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def update_run(
    self, run_id: UUID, run_update: PipelineRunUpdate
) -> PipelineRunResponse:
    """Updates a pipeline run.

    Args:
        run_id: The ID of the pipeline run to update.
        run_update: The update to be applied to the pipeline run.

    Returns:
        The updated pipeline run.

    Raises:
        KeyError: if the pipeline run doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if pipeline run with the given ID exists
        existing_run = session.exec(
            select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
        ).first()
        if existing_run is None:
            raise KeyError(
                f"Unable to update pipeline run with ID {run_id}: "
                f"No pipeline run with this ID found."
            )

        # Update the pipeline run
        existing_run.update(run_update=run_update)
        session.add(existing_run)
        session.commit()

        session.refresh(existing_run)
        return existing_run.to_model(include_metadata=True)
update_run_step(self, step_run_id, step_run_update)

Updates a step run.

Parameters:

Name Type Description Default
step_run_id UUID

The ID of the step to update.

required
step_run_update StepRunUpdate

The update to be applied to the step.

required

Returns:

Type Description
StepRunResponse

The updated step run.

Exceptions:

Type Description
KeyError

if the step run doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def update_run_step(
    self,
    step_run_id: UUID,
    step_run_update: StepRunUpdate,
) -> StepRunResponse:
    """Updates a step run.

    Args:
        step_run_id: The ID of the step to update.
        step_run_update: The update to be applied to the step.

    Returns:
        The updated step run.

    Raises:
        KeyError: if the step run doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if the step exists
        existing_step_run = session.exec(
            select(StepRunSchema).where(StepRunSchema.id == step_run_id)
        ).first()
        if existing_step_run is None:
            raise KeyError(
                f"Unable to update step with ID {step_run_id}: "
                f"No step with this ID found."
            )

        # Update the step
        existing_step_run.update(step_run_update)
        session.add(existing_step_run)

        # Update the output artifacts.
        for name, artifact_version_id in step_run_update.outputs.items():
            self._set_run_step_output_artifact(
                step_run_id=step_run_id,
                artifact_version_id=artifact_version_id,
                name=name,
                output_type=StepRunOutputArtifactType.DEFAULT,
                session=session,
            )

        # Update saved artifacts
        for (
            artifact_name,
            artifact_version_id,
        ) in step_run_update.saved_artifact_versions.items():
            self._set_run_step_output_artifact(
                step_run_id=step_run_id,
                artifact_version_id=artifact_version_id,
                name=artifact_name,
                output_type=StepRunOutputArtifactType.MANUAL,
                session=session,
            )

        # Update loaded artifacts.
        for (
            artifact_name,
            artifact_version_id,
        ) in step_run_update.loaded_artifact_versions.items():
            self._set_run_step_input_artifact(
                run_step_id=step_run_id,
                artifact_version_id=artifact_version_id,
                name=artifact_name,
                input_type=StepRunInputArtifactType.MANUAL,
                session=session,
            )

        self._update_pipeline_run_status(
            pipeline_run_id=existing_step_run.pipeline_run_id,
            session=session,
        )

        session.commit()
        session.refresh(existing_step_run)

        return existing_step_run.to_model(include_metadata=True)
update_schedule(self, schedule_id, schedule_update)

Updates a schedule.

Parameters:

Name Type Description Default
schedule_id UUID

The ID of the schedule to be updated.

required
schedule_update ScheduleUpdate

The update to be applied.

required

Returns:

Type Description
ScheduleResponse

The updated schedule.

Exceptions:

Type Description
KeyError

if the schedule doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
def update_schedule(
    self,
    schedule_id: UUID,
    schedule_update: ScheduleUpdate,
) -> ScheduleResponse:
    """Updates a schedule.

    Args:
        schedule_id: The ID of the schedule to be updated.
        schedule_update: The update to be applied.

    Returns:
        The updated schedule.

    Raises:
        KeyError: if the schedule doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if schedule with the given ID exists
        existing_schedule = session.exec(
            select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
        ).first()
        if existing_schedule is None:
            raise KeyError(
                f"Unable to update schedule with ID {schedule_id}: "
                f"No schedule with this ID found."
            )

        # Update the schedule
        existing_schedule = existing_schedule.update(schedule_update)
        session.add(existing_schedule)
        session.commit()
        return existing_schedule.to_model(include_metadata=True)
update_secret(self, secret_id, secret_update)

Updates a secret.

Secret values that are specified as None in the update that are present in the existing secret are removed from the existing secret. Values that are present in both secrets are overwritten. All other values in both the existing secret and the update are kept (merged).

If the update includes a change of name or scope, the scoping rules enforced in the secrets store are used to validate the update:

  • only one workspace-scoped secret with the given name can exist in the target workspace.
  • only one user-scoped secret with the given name can exist in the target workspace for the target user.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_update SecretUpdate

The update to be applied.

required

Returns:

Type Description
SecretResponse

The updated secret.

Exceptions:

Type Description
KeyError

if the secret doesn't exist.

EntityExistsError

If a secret with the same name already exists in the same scope.

Source code in zenml/zen_stores/sql_zen_store.py
def update_secret(
    self, secret_id: UUID, secret_update: SecretUpdate
) -> SecretResponse:
    """Updates a secret.

    Secret values that are specified as `None` in the update that are
    present in the existing secret are removed from the existing secret.
    Values that are present in both secrets are overwritten. All other
    values in both the existing secret and the update are kept (merged).

    If the update includes a change of name or scope, the scoping rules
    enforced in the secrets store are used to validate the update:

      - only one workspace-scoped secret with the given name can exist
        in the target workspace.
      - only one user-scoped secret with the given name can exist in the
        target workspace for the target user.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_update: The update to be applied.

    Returns:
        The updated secret.

    Raises:
        KeyError: if the secret doesn't exist.
        EntityExistsError: If a secret with the same name already exists in
            the same scope.
    """
    with Session(self.engine) as session:
        existing_secret = session.exec(
            select(SecretSchema).where(SecretSchema.id == secret_id)
        ).first()

        if not existing_secret:
            raise KeyError(f"Secret with ID {secret_id} not found.")

        # A change in name or scope requires a check of the scoping rules.
        if (
            secret_update.name is not None
            and existing_secret.name != secret_update.name
            or secret_update.scope is not None
            and existing_secret.scope != secret_update.scope
        ):
            secret_exists, msg = self._check_sql_secret_scope(
                session=session,
                secret_name=secret_update.name or existing_secret.name,
                scope=secret_update.scope
                or SecretScope(existing_secret.scope),
                workspace=existing_secret.workspace.id,
                user=existing_secret.user.id,
                exclude_secret_id=secret_id,
            )

            if secret_exists:
                raise EntityExistsError(msg)

        existing_secret.update(
            secret_update=secret_update,
        )
        session.add(existing_secret)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(existing_secret)
        secret_model = existing_secret.to_model(include_metadata=True)

    if secret_update.values is not None:
        # Update the secret values in the configured secrets store
        updated_values = self._update_secret_values(
            secret_id=secret_id,
            values=secret_update.get_secret_values_update(),
        )
        secret_model.set_secrets(updated_values)
    else:
        secret_model.set_secrets(self._get_secret_values(secret_id))

    return secret_model
update_service_account(self, service_account_name_or_id, service_account_update)

Updates an existing service account.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or the ID of the service account to update.

required
service_account_update ServiceAccountUpdate

The update to be applied to the service account.

required

Returns:

Type Description
ServiceAccountResponse

The updated service account.

Exceptions:

Type Description
EntityExistsError

If a user or service account with the given name already exists.

Source code in zenml/zen_stores/sql_zen_store.py
def update_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
    service_account_update: ServiceAccountUpdate,
) -> ServiceAccountResponse:
    """Updates an existing service account.

    Args:
        service_account_name_or_id: The name or the ID of the service
            account to update.
        service_account_update: The update to be applied to the service
            account.

    Returns:
        The updated service account.

    Raises:
        EntityExistsError: If a user or service account with the given name
            already exists.
    """
    with Session(self.engine) as session:
        existing_service_account = self._get_account_schema(
            service_account_name_or_id,
            session=session,
            service_account=True,
        )

        if (
            service_account_update.name is not None
            and service_account_update.name
            != existing_service_account.name
        ):
            try:
                self._get_account_schema(
                    service_account_update.name,
                    session=session,
                    service_account=True,
                )
                raise EntityExistsError(
                    f"Unable to update service account with name "
                    f"'{service_account_update.name}': Found an existing "
                    "service account with this name."
                )
            except KeyError:
                pass

        existing_service_account.update_service_account(
            service_account_update=service_account_update
        )
        session.add(existing_service_account)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(existing_service_account)
        return existing_service_account.to_service_account_model(
            include_metadata=True
        )
update_service_connector(self, service_connector_id, update)

Updates an existing service connector.

The update model contains the fields to be updated. If a field value is set to None in the model, the field is not updated, but there are special rules concerning some fields:

  • the configuration and secrets fields together represent a full valid configuration update, not just a partial update. If either is set (i.e. not None) in the update, their values are merged together and will replace the existing configuration and secrets values.
  • the resource_id field value is also a full replacement value: if set to None, the resource ID is removed from the service connector.
  • the expiration_seconds field value is also a full replacement value: if set to None, the expiration is removed from the service connector.
  • the secret_id field value in the update is ignored, given that secrets are managed internally by the ZenML store.
  • the labels field is also a full labels update: if set (i.e. not None), all existing labels are removed and replaced by the new labels in the update.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to update.

required
update ServiceConnectorUpdate

The update to be applied to the service connector.

required

Returns:

Type Description
ServiceConnectorResponse

The updated service connector.

Exceptions:

Type Description
KeyError

If no service connector with the given ID exists.

IllegalOperationError

If the service connector is referenced by one or more stack components and the update would change the connector type, resource type or resource ID.

Source code in zenml/zen_stores/sql_zen_store.py
def update_service_connector(
    self, service_connector_id: UUID, update: ServiceConnectorUpdate
) -> ServiceConnectorResponse:
    """Updates an existing service connector.

    The update model contains the fields to be updated. If a field value is
    set to None in the model, the field is not updated, but there are
    special rules concerning some fields:

    * the `configuration` and `secrets` fields together represent a full
    valid configuration update, not just a partial update. If either is
    set (i.e. not None) in the update, their values are merged together and
    will replace the existing configuration and secrets values.
    * the `resource_id` field value is also a full replacement value: if set
    to `None`, the resource ID is removed from the service connector.
    * the `expiration_seconds` field value is also a full replacement value:
    if set to `None`, the expiration is removed from the service connector.
    * the `secret_id` field value in the update is ignored, given that
    secrets are managed internally by the ZenML store.
    * the `labels` field is also a full labels update: if set (i.e. not
    `None`), all existing labels are removed and replaced by the new labels
    in the update.

    Args:
        service_connector_id: The ID of the service connector to update.
        update: The update to be applied to the service connector.

    Returns:
        The updated service connector.

    Raises:
        KeyError: If no service connector with the given ID exists.
        IllegalOperationError: If the service connector is referenced by
            one or more stack components and the update would change the
            connector type, resource type or resource ID.
    """
    with Session(self.engine) as session:
        existing_connector = session.exec(
            select(ServiceConnectorSchema).where(
                ServiceConnectorSchema.id == service_connector_id
            )
        ).first()

        if existing_connector is None:
            raise KeyError(
                f"Unable to update service connector with ID "
                f"'{service_connector_id}': Found no existing service "
                "connector with this ID."
            )

        # In case of a renaming update, make sure no service connector uses
        # that name already
        if update.name and existing_connector.name != update.name:
            self._fail_if_service_connector_with_name_exists(
                name=update.name,
                workspace_id=existing_connector.workspace_id,
                session=session,
            )

        existing_connector_model = existing_connector.to_model(
            include_metadata=True
        )

        if len(existing_connector.components):
            # If the service connector is already used in one or more
            # stack components, the update is no longer allowed to change
            # the service connector's authentication method, connector type,
            # resource type, or resource ID
            if (
                update.connector_type
                and update.type != existing_connector_model.connector_type
            ):
                raise IllegalOperationError(
                    "The service type of a service connector that is "
                    "already actively used in one or more stack components "
                    "cannot be changed."
                )

            if (
                update.auth_method
                and update.auth_method
                != existing_connector_model.auth_method
            ):
                raise IllegalOperationError(
                    "The authentication method of a service connector that "
                    "is already actively used in one or more stack "
                    "components cannot be changed."
                )

            if (
                update.resource_types
                and update.resource_types
                != existing_connector_model.resource_types
            ):
                raise IllegalOperationError(
                    "The resource type of a service connector that is "
                    "already actively used in one or more stack components "
                    "cannot be changed."
                )

            # The resource ID field cannot be used as a partial update: if
            # set to None, the existing resource ID is also removed
            if update.resource_id != existing_connector_model.resource_id:
                raise IllegalOperationError(
                    "The resource ID of a service connector that is "
                    "already actively used in one or more stack components "
                    "cannot be changed."
                )

        # If the connector type is locally available, we validate the update
        # against the connector type schema before storing it in the
        # database
        if service_connector_registry.is_registered(
            existing_connector.connector_type
        ):
            connector_type = (
                service_connector_registry.get_service_connector_type(
                    existing_connector.connector_type
                )
            )
            # We need the auth method to be set to be able to validate the
            # configuration
            update.auth_method = (
                update.auth_method or existing_connector_model.auth_method
            )
            # Validate the configuration update. If the configuration or
            # secrets fields are set, together they are merged into a
            # full configuration that is validated against the connector
            # type schema and replaces the existing configuration and
            # secrets values
            update.validate_and_configure_resources(
                connector_type=connector_type,
                resource_types=update.resource_types,
                resource_id=update.resource_id,
                configuration=update.configuration,
                secrets=update.secrets,
            )

        # Update secret
        secret_id = self._update_connector_secret(
            existing_connector=existing_connector_model,
            updated_connector=update,
        )

        existing_connector.update(
            connector_update=update, secret_id=secret_id
        )
        session.add(existing_connector)
        session.commit()

        connector = existing_connector.to_model(include_metadata=True)
        self._populate_connector_type(connector)
        return connector
update_stack(self, stack_id, stack_update)

Update a stack.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack update.

required
stack_update StackUpdate

The update request on the stack.

required

Returns:

Type Description
StackResponse

The updated stack.

Exceptions:

Type Description
KeyError

if the stack doesn't exist.

IllegalOperationError

if the stack is a default stack.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.UPDATED_STACK)
def update_stack(
    self, stack_id: UUID, stack_update: StackUpdate
) -> StackResponse:
    """Update a stack.

    Args:
        stack_id: The ID of the stack update.
        stack_update: The update request on the stack.

    Returns:
        The updated stack.

    Raises:
        KeyError: if the stack doesn't exist.
        IllegalOperationError: if the stack is a default stack.
    """
    with Session(self.engine) as session:
        # Check if stack with the domain key (name, workspace, owner)
        # already exists
        existing_stack = session.exec(
            select(StackSchema).where(StackSchema.id == stack_id)
        ).first()
        if existing_stack is None:
            raise KeyError(
                f"Unable to update stack with id '{stack_id}': Found no"
                f"existing stack with this id."
            )
        if existing_stack.name == DEFAULT_STACK_AND_COMPONENT_NAME:
            raise IllegalOperationError(
                "The default stack cannot be modified."
            )
        # In case of a renaming update, make sure no stack already exists
        # with that name
        if stack_update.name:
            if existing_stack.name != stack_update.name:
                self._fail_if_stack_with_name_exists(
                    stack=stack_update,
                    session=session,
                )

        components = []
        if stack_update.components:
            filters = [
                (StackComponentSchema.id == component_id)
                for list_of_component_ids in stack_update.components.values()
                for component_id in list_of_component_ids
            ]
            components = session.exec(
                select(StackComponentSchema).where(or_(*filters))
            ).all()

        existing_stack.update(
            stack_update=stack_update,
            components=components,
        )

        session.add(existing_stack)
        session.commit()
        session.refresh(existing_stack)

        return existing_stack.to_model(include_metadata=True)
update_stack_component(self, component_id, component_update)

Update an existing stack component.

Parameters:

Name Type Description Default
component_id UUID

The ID of the stack component to update.

required
component_update ComponentUpdate

The update to be applied to the stack component.

required

Returns:

Type Description
ComponentResponse

The updated stack component.

Exceptions:

Type Description
KeyError

if the stack component doesn't exist.

IllegalOperationError

if the stack component is a default stack component.

Source code in zenml/zen_stores/sql_zen_store.py
def update_stack_component(
    self, component_id: UUID, component_update: ComponentUpdate
) -> ComponentResponse:
    """Update an existing stack component.

    Args:
        component_id: The ID of the stack component to update.
        component_update: The update to be applied to the stack component.

    Returns:
        The updated stack component.

    Raises:
        KeyError: if the stack component doesn't exist.
        IllegalOperationError: if the stack component is a default stack
            component.
    """
    with Session(self.engine) as session:
        existing_component = session.exec(
            select(StackComponentSchema).where(
                StackComponentSchema.id == component_id
            )
        ).first()

        if existing_component is None:
            raise KeyError(
                f"Unable to update component with id "
                f"'{component_id}': Found no"
                f"existing component with this id."
            )

        if (
            existing_component.name == DEFAULT_STACK_AND_COMPONENT_NAME
            and existing_component.type
            in [
                StackComponentType.ORCHESTRATOR,
                StackComponentType.ARTIFACT_STORE,
            ]
        ):
            raise IllegalOperationError(
                f"The default {existing_component.type} cannot be modified."
            )

        # In case of a renaming update, make sure no component of the same
        # type already exists with that name
        if component_update.name:
            if existing_component.name != component_update.name:
                self._fail_if_component_with_name_type_exists(
                    name=component_update.name,
                    component_type=existing_component.type,
                    workspace_id=existing_component.workspace_id,
                    session=session,
                )

        existing_component.update(component_update=component_update)

        if component_update.connector:
            service_connector = session.exec(
                select(ServiceConnectorSchema).where(
                    ServiceConnectorSchema.id == component_update.connector
                )
            ).first()

            if service_connector is None:
                raise KeyError(
                    "Service connector with ID "
                    f"{component_update.connector} not found."
                )
            existing_component.connector = service_connector
            existing_component.connector_resource_id = (
                component_update.connector_resource_id
            )
        else:
            existing_component.connector = None
            existing_component.connector_resource_id = None

        session.add(existing_component)
        session.commit()

        return existing_component.to_model(include_metadata=True)
update_tag(self, tag_name_or_id, tag_update_model)

Update tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to be updated.

required
tag_update_model TagUpdate

Tag to use for the update.

required

Returns:

Type Description
TagResponse

An updated tag.

Exceptions:

Type Description
KeyError

If the tag is not found

Source code in zenml/zen_stores/sql_zen_store.py
def update_tag(
    self,
    tag_name_or_id: Union[str, UUID],
    tag_update_model: TagUpdate,
) -> TagResponse:
    """Update tag.

    Args:
        tag_name_or_id: name or id of the tag to be updated.
        tag_update_model: Tag to use for the update.

    Returns:
        An updated tag.

    Raises:
        KeyError: If the tag is not found
    """
    with Session(self.engine) as session:
        tag = self._get_tag_schema(
            tag_name_or_id=tag_name_or_id, session=session
        )

        if not tag:
            raise KeyError(f"Tag with ID `{tag_name_or_id}` not found.")

        tag.update(update=tag_update_model)
        session.add(tag)
        session.commit()

        # Refresh the tag that was just created
        session.refresh(tag)
        return tag.to_model(include_metadata=True)
update_trigger(self, trigger_id, trigger_update)

Update a trigger.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger update.

required
trigger_update TriggerUpdate

The update request on the trigger.

required

Returns:

Type Description
TriggerResponse

The updated trigger.

Exceptions:

Type Description
KeyError

if the trigger doesn't exist.

Source code in zenml/zen_stores/sql_zen_store.py
@track_decorator(AnalyticsEvent.UPDATED_TRIGGER)
def update_trigger(
    self, trigger_id: UUID, trigger_update: TriggerUpdate
) -> TriggerResponse:
    """Update a trigger.

    Args:
        trigger_id: The ID of the trigger update.
        trigger_update: The update request on the trigger.

    Returns:
        The updated trigger.

    Raises:
        KeyError: if the trigger doesn't exist.
    """
    with Session(self.engine) as session:
        # Check if trigger with the domain key (name, workspace, owner)
        # already exists
        existing_trigger = session.exec(
            select(TriggerSchema).where(TriggerSchema.id == trigger_id)
        ).first()
        if existing_trigger is None:
            raise KeyError(
                f"Unable to update trigger with id '{trigger_id}': Found no"
                f"existing trigger with this id."
            )

        if trigger_update.service_account_id:
            # Verify that the given service account exists
            self._get_account_schema(
                account_name_or_id=trigger_update.service_account_id,
                session=session,
                service_account=True,
            )

        # In case of a renaming update, make sure no trigger already exists
        # with that name
        if trigger_update.name:
            if existing_trigger.name != trigger_update.name:
                self._fail_if_trigger_with_name_exists(
                    trigger_name=trigger_update.name,
                    workspace_id=existing_trigger.workspace.id,
                    session=session,
                )

        existing_trigger.update(
            trigger_update=trigger_update,
        )

        session.add(existing_trigger)
        session.commit()
        session.refresh(existing_trigger)

        return existing_trigger.to_model(
            include_metadata=True, include_resources=True
        )
update_user(self, user_id, user_update)

Updates an existing user.

Parameters:

Name Type Description Default
user_id UUID

The id of the user to update.

required
user_update UserUpdate

The update to be applied to the user.

required

Returns:

Type Description
UserResponse

The updated user.

Exceptions:

Type Description
IllegalOperationError

If the request tries to update the username for the default user account.

EntityExistsError

If the request tries to update the username to a name that is already taken by another user or service account.

Source code in zenml/zen_stores/sql_zen_store.py
def update_user(
    self, user_id: UUID, user_update: UserUpdate
) -> UserResponse:
    """Updates an existing user.

    Args:
        user_id: The id of the user to update.
        user_update: The update to be applied to the user.

    Returns:
        The updated user.

    Raises:
        IllegalOperationError: If the request tries to update the username
            for the default user account.
        EntityExistsError: If the request tries to update the username to
            a name that is already taken by another user or service account.
    """
    with Session(self.engine) as session:
        existing_user = self._get_account_schema(
            user_id, session=session, service_account=False
        )

        if (
            user_update.name is not None
            and user_update.name != existing_user.name
        ):
            if existing_user.name == self._default_user_name:
                raise IllegalOperationError(
                    "The username of the default user account cannot be "
                    "changed."
                )

            try:
                self._get_account_schema(
                    user_update.name,
                    session=session,
                    service_account=False,
                )
                raise EntityExistsError(
                    f"Unable to update user account with name "
                    f"'{user_update.name}': Found an existing user "
                    "account with this name."
                )
            except KeyError:
                pass

        existing_user.update_user(user_update=user_update)
        session.add(existing_user)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(existing_user)
        return existing_user.to_model(include_metadata=True)
update_workspace(self, workspace_id, workspace_update)

Update an existing workspace.

Parameters:

Name Type Description Default
workspace_id UUID

The ID of the workspace to be updated.

required
workspace_update WorkspaceUpdate

The update to be applied to the workspace.

required

Returns:

Type Description
WorkspaceResponse

The updated workspace.

Exceptions:

Type Description
IllegalOperationError

if the workspace is the default workspace.

KeyError

if the workspace does not exist.

Source code in zenml/zen_stores/sql_zen_store.py
def update_workspace(
    self, workspace_id: UUID, workspace_update: WorkspaceUpdate
) -> WorkspaceResponse:
    """Update an existing workspace.

    Args:
        workspace_id: The ID of the workspace to be updated.
        workspace_update: The update to be applied to the workspace.

    Returns:
        The updated workspace.

    Raises:
        IllegalOperationError: if the workspace is the default workspace.
        KeyError: if the workspace does not exist.
    """
    with Session(self.engine) as session:
        existing_workspace = session.exec(
            select(WorkspaceSchema).where(
                WorkspaceSchema.id == workspace_id
            )
        ).first()
        if existing_workspace is None:
            raise KeyError(
                f"Unable to update workspace with id "
                f"'{workspace_id}': Found no"
                f"existing workspaces with this id."
            )
        if (
            existing_workspace.name == self._default_workspace_name
            and "name" in workspace_update.__fields_set__
            and workspace_update.name != existing_workspace.name
        ):
            raise IllegalOperationError(
                "The name of the default workspace cannot be changed."
            )

        # Update the workspace
        existing_workspace.update(workspace_update=workspace_update)
        session.add(existing_workspace)
        session.commit()

        # Refresh the Model that was just created
        session.refresh(existing_workspace)
        return existing_workspace.to_model(include_metadata=True)
verify_service_connector(self, service_connector_id, resource_type=None, resource_id=None, list_resources=True)

Verifies if a service connector instance has access to one or more resources.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to verify.

required
resource_type Optional[str]

The type of resource to verify access to.

None
resource_id Optional[str]

The ID of the resource to verify access to.

None
list_resources bool

If True, the list of all resources accessible through the service connector and matching the supplied resource type and ID are returned.

True

Returns:

Type Description
ServiceConnectorResourcesModel

The list of resources that the service connector has access to, scoped to the supplied resource type and ID, if provided.

Source code in zenml/zen_stores/sql_zen_store.py
def verify_service_connector(
    self,
    service_connector_id: UUID,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
    list_resources: bool = True,
) -> ServiceConnectorResourcesModel:
    """Verifies if a service connector instance has access to one or more resources.

    Args:
        service_connector_id: The ID of the service connector to verify.
        resource_type: The type of resource to verify access to.
        resource_id: The ID of the resource to verify access to.
        list_resources: If True, the list of all resources accessible
            through the service connector and matching the supplied resource
            type and ID are returned.

    Returns:
        The list of resources that the service connector has access to,
        scoped to the supplied resource type and ID, if provided.
    """
    connector = self.get_service_connector(service_connector_id)

    connector_instance = service_connector_registry.instantiate_connector(
        model=connector
    )

    return connector_instance.verify(
        resource_type=resource_type,
        resource_id=resource_id,
        list_resources=list_resources,
    )
verify_service_connector_config(self, service_connector, list_resources=True)

Verifies if a service connector configuration has access to resources.

Parameters:

Name Type Description Default
service_connector ServiceConnectorRequest

The service connector configuration to verify.

required
list_resources bool

If True, the list of all resources accessible through the service connector is returned.

True

Returns:

Type Description
ServiceConnectorResourcesModel

The list of resources that the service connector configuration has access to.

Source code in zenml/zen_stores/sql_zen_store.py
def verify_service_connector_config(
    self,
    service_connector: ServiceConnectorRequest,
    list_resources: bool = True,
) -> ServiceConnectorResourcesModel:
    """Verifies if a service connector configuration has access to resources.

    Args:
        service_connector: The service connector configuration to verify.
        list_resources: If True, the list of all resources accessible
            through the service connector is returned.

    Returns:
        The list of resources that the service connector configuration has
        access to.
    """
    connector_instance = service_connector_registry.instantiate_connector(
        model=service_connector
    )
    return connector_instance.verify(list_resources=list_resources)

SqlZenStoreConfiguration (StoreConfiguration) pydantic-model

SQL ZenML store configuration.

Attributes:

Name Type Description
type StoreType

The type of the store.

secrets_store Optional[zenml.config.secrets_store_config.SecretsStoreConfiguration]

The configuration of the secrets store to use. This defaults to a SQL secrets store that extends the SQL ZenML store.

backup_secrets_store Optional[zenml.config.secrets_store_config.SecretsStoreConfiguration]

The configuration of a backup secrets store to use in addition to the primary one as an intermediate step during the migration to a new secrets store.

driver Optional[zenml.zen_stores.sql_zen_store.SQLDatabaseDriver]

The SQL database driver.

database Optional[str]

database name. If not already present on the server, it will be created automatically on first access.

username Optional[str]

The database username.

password Optional[str]

The database password.

ssl_ca Optional[str]

certificate authority certificate. Required for SSL enabled authentication if the CA certificate is not part of the certificates shipped by the operating system.

ssl_cert Optional[str]

client certificate. Required for SSL enabled authentication if client certificates are used.

ssl_key Optional[str]

client certificate private key. Required for SSL enabled if client certificates are used.

ssl_verify_server_cert bool

set to verify the identity of the server against the provided server certificate.

pool_size int

The maximum number of connections to keep in the SQLAlchemy pool.

max_overflow int

The maximum number of connections to allow in the SQLAlchemy pool in addition to the pool_size.

pool_pre_ping bool

Enable emitting a test statement on the SQL connection at the start of each connection pool checkout, to test that the database connection is still viable.

Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStoreConfiguration(StoreConfiguration):
    """SQL ZenML store configuration.

    Attributes:
        type: The type of the store.
        secrets_store: The configuration of the secrets store to use.
            This defaults to a SQL secrets store that extends the SQL ZenML
            store.
        backup_secrets_store: The configuration of a backup secrets store to
            use in addition to the primary one as an intermediate step during
            the migration to a new secrets store.
        driver: The SQL database driver.
        database: database name. If not already present on the server, it will
            be created automatically on first access.
        username: The database username.
        password: The database password.
        ssl_ca: certificate authority certificate. Required for SSL
            enabled authentication if the CA certificate is not part of the
            certificates shipped by the operating system.
        ssl_cert: client certificate. Required for SSL enabled
            authentication if client certificates are used.
        ssl_key: client certificate private key. Required for SSL
            enabled if client certificates are used.
        ssl_verify_server_cert: set to verify the identity of the server
            against the provided server certificate.
        pool_size: The maximum number of connections to keep in the SQLAlchemy
            pool.
        max_overflow: The maximum number of connections to allow in the
            SQLAlchemy pool in addition to the pool_size.
        pool_pre_ping: Enable emitting a test statement on the SQL connection
            at the start of each connection pool checkout, to test that the
            database connection is still viable.
    """

    type: StoreType = StoreType.SQL

    secrets_store: Optional[SecretsStoreConfiguration] = None
    backup_secrets_store: Optional[SecretsStoreConfiguration] = None

    driver: Optional[SQLDatabaseDriver] = None
    database: Optional[str] = None
    username: Optional[str] = None
    password: Optional[str] = None
    ssl_ca: Optional[str] = None
    ssl_cert: Optional[str] = None
    ssl_key: Optional[str] = None
    ssl_verify_server_cert: bool = False
    pool_size: int = 20
    max_overflow: int = 20
    pool_pre_ping: bool = True

    backup_strategy: DatabaseBackupStrategy = DatabaseBackupStrategy.IN_MEMORY
    # database backup directory
    backup_directory: str = Field(
        default_factory=lambda: os.path.join(
            GlobalConfiguration().config_directory,
            SQL_STORE_BACKUP_DIRECTORY_NAME,
        )
    )
    backup_database: Optional[str] = None

    @validator("secrets_store")
    def validate_secrets_store(
        cls, secrets_store: Optional[SecretsStoreConfiguration]
    ) -> SecretsStoreConfiguration:
        """Ensures that the secrets store is initialized with a default SQL secrets store.

        Args:
            secrets_store: The secrets store config to be validated.

        Returns:
            The validated secrets store config.
        """
        if secrets_store is None:
            secrets_store = SqlSecretsStoreConfiguration()

        return secrets_store

    @root_validator(pre=True)
    def _remove_grpc_attributes(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Removes old GRPC attributes.

        Args:
            values: All model attribute values.

        Returns:
            The model attribute values
        """
        grpc_attribute_keys = [
            "grpc_metadata_host",
            "grpc_metadata_port",
            "grpc_metadata_ssl_ca",
            "grpc_metadata_ssl_key",
            "grpc_metadata_ssl_cert",
        ]
        grpc_values = [values.pop(key, None) for key in grpc_attribute_keys]
        if any(grpc_values):
            logger.warning(
                "The GRPC attributes %s are unused and will be removed soon. "
                "Please remove them from SQLZenStore configuration. This will "
                "become an error in future versions of ZenML."
            )

        return values

    @root_validator
    def _validate_backup_strategy(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Validate the backup strategy.

        Args:
            values: All model attribute values.

        Returns:
            The model attribute values.

        Raises:
            ValueError: If the backup database name is not set when the backup
                database is requested.
        """
        backup_strategy = values.get("backup_strategy")
        if backup_strategy == DatabaseBackupStrategy.DATABASE and (
            not values.get("backup_database")
        ):
            raise ValueError(
                "The `backup_database` attribute must also be set if the "
                "backup strategy is set to use a backup database."
            )

        return values

    @root_validator
    def _validate_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
        """Validate the SQL URL.

        The validator also moves the MySQL username, password and database
        parameters from the URL into the other configuration arguments, if they
        are present in the URL.

        Args:
            values: The values to validate.

        Returns:
            The validated values.

        Raises:
            ValueError: If the URL is invalid or the SQL driver is not
                supported.
        """
        url = values.get("url")
        if url is None:
            return values

        # When running inside a container, if the URL uses localhost, the
        # target service will not be available. We try to replace localhost
        # with one of the special Docker or K3D internal hostnames.
        url = replace_localhost_with_internal_hostname(url)

        try:
            sql_url = make_url(url)
        except ArgumentError as e:
            raise ValueError(
                "Invalid SQL URL `%s`: %s. The URL must be in the format "
                "`driver://[[username:password@]hostname:port]/database["
                "?<extra-args>]`.",
                url,
                str(e),
            )

        if sql_url.drivername not in SQLDatabaseDriver.values():
            raise ValueError(
                "Invalid SQL driver value `%s`: The driver must be one of: %s.",
                url,
                ", ".join(SQLDatabaseDriver.values()),
            )
        values["driver"] = SQLDatabaseDriver(sql_url.drivername)
        if sql_url.drivername == SQLDatabaseDriver.SQLITE:
            if (
                sql_url.username
                or sql_url.password
                or sql_url.query
                or sql_url.database is None
            ):
                raise ValueError(
                    "Invalid SQLite URL `%s`: The URL must be in the "
                    "format `sqlite:///path/to/database.db`.",
                    url,
                )
            if values.get("username") or values.get("password"):
                raise ValueError(
                    "Invalid SQLite configuration: The username and password "
                    "must not be set",
                    url,
                )
            values["database"] = sql_url.database
        elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
            if sql_url.username:
                values["username"] = sql_url.username
                sql_url = sql_url._replace(username=None)
            if sql_url.password:
                values["password"] = sql_url.password
                sql_url = sql_url._replace(password=None)
            if sql_url.database:
                values["database"] = sql_url.database
                sql_url = sql_url._replace(database=None)
            if sql_url.query:
                for k, v in sql_url.query.items():
                    if k == "ssl_ca":
                        values["ssl_ca"] = v
                    elif k == "ssl_cert":
                        values["ssl_cert"] = v
                    elif k == "ssl_key":
                        values["ssl_key"] = v
                    elif k == "ssl_verify_server_cert":
                        values["ssl_verify_server_cert"] = v
                    else:
                        raise ValueError(
                            "Invalid MySQL URL query parameter `%s`: The "
                            "parameter must be one of: ssl_ca, ssl_cert, "
                            "ssl_key, or ssl_verify_server_cert.",
                            k,
                        )
                sql_url = sql_url._replace(query={})

            database = values.get("database")
            if (
                not values.get("username")
                or not values.get("password")
                or not database
            ):
                raise ValueError(
                    "Invalid MySQL configuration: The username, password and "
                    "database must be set in the URL or as configuration "
                    "attributes",
                )

            regexp = r"^[^\\/?%*:|\"<>.-]{1,64}$"
            match = re.match(regexp, database)
            if not match:
                raise ValueError(
                    f"The database name does not conform to the required "
                    f"format "
                    f"rules ({regexp}): {database}"
                )

            # Save the certificates in a secure location on disk
            secret_folder = Path(
                GlobalConfiguration().local_stores_path,
                "certificates",
            )
            for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
                content = values.get(key)
                if content and not os.path.isfile(content):
                    fileio.makedirs(str(secret_folder))
                    file_path = Path(secret_folder, f"{key}.pem")
                    with open(file_path, "w") as f:
                        f.write(content)
                    file_path.chmod(0o600)
                    values[key] = str(file_path)

        values["url"] = str(sql_url)
        return values

    @staticmethod
    def get_local_url(path: str) -> str:
        """Get a local SQL url for a given local path.

        Args:
            path: The path to the local sqlite file.

        Returns:
            The local SQL url for the given path.
        """
        return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"

    @classmethod
    def supports_url_scheme(cls, url: str) -> bool:
        """Check if a URL scheme is supported by this store.

        Args:
            url: The URL to check.

        Returns:
            True if the URL scheme is supported, False otherwise.
        """
        return make_url(url).drivername in SQLDatabaseDriver.values()

    def expand_certificates(self) -> None:
        """Expands the certificates in the verify_ssl field."""
        # Load the certificate values back into the configuration
        for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
            file_path = getattr(self, key, None)
            if file_path and os.path.isfile(file_path):
                with open(file_path, "r") as f:
                    setattr(self, key, f.read())

    def get_sqlalchemy_config(
        self,
        database: Optional[str] = None,
    ) -> Tuple[URL, Dict[str, Any], Dict[str, Any]]:
        """Get the SQLAlchemy engine configuration for the SQL ZenML store.

        Args:
            database: Custom database name to use. If not set, the database name
                from the configuration will be used.

        Returns:
            The URL and connection arguments for the SQLAlchemy engine.

        Raises:
            NotImplementedError: If the SQL driver is not supported.
        """
        sql_url = make_url(self.url)
        sqlalchemy_connect_args: Dict[str, Any] = {}
        engine_args = {}
        if sql_url.drivername == SQLDatabaseDriver.SQLITE:
            assert self.database is not None
            # The following default value is needed for sqlite to avoid the
            # Error:
            #   sqlite3.ProgrammingError: SQLite objects created in a thread can
            #   only be used in that same thread.
            sqlalchemy_connect_args = {"check_same_thread": False}
        elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
            # all these are guaranteed by our root validator
            assert self.database is not None
            assert self.username is not None
            assert self.password is not None
            assert sql_url.host is not None

            if not database:
                database = self.database

            engine_args = {
                "pool_size": self.pool_size,
                "max_overflow": self.max_overflow,
                "pool_pre_ping": self.pool_pre_ping,
            }

            sql_url = sql_url._replace(
                drivername="mysql+pymysql",
                username=self.username,
                password=self.password,
                database=database,
            )

            sqlalchemy_ssl_args: Dict[str, Any] = {}

            # Handle SSL params
            for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
                ssl_setting = getattr(self, key)
                if not ssl_setting:
                    continue
                if not os.path.isfile(ssl_setting):
                    logger.warning(
                        f"Database SSL setting `{key}` is not a file. "
                    )
                sqlalchemy_ssl_args[key.lstrip("ssl_")] = ssl_setting
            if len(sqlalchemy_ssl_args) > 0:
                sqlalchemy_ssl_args["check_hostname"] = (
                    self.ssl_verify_server_cert
                )
                sqlalchemy_connect_args["ssl"] = sqlalchemy_ssl_args
        else:
            raise NotImplementedError(
                f"SQL driver `{sql_url.drivername}` is not supported."
            )

        return sql_url, sqlalchemy_connect_args, engine_args

    class Config:
        """Pydantic configuration class."""

        # Don't validate attributes when assigning them. This is necessary
        # because the certificate attributes can be expanded to the contents
        # of the certificate files.
        validate_assignment = False
        # Forbid extra attributes set in the class.
        extra = "forbid"
Config

Pydantic configuration class.

Source code in zenml/zen_stores/sql_zen_store.py
class Config:
    """Pydantic configuration class."""

    # Don't validate attributes when assigning them. This is necessary
    # because the certificate attributes can be expanded to the contents
    # of the certificate files.
    validate_assignment = False
    # Forbid extra attributes set in the class.
    extra = "forbid"
expand_certificates(self)

Expands the certificates in the verify_ssl field.

Source code in zenml/zen_stores/sql_zen_store.py
def expand_certificates(self) -> None:
    """Expands the certificates in the verify_ssl field."""
    # Load the certificate values back into the configuration
    for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
        file_path = getattr(self, key, None)
        if file_path and os.path.isfile(file_path):
            with open(file_path, "r") as f:
                setattr(self, key, f.read())
get_local_url(path) staticmethod

Get a local SQL url for a given local path.

Parameters:

Name Type Description Default
path str

The path to the local sqlite file.

required

Returns:

Type Description
str

The local SQL url for the given path.

Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def get_local_url(path: str) -> str:
    """Get a local SQL url for a given local path.

    Args:
        path: The path to the local sqlite file.

    Returns:
        The local SQL url for the given path.
    """
    return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"
get_sqlalchemy_config(self, database=None)

Get the SQLAlchemy engine configuration for the SQL ZenML store.

Parameters:

Name Type Description Default
database Optional[str]

Custom database name to use. If not set, the database name from the configuration will be used.

None

Returns:

Type Description
Tuple[sqlalchemy.engine.url.URL, Dict[str, Any], Dict[str, Any]]

The URL and connection arguments for the SQLAlchemy engine.

Exceptions:

Type Description
NotImplementedError

If the SQL driver is not supported.

Source code in zenml/zen_stores/sql_zen_store.py
def get_sqlalchemy_config(
    self,
    database: Optional[str] = None,
) -> Tuple[URL, Dict[str, Any], Dict[str, Any]]:
    """Get the SQLAlchemy engine configuration for the SQL ZenML store.

    Args:
        database: Custom database name to use. If not set, the database name
            from the configuration will be used.

    Returns:
        The URL and connection arguments for the SQLAlchemy engine.

    Raises:
        NotImplementedError: If the SQL driver is not supported.
    """
    sql_url = make_url(self.url)
    sqlalchemy_connect_args: Dict[str, Any] = {}
    engine_args = {}
    if sql_url.drivername == SQLDatabaseDriver.SQLITE:
        assert self.database is not None
        # The following default value is needed for sqlite to avoid the
        # Error:
        #   sqlite3.ProgrammingError: SQLite objects created in a thread can
        #   only be used in that same thread.
        sqlalchemy_connect_args = {"check_same_thread": False}
    elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
        # all these are guaranteed by our root validator
        assert self.database is not None
        assert self.username is not None
        assert self.password is not None
        assert sql_url.host is not None

        if not database:
            database = self.database

        engine_args = {
            "pool_size": self.pool_size,
            "max_overflow": self.max_overflow,
            "pool_pre_ping": self.pool_pre_ping,
        }

        sql_url = sql_url._replace(
            drivername="mysql+pymysql",
            username=self.username,
            password=self.password,
            database=database,
        )

        sqlalchemy_ssl_args: Dict[str, Any] = {}

        # Handle SSL params
        for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
            ssl_setting = getattr(self, key)
            if not ssl_setting:
                continue
            if not os.path.isfile(ssl_setting):
                logger.warning(
                    f"Database SSL setting `{key}` is not a file. "
                )
            sqlalchemy_ssl_args[key.lstrip("ssl_")] = ssl_setting
        if len(sqlalchemy_ssl_args) > 0:
            sqlalchemy_ssl_args["check_hostname"] = (
                self.ssl_verify_server_cert
            )
            sqlalchemy_connect_args["ssl"] = sqlalchemy_ssl_args
    else:
        raise NotImplementedError(
            f"SQL driver `{sql_url.drivername}` is not supported."
        )

    return sql_url, sqlalchemy_connect_args, engine_args
supports_url_scheme(url) classmethod

Check if a URL scheme is supported by this store.

Parameters:

Name Type Description Default
url str

The URL to check.

required

Returns:

Type Description
bool

True if the URL scheme is supported, False otherwise.

Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
    """Check if a URL scheme is supported by this store.

    Args:
        url: The URL to check.

    Returns:
        True if the URL scheme is supported, False otherwise.
    """
    return make_url(url).drivername in SQLDatabaseDriver.values()
validate_secrets_store(secrets_store) classmethod

Ensures that the secrets store is initialized with a default SQL secrets store.

Parameters:

Name Type Description Default
secrets_store Optional[zenml.config.secrets_store_config.SecretsStoreConfiguration]

The secrets store config to be validated.

required

Returns:

Type Description
SecretsStoreConfiguration

The validated secrets store config.

Source code in zenml/zen_stores/sql_zen_store.py
@validator("secrets_store")
def validate_secrets_store(
    cls, secrets_store: Optional[SecretsStoreConfiguration]
) -> SecretsStoreConfiguration:
    """Ensures that the secrets store is initialized with a default SQL secrets store.

    Args:
        secrets_store: The secrets store config to be validated.

    Returns:
        The validated secrets store config.
    """
    if secrets_store is None:
        secrets_store = SqlSecretsStoreConfiguration()

    return secrets_store

zen_store_interface

ZenML Store interface.

ZenStoreInterface (ABC)

ZenML store interface.

All ZenML stores must implement the methods in this interface.

The methods in this interface are organized in the following way:

  • they are grouped into categories based on the type of resource that they operate on (e.g. stacks, stack components, etc.)

  • each category has a set of CRUD methods (create, read, update, delete) that operate on the resources in that category. The order of the methods in each category should be:

  • create methods - store a new resource. These methods should fill in generated fields (e.g. UUIDs, creation timestamps) in the resource and return the updated resource.

  • get methods - retrieve a single existing resource identified by a unique key or identifier from the store. These methods should always return a resource and raise an exception if the resource does not exist.
  • list methods - retrieve a list of resources from the store. These methods should accept a set of filter parameters that can be used to filter the list of resources retrieved from the store.
  • update methods - update an existing resource in the store. These methods should expect the updated resource to be correctly identified by its unique key or identifier and raise an exception if the resource does not exist.
  • delete methods - delete an existing resource from the store. These methods should expect the resource to be correctly identified by its unique key or identifier. If the resource does not exist, an exception should be raised.

Best practices for implementing and keeping this interface clean and easy to maintain and extend:

  • keep methods organized by resource type and ordered by CRUD operation
  • for resources with multiple keys, don't implement multiple get or list methods here if the same functionality can be achieved by a single get or list method. Instead, implement them in the BaseZenStore class and have them call the generic get or list method in this interface.
  • keep the logic required to convert between ZenML domain Model classes and internal store representations outside the ZenML domain Model classes
  • methods for resources that have two or more unique keys (e.g. a Workspace is uniquely identified by its name as well as its UUID) should reflect that in the method variants and/or method arguments:
    • methods that take in a resource identifier as argument should accept all variants of the identifier (e.g. workspace_name_or_uuid for methods that get/list/update/delete Workspaces)
    • if a compound key is involved, separate get methods should be implemented (e.g. get_pipeline to get a pipeline by ID and get_pipeline_in_workspace to get a pipeline by its name and the ID of the workspace it belongs to)
  • methods for resources that are scoped as children of other resources (e.g. a Stack is always owned by a Workspace) should reflect the key(s) of the parent resource in the provided methods and method arguments:
    • create methods should take the parent resource UUID(s) as an argument (e.g. create_stack takes in the workspace ID)
    • get methods should be provided to retrieve a resource by the compound key that includes the parent resource key(s)
    • list methods should feature optional filter arguments that reflect the parent resource key(s)
Source code in zenml/zen_stores/zen_store_interface.py
class ZenStoreInterface(ABC):
    """ZenML store interface.

    All ZenML stores must implement the methods in this interface.

    The methods in this interface are organized in the following way:

     * they are grouped into categories based on the type of resource
       that they operate on (e.g. stacks, stack components, etc.)

     * each category has a set of CRUD methods (create, read, update, delete)
       that operate on the resources in that category. The order of the methods
       in each category should be:

       * create methods - store a new resource. These methods
         should fill in generated fields (e.g. UUIDs, creation timestamps) in
         the resource and return the updated resource.
       * get methods - retrieve a single existing resource identified by a
         unique key or identifier from the store. These methods should always
         return a resource and raise an exception if the resource does not
         exist.
       * list methods - retrieve a list of resources from the store. These
         methods should accept a set of filter parameters that can be used to
         filter the list of resources retrieved from the store.
       * update methods - update an existing resource in the store. These
         methods should expect the updated resource to be correctly identified
         by its unique key or identifier and raise an exception if the resource
         does not exist.
       * delete methods - delete an existing resource from the store. These
         methods should expect the resource to be correctly identified by its
         unique key or identifier. If the resource does not exist,
         an exception should be raised.

    Best practices for implementing and keeping this interface clean and easy to
    maintain and extend:

      * keep methods organized by resource type and ordered by CRUD operation
      * for resources with multiple keys, don't implement multiple get or list
      methods here if the same functionality can be achieved by a single get or
      list method. Instead, implement them in the BaseZenStore class and have
      them call the generic get or list method in this interface.
      * keep the logic required to convert between ZenML domain Model classes
      and internal store representations outside the ZenML domain Model classes
      * methods for resources that have two or more unique keys (e.g. a Workspace
      is uniquely identified by its name as well as its UUID) should reflect
      that in the method variants and/or method arguments:
        * methods that take in a resource identifier as argument should accept
        all variants of the identifier (e.g. `workspace_name_or_uuid` for methods
        that get/list/update/delete Workspaces)
        * if a compound key is involved, separate get methods should be
        implemented (e.g. `get_pipeline` to get a pipeline by ID and
        `get_pipeline_in_workspace` to get a pipeline by its name and the ID of
        the workspace it belongs to)
      * methods for resources that are scoped as children of other resources
      (e.g. a Stack is always owned by a Workspace) should reflect the
      key(s) of the parent resource in the provided methods and method
      arguments:
        * create methods should take the parent resource UUID(s) as an argument
        (e.g. `create_stack` takes in the workspace ID)
        * get methods should be provided to retrieve a resource by the compound
        key that includes the parent resource key(s)
        * list methods should feature optional filter arguments that reflect
        the parent resource key(s)
    """

    # ---------------------------------
    # Initialization and configuration
    # ---------------------------------

    @abstractmethod
    def _initialize(self) -> None:
        """Initialize the store.

        This method is called immediately after the store is created. It should
        be used to set up the backend (database, connection etc.).
        """

    @abstractmethod
    def get_store_info(self) -> ServerModel:
        """Get information about the store.

        Returns:
            Information about the store.
        """

    @abstractmethod
    def get_deployment_id(self) -> UUID:
        """Get the ID of the deployment.

        Returns:
            The ID of the deployment.
        """

    # -------------------- API Keys --------------------

    @abstractmethod
    def create_api_key(
        self, service_account_id: UUID, api_key: APIKeyRequest
    ) -> APIKeyResponse:
        """Create a new API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                create the API key.
            api_key: The API key to create.

        Returns:
            The created API key.

        Raises:
            KeyError: If the service account doesn't exist.
            EntityExistsError: If an API key with the same name is already
                configured for the same service account.
        """

    @abstractmethod
    def get_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        hydrate: bool = True,
    ) -> APIKeyResponse:
        """Get an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to fetch
                the API key.
            api_key_name_or_id: The name or ID of the API key to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The API key with the given ID.

        Raises:
            KeyError: if an API key with the given name or ID is not configured
                for the given service account.
        """

    @abstractmethod
    def list_api_keys(
        self,
        service_account_id: UUID,
        filter_model: APIKeyFilter,
        hydrate: bool = False,
    ) -> Page[APIKeyResponse]:
        """List all API keys for a service account matching the given filter criteria.

        Args:
            service_account_id: The ID of the service account for which to list
                the API keys.
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all API keys matching the filter criteria.
        """

    @abstractmethod
    def update_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        api_key_update: APIKeyUpdate,
    ) -> APIKeyResponse:
        """Update an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to update
                the API key.
            api_key_name_or_id: The name or ID of the API key to update.
            api_key_update: The update request on the API key.

        Returns:
            The updated API key.

        Raises:
            KeyError: if an API key with the given name or ID is not configured
                for the given service account.
            EntityExistsError: if the API key update would result in a name
                conflict with an existing API key for the same service account.
        """

    @abstractmethod
    def rotate_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
        rotate_request: APIKeyRotateRequest,
    ) -> APIKeyResponse:
        """Rotate an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                rotate the API key.
            api_key_name_or_id: The name or ID of the API key to rotate.
            rotate_request: The rotate request on the API key.

        Returns:
            The updated API key.

        Raises:
            KeyError: if an API key with the given name or ID is not configured
                for the given service account.
        """

    @abstractmethod
    def delete_api_key(
        self,
        service_account_id: UUID,
        api_key_name_or_id: Union[str, UUID],
    ) -> None:
        """Delete an API key for a service account.

        Args:
            service_account_id: The ID of the service account for which to
                delete the API key.
            api_key_name_or_id: The name or ID of the API key to delete.

        Raises:
            KeyError: if an API key with the given name or ID is not configured
                for the given service account.
        """

    # -------------------- Artifacts --------------------

    @abstractmethod
    def create_artifact(self, artifact: ArtifactRequest) -> ArtifactResponse:
        """Creates a new artifact.

        Args:
            artifact: The artifact to create.

        Returns:
            The newly created artifact.

        Raises:
            EntityExistsError: If an artifact with the same name already exists.
        """

    @abstractmethod
    def get_artifact(
        self, artifact_id: UUID, hydrate: bool = True
    ) -> ArtifactResponse:
        """Gets an artifact.

        Args:
            artifact_id: The ID of the artifact to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact.

        Raises:
            KeyError: if the artifact doesn't exist.
        """

    @abstractmethod
    def list_artifacts(
        self, filter_model: ArtifactFilter, hydrate: bool = False
    ) -> Page[ArtifactResponse]:
        """List all artifacts matching the given filter criteria.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all artifacts matching the filter criteria.
        """

    @abstractmethod
    def update_artifact(
        self, artifact_id: UUID, artifact_update: ArtifactUpdate
    ) -> ArtifactResponse:
        """Updates an artifact.

        Args:
            artifact_id: The ID of the artifact to update.
            artifact_update: The update to be applied to the artifact.

        Returns:
            The updated artifact.

        Raises:
            KeyError: if the artifact doesn't exist.
        """

    @abstractmethod
    def delete_artifact(self, artifact_id: UUID) -> None:
        """Deletes an artifact.

        Args:
            artifact_id: The ID of the artifact to delete.

        Raises:
            KeyError: if the artifact doesn't exist.
        """

    # -------------------- Artifact Versions --------------------

    @abstractmethod
    def create_artifact_version(
        self, artifact_version: ArtifactVersionRequest
    ) -> ArtifactVersionResponse:
        """Creates an artifact version.

        Args:
            artifact_version: The artifact version to create.

        Returns:
            The created artifact version.
        """

    @abstractmethod
    def get_artifact_version(
        self, artifact_version_id: UUID, hydrate: bool = True
    ) -> ArtifactVersionResponse:
        """Gets an artifact version.

        Args:
            artifact_version_id: The ID of the artifact version to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact version.

        Raises:
            KeyError: if the artifact version doesn't exist.
        """

    @abstractmethod
    def list_artifact_versions(
        self,
        artifact_version_filter_model: ArtifactVersionFilter,
        hydrate: bool = False,
    ) -> Page[ArtifactVersionResponse]:
        """List all artifact versions matching the given filter criteria.

        Args:
            artifact_version_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all artifact versions matching the filter criteria.
        """

    @abstractmethod
    def update_artifact_version(
        self,
        artifact_version_id: UUID,
        artifact_version_update: ArtifactVersionUpdate,
    ) -> ArtifactVersionResponse:
        """Updates an artifact version.

        Args:
            artifact_version_id: The ID of the artifact version to update.
            artifact_version_update: The update to be applied to the artifact
                version.

        Returns:
            The updated artifact version.

        Raises:
            KeyError: if the artifact version doesn't exist.
        """

    @abstractmethod
    def delete_artifact_version(self, artifact_version_id: UUID) -> None:
        """Deletes an artifact version.

        Args:
            artifact_version_id: The ID of the artifact version to delete.

        Raises:
            KeyError: if the artifact version doesn't exist.
        """

    @abstractmethod
    def prune_artifact_versions(
        self,
        only_versions: bool = True,
    ) -> None:
        """Prunes unused artifact versions and their artifacts.

        Args:
            only_versions: Only delete artifact versions, keeping artifacts
        """

    # -------------------- Artifact Visualization --------------------

    @abstractmethod
    def get_artifact_visualization(
        self, artifact_visualization_id: UUID, hydrate: bool = True
    ) -> ArtifactVisualizationResponse:
        """Gets an artifact visualization.

        Args:
            artifact_visualization_id: The ID of the artifact visualization
                to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The artifact visualization.

        Raises:
            KeyError: if the artifact visualization doesn't exist.
        """

    # -------------------- Code References --------------------

    @abstractmethod
    def get_code_reference(
        self, code_reference_id: UUID, hydrate: bool = True
    ) -> CodeReferenceResponse:
        """Gets a specific code reference.

        Args:
            code_reference_id: The ID of the code reference to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested code reference, if it was found.

        Raises:
            KeyError: If no code reference with the given ID exists.
        """

    # -------------------- Code repositories --------------------

    @abstractmethod
    def create_code_repository(
        self, code_repository: CodeRepositoryRequest
    ) -> CodeRepositoryResponse:
        """Creates a new code repository.

        Args:
            code_repository: Code repository to be created.

        Returns:
            The newly created code repository.

        Raises:
            EntityExistsError: If a code repository with the given name already
                exists.
        """

    @abstractmethod
    def get_code_repository(
        self, code_repository_id: UUID, hydrate: bool = True
    ) -> CodeRepositoryResponse:
        """Gets a specific code repository.

        Args:
            code_repository_id: The ID of the code repository to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested code repository, if it was found.

        Raises:
            KeyError: If no code repository with the given ID exists.
        """

    @abstractmethod
    def list_code_repositories(
        self, filter_model: CodeRepositoryFilter, hydrate: bool = False
    ) -> Page[CodeRepositoryResponse]:
        """List all code repositories.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all code repositories.
        """

    @abstractmethod
    def update_code_repository(
        self, code_repository_id: UUID, update: CodeRepositoryUpdate
    ) -> CodeRepositoryResponse:
        """Updates an existing code repository.

        Args:
            code_repository_id: The ID of the code repository to update.
            update: The update to be applied to the code repository.

        Returns:
            The updated code repository.

        Raises:
            KeyError: If no code repository with the given name exists.
        """

    @abstractmethod
    def delete_code_repository(self, code_repository_id: UUID) -> None:
        """Deletes a code repository.

        Args:
            code_repository_id: The ID of the code repository to delete.

        Raises:
            KeyError: If no code repository with the given ID exists.
        """

    # -------------------- Components --------------------

    @abstractmethod
    def create_stack_component(
        self, component: ComponentRequest
    ) -> ComponentResponse:
        """Create a stack component.

        Args:
            component: The stack component to create.

        Returns:
            The created stack component.

        Raises:
            StackComponentExistsError: If a stack component with the same name
                and type is already owned by this user in this workspace.
        """

    @abstractmethod
    def get_stack_component(
        self,
        component_id: UUID,
        hydrate: bool = True,
    ) -> ComponentResponse:
        """Get a stack component by ID.

        Args:
            component_id: The ID of the stack component to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack component.

        Raises:
            KeyError: if the stack component doesn't exist.
        """

    @abstractmethod
    def list_stack_components(
        self,
        component_filter_model: ComponentFilter,
        hydrate: bool = False,
    ) -> Page[ComponentResponse]:
        """List all stack components matching the given filter criteria.

        Args:
            component_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all stack components matching the filter criteria.
        """

    @abstractmethod
    def update_stack_component(
        self,
        component_id: UUID,
        component_update: ComponentUpdate,
    ) -> ComponentResponse:
        """Update an existing stack component.

        Args:
            component_id: The ID of the stack component to update.
            component_update: The update to be applied to the stack component.

        Returns:
            The updated stack component.

        Raises:
            KeyError: if the stack component doesn't exist.
        """

    @abstractmethod
    def delete_stack_component(self, component_id: UUID) -> None:
        """Delete a stack component.

        Args:
            component_id: The ID of the stack component to delete.

        Raises:
            KeyError: if the stack component doesn't exist.
            ValueError: if the stack component is part of one or more stacks.
        """

    # -------------------- Devices --------------------

    @abstractmethod
    def get_authorized_device(
        self, device_id: UUID, hydrate: bool = True
    ) -> OAuthDeviceResponse:
        """Gets a specific OAuth 2.0 authorized device.

        Args:
            device_id: The ID of the device to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested device, if it was found.

        Raises:
            KeyError: If no device with the given ID exists.
        """

    @abstractmethod
    def list_authorized_devices(
        self, filter_model: OAuthDeviceFilter, hydrate: bool = False
    ) -> Page[OAuthDeviceResponse]:
        """List all OAuth 2.0 authorized devices for a user.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all matching OAuth 2.0 authorized devices.
        """

    @abstractmethod
    def update_authorized_device(
        self, device_id: UUID, update: OAuthDeviceUpdate
    ) -> OAuthDeviceResponse:
        """Updates an existing OAuth 2.0 authorized device for internal use.

        Args:
            device_id: The ID of the device to update.
            update: The update to be applied to the device.

        Returns:
            The updated OAuth 2.0 authorized device.

        Raises:
            KeyError: If no device with the given ID exists.
        """

    @abstractmethod
    def delete_authorized_device(self, device_id: UUID) -> None:
        """Deletes an OAuth 2.0 authorized device.

        Args:
            device_id: The ID of the device to delete.

        Raises:
            KeyError: If no device with the given ID exists.
        """

    # -------------------- Flavors --------------------

    @abstractmethod
    def create_flavor(
        self,
        flavor: FlavorRequest,
    ) -> FlavorResponse:
        """Creates a new stack component flavor.

        Args:
            flavor: The stack component flavor to create.

        Returns:
            The newly created flavor.

        Raises:
            EntityExistsError: If a flavor with the same name and type
                is already owned by this user in this workspace.
        """

    @abstractmethod
    def get_flavor(
        self, flavor_id: UUID, hydrate: bool = True
    ) -> FlavorResponse:
        """Get a stack component flavor by ID.

        Args:
            flavor_id: The ID of the flavor to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack component flavor.

        Raises:
            KeyError: if the stack component flavor doesn't exist.
        """

    @abstractmethod
    def update_flavor(
        self, flavor_id: UUID, flavor_update: FlavorUpdate
    ) -> FlavorResponse:
        """Updates an existing user.

        Args:
            flavor_id: The id of the flavor to update.
            flavor_update: The update to be applied to the flavor.

        Returns:
            The updated flavor.
        """

    @abstractmethod
    def list_flavors(
        self,
        flavor_filter_model: FlavorFilter,
        hydrate: bool = False,
    ) -> Page[FlavorResponse]:
        """List all stack component flavors matching the given filter criteria.

        Args:
            flavor_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            List of all the stack component flavors matching the given criteria.
        """

    @abstractmethod
    def delete_flavor(self, flavor_id: UUID) -> None:
        """Delete a stack component flavor.

        Args:
            flavor_id: The ID of the stack component flavor to delete.

        Raises:
            KeyError: if the stack component flavor doesn't exist.
        """

    # -------------------- Logs --------------------
    @abstractmethod
    def get_logs(self, logs_id: UUID, hydrate: bool = True) -> LogsResponse:
        """Get logs by its unique ID.

        Args:
            logs_id: The ID of the logs to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The logs with the given ID.

        Raises:
            KeyError: if the logs doesn't exist.
        """

    # -------------------- Pipelines --------------------

    @abstractmethod
    def create_pipeline(
        self,
        pipeline: PipelineRequest,
    ) -> PipelineResponse:
        """Creates a new pipeline in a workspace.

        Args:
            pipeline: The pipeline to create.

        Returns:
            The newly created pipeline.

        Raises:
            KeyError: if the workspace does not exist.
            EntityExistsError: If an identical pipeline already exists.
        """

    @abstractmethod
    def get_pipeline(
        self, pipeline_id: UUID, hydrate: bool = True
    ) -> PipelineResponse:
        """Get a pipeline with a given ID.

        Args:
            pipeline_id: ID of the pipeline.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The pipeline.

        Raises:
            KeyError: if the pipeline does not exist.
        """

    @abstractmethod
    def list_pipelines(
        self,
        pipeline_filter_model: PipelineFilter,
        hydrate: bool = False,
    ) -> Page[PipelineResponse]:
        """List all pipelines matching the given filter criteria.

        Args:
            pipeline_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all pipelines matching the filter criteria.
        """

    @abstractmethod
    def update_pipeline(
        self,
        pipeline_id: UUID,
        pipeline_update: PipelineUpdate,
    ) -> PipelineResponse:
        """Updates a pipeline.

        Args:
            pipeline_id: The ID of the pipeline to be updated.
            pipeline_update: The update to be applied.

        Returns:
            The updated pipeline.

        Raises:
            KeyError: if the pipeline doesn't exist.
        """

    @abstractmethod
    def delete_pipeline(self, pipeline_id: UUID) -> None:
        """Deletes a pipeline.

        Args:
            pipeline_id: The ID of the pipeline to delete.

        Raises:
            KeyError: if the pipeline doesn't exist.
        """

    # -------------------- Pipeline builds --------------------

    @abstractmethod
    def create_build(
        self,
        build: PipelineBuildRequest,
    ) -> PipelineBuildResponse:
        """Creates a new build in a workspace.

        Args:
            build: The build to create.

        Returns:
            The newly created build.

        Raises:
            KeyError: If the workspace does not exist.
            EntityExistsError: If an identical build already exists.
        """

    @abstractmethod
    def get_build(
        self, build_id: UUID, hydrate: bool = True
    ) -> PipelineBuildResponse:
        """Get a build with a given ID.

        Args:
            build_id: ID of the build.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The build.

        Raises:
            KeyError: If the build does not exist.
        """

    @abstractmethod
    def list_builds(
        self,
        build_filter_model: PipelineBuildFilter,
        hydrate: bool = False,
    ) -> Page[PipelineBuildResponse]:
        """List all builds matching the given filter criteria.

        Args:
            build_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all builds matching the filter criteria.
        """

    @abstractmethod
    def delete_build(self, build_id: UUID) -> None:
        """Deletes a build.

        Args:
            build_id: The ID of the build to delete.

        Raises:
            KeyError: if the build doesn't exist.
        """

    # -------------------- Pipeline deployments --------------------

    @abstractmethod
    def create_deployment(
        self,
        deployment: PipelineDeploymentRequest,
    ) -> PipelineDeploymentResponse:
        """Creates a new deployment in a workspace.

        Args:
            deployment: The deployment to create.

        Returns:
            The newly created deployment.

        Raises:
            KeyError: If the workspace does not exist.
            EntityExistsError: If an identical deployment already exists.
        """

    @abstractmethod
    def get_deployment(
        self, deployment_id: UUID, hydrate: bool = True
    ) -> PipelineDeploymentResponse:
        """Get a deployment with a given ID.

        Args:
            deployment_id: ID of the deployment.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The deployment.

        Raises:
            KeyError: If the deployment does not exist.
        """

    @abstractmethod
    def list_deployments(
        self,
        deployment_filter_model: PipelineDeploymentFilter,
        hydrate: bool = False,
    ) -> Page[PipelineDeploymentResponse]:
        """List all deployments matching the given filter criteria.

        Args:
            deployment_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all deployments matching the filter criteria.
        """

    @abstractmethod
    def delete_deployment(self, deployment_id: UUID) -> None:
        """Deletes a deployment.

        Args:
            deployment_id: The ID of the deployment to delete.

        Raises:
            KeyError: If the deployment doesn't exist.
        """

    # -------------------- Event Sources  --------------------

    @abstractmethod
    def create_event_source(
        self, event_source: EventSourceRequest
    ) -> EventSourceResponse:
        """Create an event_source.

        Args:
            event_source: The event_source to create.

        Returns:
            The created event_source.
        """

    @abstractmethod
    def get_event_source(
        self,
        event_source_id: UUID,
        hydrate: bool = True,
    ) -> EventSourceResponse:
        """Get an event_source by ID.

        Args:
            event_source_id: The ID of the event_source to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The event_source.

        Raises:
            KeyError: if the stack event_source doesn't exist.
        """

    @abstractmethod
    def list_event_sources(
        self,
        event_source_filter_model: EventSourceFilter,
        hydrate: bool = False,
    ) -> Page[EventSourceResponse]:
        """List all event_sources matching the given filter criteria.

        Args:
            event_source_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all event_sources matching the filter criteria.
        """

    @abstractmethod
    def update_event_source(
        self,
        event_source_id: UUID,
        event_source_update: EventSourceUpdate,
    ) -> EventSourceResponse:
        """Update an existing event_source.

        Args:
            event_source_id: The ID of the event_source to update.
            event_source_update: The update to be applied to the event_source.

        Returns:
            The updated event_source.

        Raises:
            KeyError: if the event_source doesn't exist.
        """

    @abstractmethod
    def delete_event_source(self, event_source_id: UUID) -> None:
        """Delete an event_source.

        Args:
            event_source_id: The ID of the event_source to delete.

        Raises:
            KeyError: if the event_source doesn't exist.
        """

    # -------------------- Pipeline runs --------------------

    @abstractmethod
    def create_run(
        self, pipeline_run: PipelineRunRequest
    ) -> PipelineRunResponse:
        """Creates a pipeline run.

        Args:
            pipeline_run: The pipeline run to create.

        Returns:
            The created pipeline run.

        Raises:
            EntityExistsError: If an identical pipeline run already exists.
            KeyError: If the pipeline does not exist.
        """

    @abstractmethod
    def get_run(
        self, run_name_or_id: Union[str, UUID], hydrate: bool = True
    ) -> PipelineRunResponse:
        """Gets a pipeline run.

        Args:
            run_name_or_id: The name or ID of the pipeline run to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The pipeline run.

        Raises:
            KeyError: if the pipeline run doesn't exist.
        """

    @abstractmethod
    def list_runs(
        self,
        runs_filter_model: PipelineRunFilter,
        hydrate: bool = False,
    ) -> Page[PipelineRunResponse]:
        """List all pipeline runs matching the given filter criteria.

        Args:
            runs_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all pipeline runs matching the filter criteria.
        """

    @abstractmethod
    def update_run(
        self, run_id: UUID, run_update: PipelineRunUpdate
    ) -> PipelineRunResponse:
        """Updates a pipeline run.

        Args:
            run_id: The ID of the pipeline run to update.
            run_update: The update to be applied to the pipeline run.

        Returns:
            The updated pipeline run.

        Raises:
            KeyError: if the pipeline run doesn't exist.
        """

    @abstractmethod
    def delete_run(self, run_id: UUID) -> None:
        """Deletes a pipeline run.

        Args:
            run_id: The ID of the pipeline run to delete.

        Raises:
            KeyError: if the pipeline run doesn't exist.
        """

    @abstractmethod
    def get_or_create_run(
        self, pipeline_run: PipelineRunRequest
    ) -> Tuple[PipelineRunResponse, bool]:
        """Gets or creates a pipeline run.

        If a run with the same ID or name already exists, it is returned.
        Otherwise, a new run is created.

        Args:
            pipeline_run: The pipeline run to get or create.

        Returns:
            The pipeline run, and a boolean indicating whether the run was
            created or not.
        """

    # -------------------- Run metadata --------------------

    @abstractmethod
    def create_run_metadata(
        self, run_metadata: RunMetadataRequest
    ) -> List[RunMetadataResponse]:
        """Creates run metadata.

        Args:
            run_metadata: The run metadata to create.

        Returns:
            The created run metadata.
        """

    @abstractmethod
    def get_run_metadata(
        self, run_metadata_id: UUID, hydrate: bool = True
    ) -> RunMetadataResponse:
        """Get run metadata by its unique ID.

        Args:
            run_metadata_id: The ID of the run metadata to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The run metadata with the given ID.

        Raises:
            KeyError: if the run metadata doesn't exist.
        """

    @abstractmethod
    def list_run_metadata(
        self,
        run_metadata_filter_model: RunMetadataFilter,
        hydrate: bool = False,
    ) -> Page[RunMetadataResponse]:
        """List run metadata.

        Args:
            run_metadata_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The run metadata.
        """

    # -------------------- Schedules --------------------

    @abstractmethod
    def create_schedule(self, schedule: ScheduleRequest) -> ScheduleResponse:
        """Creates a new schedule.

        Args:
            schedule: The schedule to create.

        Returns:
            The newly created schedule.
        """

    @abstractmethod
    def get_schedule(
        self, schedule_id: UUID, hydrate: bool = True
    ) -> ScheduleResponse:
        """Get a schedule with a given ID.

        Args:
            schedule_id: ID of the schedule.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The schedule.

        Raises:
            KeyError: if the schedule does not exist.
        """

    @abstractmethod
    def list_schedules(
        self,
        schedule_filter_model: ScheduleFilter,
        hydrate: bool = False,
    ) -> Page[ScheduleResponse]:
        """List all schedules in the workspace.

        Args:
            schedule_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of schedules.
        """

    @abstractmethod
    def update_schedule(
        self,
        schedule_id: UUID,
        schedule_update: ScheduleUpdate,
    ) -> ScheduleResponse:
        """Updates a schedule.

        Args:
            schedule_id: The ID of the schedule to be updated.
            schedule_update: The update to be applied.

        Returns:
            The updated schedule.

        Raises:
            KeyError: if the schedule doesn't exist.
        """

    @abstractmethod
    def delete_schedule(self, schedule_id: UUID) -> None:
        """Deletes a schedule.

        Args:
            schedule_id: The ID of the schedule to delete.

        Raises:
            KeyError: if the schedule doesn't exist.
        """

    # --------------------  Secrets --------------------

    @abstractmethod
    def create_secret(
        self,
        secret: SecretRequest,
    ) -> SecretResponse:
        """Creates a new secret.

        The new secret is also validated against the scoping rules enforced in
        the secrets store:

          - only one workspace-scoped secret with the given name can exist
            in the target workspace.
          - only one user-scoped secret with the given name can exist in the
            target workspace for the target user.

        Args:
            secret: The secret to create.

        Returns:
            The newly created secret.

        Raises:
            KeyError: if the user or workspace does not exist.
            EntityExistsError: If a secret with the same name already exists in
                the same scope.
        """

    @abstractmethod
    def get_secret(
        self, secret_id: UUID, hydrate: bool = True
    ) -> SecretResponse:
        """Get a secret with a given name.

        Args:
            secret_id: ID of the secret.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The secret.

        Raises:
            KeyError: if the secret does not exist.
        """

    @abstractmethod
    def list_secrets(
        self, secret_filter_model: SecretFilter, hydrate: bool = False
    ) -> Page[SecretResponse]:
        """List all secrets matching the given filter criteria.

        Note that returned secrets do not include any secret values. To fetch
        the secret values, use `get_secret`.

        Args:
            secret_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all secrets matching the filter criteria, with pagination
            information and sorted according to the filter criteria. The
            returned secrets do not include any secret values, only metadata. To
            fetch the secret values, use `get_secret` individually with each
            secret.
        """

    @abstractmethod
    def update_secret(
        self,
        secret_id: UUID,
        secret_update: SecretUpdate,
    ) -> SecretResponse:
        """Updates a secret.

        Secret values that are specified as `None` in the update that are
        present in the existing secret are removed from the existing secret.
        Values that are present in both secrets are overwritten. All other
        values in both the existing secret and the update are kept (merged).

        If the update includes a change of name or scope, the scoping rules
        enforced in the secrets store are used to validate the update:

          - only one workspace-scoped secret with the given name can exist
            in the target workspace.
          - only one user-scoped secret with the given name can exist in the
            target workspace for the target user.

        Args:
            secret_id: The ID of the secret to be updated.
            secret_update: The update to be applied.

        Returns:
            The updated secret.

        Raises:
            KeyError: if the secret doesn't exist.
            EntityExistsError: If a secret with the same name already exists in
                the same scope.
        """

    @abstractmethod
    def delete_secret(self, secret_id: UUID) -> None:
        """Deletes a secret.

        Args:
            secret_id: The ID of the secret to delete.

        Raises:
            KeyError: if the secret doesn't exist.
        """

    @abstractmethod
    def backup_secrets(
        self, ignore_errors: bool = True, delete_secrets: bool = False
    ) -> None:
        """Backs up all secrets to the configured backup secrets store.

        Args:
            ignore_errors: Whether to ignore individual errors during the backup
                process and attempt to backup all secrets.
            delete_secrets: Whether to delete the secrets that have been
                successfully backed up from the primary secrets store. Setting
                this flag effectively moves all secrets from the primary secrets
                store to the backup secrets store.

        Raises:
            BackupSecretsStoreNotConfiguredError: if no backup secrets store is
                configured.
        """

    @abstractmethod
    def restore_secrets(
        self, ignore_errors: bool = False, delete_secrets: bool = False
    ) -> None:
        """Restore all secrets from the configured backup secrets store.

        Args:
            ignore_errors: Whether to ignore individual errors during the
                restore process and attempt to restore all secrets.
            delete_secrets: Whether to delete the secrets that have been
                successfully restored from the backup secrets store. Setting
                this flag effectively moves all secrets from the backup secrets
                store to the primary secrets store.

        Raises:
            BackupSecretsStoreNotConfiguredError: if no backup secrets store is
                configured.
        """

    # --------------------  Service Accounts --------------------

    @abstractmethod
    def create_service_account(
        self, service_account: ServiceAccountRequest
    ) -> ServiceAccountResponse:
        """Creates a new service account.

        Args:
            service_account: Service account to be created.

        Returns:
            The newly created service account.

        Raises:
            EntityExistsError: If a user or service account with the given name
                already exists.
        """

    @abstractmethod
    def get_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
        hydrate: bool = True,
    ) -> ServiceAccountResponse:
        """Gets a specific service account.

        Args:
            service_account_name_or_id: The name or ID of the service account to
                get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested service account, if it was found.

        Raises:
            KeyError: If no service account with the given name or ID exists.
        """

    @abstractmethod
    def list_service_accounts(
        self,
        filter_model: ServiceAccountFilter,
        hydrate: bool = False,
    ) -> Page[ServiceAccountResponse]:
        """List all service accounts.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of filtered service accounts.
        """

    @abstractmethod
    def update_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
        service_account_update: ServiceAccountUpdate,
    ) -> ServiceAccountResponse:
        """Updates an existing service account.

        Args:
            service_account_name_or_id: The name or the ID of the service
                account to update.
            service_account_update: The update to be applied to the service
                account.

        Returns:
            The updated service account.

        Raises:
            KeyError: If no service account with the given name exists.
        """

    @abstractmethod
    def delete_service_account(
        self,
        service_account_name_or_id: Union[str, UUID],
    ) -> None:
        """Delete a service account.

        Args:
            service_account_name_or_id: The name or the ID of the service
                account to delete.

        Raises:
            IllegalOperationError: if the service account has already been used
                to create other resources.
        """

    # -------------------- Service Connectors --------------------

    @abstractmethod
    def create_service_connector(
        self,
        service_connector: ServiceConnectorRequest,
    ) -> ServiceConnectorResponse:
        """Creates a new service connector.

        Args:
            service_connector: Service connector to be created.

        Returns:
            The newly created service connector.

        Raises:
            EntityExistsError: If a service connector with the given name
                is already owned by this user in this workspace.
        """

    @abstractmethod
    def get_service_connector(
        self, service_connector_id: UUID, hydrate: bool = True
    ) -> ServiceConnectorResponse:
        """Gets a specific service connector.

        Args:
            service_connector_id: The ID of the service connector to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested service connector, if it was found.

        Raises:
            KeyError: If no service connector with the given ID exists.
        """

    @abstractmethod
    def list_service_connectors(
        self,
        filter_model: ServiceConnectorFilter,
        hydrate: bool = False,
    ) -> Page[ServiceConnectorResponse]:
        """List all service connectors.

        Args:
            filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all service connectors.
        """

    @abstractmethod
    def update_service_connector(
        self, service_connector_id: UUID, update: ServiceConnectorUpdate
    ) -> ServiceConnectorResponse:
        """Updates an existing service connector.

        The update model contains the fields to be updated. If a field value is
        set to None in the model, the field is not updated, but there are
        special rules concerning some fields:

        * the `configuration` and `secrets` fields together represent a full
        valid configuration update, not just a partial update. If either is
        set (i.e. not None) in the update, their values are merged together and
        will replace the existing configuration and secrets values.
        * the `resource_id` field value is also a full replacement value: if set
        to `None`, the resource ID is removed from the service connector.
        * the `expiration_seconds` field value is also a full replacement value:
        if set to `None`, the expiration is removed from the service connector.
        * the `secret_id` field value in the update is ignored, given that
        secrets are managed internally by the ZenML store.
        * the `labels` field is also a full labels update: if set (i.e. not
        `None`), all existing labels are removed and replaced by the new labels
        in the update.

        Args:
            service_connector_id: The ID of the service connector to update.
            update: The update to be applied to the service connector.

        Returns:
            The updated service connector.

        Raises:
            KeyError: If no service connector with the given name exists.
        """

    @abstractmethod
    def delete_service_connector(self, service_connector_id: UUID) -> None:
        """Deletes a service connector.

        Args:
            service_connector_id: The ID of the service connector to delete.

        Raises:
            KeyError: If no service connector with the given ID exists.
        """

    @abstractmethod
    def verify_service_connector_config(
        self,
        service_connector: ServiceConnectorRequest,
        list_resources: bool = True,
    ) -> ServiceConnectorResourcesModel:
        """Verifies if a service connector configuration has access to resources.

        Args:
            service_connector: The service connector configuration to verify.
            list_resources: If True, the list of all resources accessible
                through the service connector is returned.

        Returns:
            The list of resources that the service connector configuration has
            access to.

        Raises:
            NotImplementError: If the service connector cannot be verified
                on the store e.g. due to missing package dependencies.
        """

    @abstractmethod
    def verify_service_connector(
        self,
        service_connector_id: UUID,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
        list_resources: bool = True,
    ) -> ServiceConnectorResourcesModel:
        """Verifies if a service connector instance has access to one or more resources.

        Args:
            service_connector_id: The ID of the service connector to verify.
            resource_type: The type of resource to verify access to.
            resource_id: The ID of the resource to verify access to.
            list_resources: If True, the list of all resources accessible
                through the service connector and matching the supplied resource
                type and ID are returned.

        Returns:
            The list of resources that the service connector has access to,
            scoped to the supplied resource type and ID, if provided.

        Raises:
            KeyError: If no service connector with the given name exists.
            NotImplementError: If the service connector cannot be verified
                e.g. due to missing package dependencies.
        """

    @abstractmethod
    def get_service_connector_client(
        self,
        service_connector_id: UUID,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
    ) -> ServiceConnectorResponse:
        """Get a service connector client for a service connector and given resource.

        Args:
            service_connector_id: The ID of the base service connector to use.
            resource_type: The type of resource to get a client for.
            resource_id: The ID of the resource to get a client for.

        Returns:
            A service connector client that can be used to access the given
            resource.

        Raises:
            KeyError: If no service connector with the given name exists.
            NotImplementError: If the service connector cannot be instantiated
                on the store e.g. due to missing package dependencies.
        """

    @abstractmethod
    def list_service_connector_resources(
        self,
        workspace_name_or_id: Union[str, UUID],
        connector_type: Optional[str] = None,
        resource_type: Optional[str] = None,
        resource_id: Optional[str] = None,
    ) -> List[ServiceConnectorResourcesModel]:
        """List resources that can be accessed by service connectors.

        Args:
            workspace_name_or_id: The name or ID of the workspace to scope to.
            connector_type: The type of service connector to scope to.
            resource_type: The type of resource to scope to.
            resource_id: The ID of the resource to scope to.

        Returns:
            The matching list of resources that available service
            connectors have access to.
        """

    @abstractmethod
    def list_service_connector_types(
        self,
        connector_type: Optional[str] = None,
        resource_type: Optional[str] = None,
        auth_method: Optional[str] = None,
    ) -> List[ServiceConnectorTypeModel]:
        """Get a list of service connector types.

        Args:
            connector_type: Filter by connector type.
            resource_type: Filter by resource type.
            auth_method: Filter by authentication method.

        Returns:
            List of service connector types.
        """

    @abstractmethod
    def get_service_connector_type(
        self,
        connector_type: str,
    ) -> ServiceConnectorTypeModel:
        """Returns the requested service connector type.

        Args:
            connector_type: the service connector type identifier.

        Returns:
            The requested service connector type.

        Raises:
            KeyError: If no service connector type with the given ID exists.
        """

    # -------------------- Stacks --------------------

    @abstractmethod
    def create_stack(self, stack: StackRequest) -> StackResponse:
        """Create a new stack.

        Args:
            stack: The stack to create.

        Returns:
            The created stack.

        Raises:
            StackExistsError: If a stack with the same name is already owned
                by this user in this workspace.
        """

    @abstractmethod
    def get_stack(self, stack_id: UUID, hydrate: bool = True) -> StackResponse:
        """Get a stack by its unique ID.

        Args:
            stack_id: The ID of the stack to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The stack with the given ID.

        Raises:
            KeyError: if the stack doesn't exist.
        """

    @abstractmethod
    def list_stacks(
        self,
        stack_filter_model: StackFilter,
        hydrate: bool = False,
    ) -> Page[StackResponse]:
        """List all stacks matching the given filter criteria.

        Args:
            stack_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all stacks matching the filter criteria.
        """

    @abstractmethod
    def update_stack(
        self, stack_id: UUID, stack_update: StackUpdate
    ) -> StackResponse:
        """Update a stack.

        Args:
            stack_id: The ID of the stack update.
            stack_update: The update request on the stack.

        Returns:
            The updated stack.

        Raises:
            KeyError: if the stack doesn't exist.
        """

    @abstractmethod
    def delete_stack(self, stack_id: UUID) -> None:
        """Delete a stack.

        Args:
            stack_id: The ID of the stack to delete.

        Raises:
            KeyError: if the stack doesn't exist.
        """

    # -------------------- Step runs --------------------

    @abstractmethod
    def create_run_step(self, step_run: StepRunRequest) -> StepRunResponse:
        """Creates a step run.

        Args:
            step_run: The step run to create.

        Returns:
            The created step run.

        Raises:
            EntityExistsError: if the step run already exists.
            KeyError: if the pipeline run doesn't exist.
        """

    @abstractmethod
    def get_run_step(
        self, step_run_id: UUID, hydrate: bool = True
    ) -> StepRunResponse:
        """Get a step run by ID.

        Args:
            step_run_id: The ID of the step run to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The step run.

        Raises:
            KeyError: if the step run doesn't exist.
        """

    @abstractmethod
    def list_run_steps(
        self,
        step_run_filter_model: StepRunFilter,
        hydrate: bool = False,
    ) -> Page[StepRunResponse]:
        """List all step runs matching the given filter criteria.

        Args:
            step_run_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all step runs matching the filter criteria.
        """

    @abstractmethod
    def update_run_step(
        self,
        step_run_id: UUID,
        step_run_update: StepRunUpdate,
    ) -> StepRunResponse:
        """Updates a step run.

        Args:
            step_run_id: The ID of the step to update.
            step_run_update: The update to be applied to the step.

        Returns:
            The updated step run.

        Raises:
            KeyError: if the step run doesn't exist.
        """

    # -------------------- Triggers  --------------------

    @abstractmethod
    def create_trigger(self, trigger: TriggerRequest) -> TriggerResponse:
        """Create an trigger.

        Args:
            trigger: The trigger to create.

        Returns:
            The created trigger.
        """

    @abstractmethod
    def get_trigger(
        self,
        trigger_id: UUID,
        hydrate: bool = True,
    ) -> TriggerResponse:
        """Get an trigger by ID.

        Args:
            trigger_id: The ID of the trigger to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The trigger.

        Raises:
            KeyError: if the stack trigger doesn't exist.
        """

    @abstractmethod
    def list_triggers(
        self,
        trigger_filter_model: TriggerFilter,
        hydrate: bool = False,
    ) -> Page[TriggerResponse]:
        """List all triggers matching the given filter criteria.

        Args:
            trigger_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all triggers matching the filter criteria.
        """

    @abstractmethod
    def update_trigger(
        self,
        trigger_id: UUID,
        trigger_update: TriggerUpdate,
    ) -> TriggerResponse:
        """Update an existing trigger.

        Args:
            trigger_id: The ID of the trigger to update.
            trigger_update: The update to be applied to the trigger.

        Returns:
            The updated trigger.

        Raises:
            KeyError: if the trigger doesn't exist.
        """

    @abstractmethod
    def delete_trigger(self, trigger_id: UUID) -> None:
        """Delete an trigger.

        Args:
            trigger_id: The ID of the trigger to delete.

        Raises:
            KeyError: if the trigger doesn't exist.
        """

    # -------------------- Trigger Executions --------------------

    @abstractmethod
    def get_trigger_execution(
        self,
        trigger_execution_id: UUID,
        hydrate: bool = True,
    ) -> TriggerExecutionResponse:
        """Get an trigger execution by ID.

        Args:
            trigger_execution_id: The ID of the trigger execution to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The trigger execution.

        Raises:
            KeyError: If the trigger execution doesn't exist.
        """

    @abstractmethod
    def list_trigger_executions(
        self,
        trigger_execution_filter_model: TriggerExecutionFilter,
        hydrate: bool = False,
    ) -> Page[TriggerExecutionResponse]:
        """List all trigger executions matching the given filter criteria.

        Args:
            trigger_execution_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all trigger executions matching the filter criteria.
        """

    @abstractmethod
    def delete_trigger_execution(self, trigger_execution_id: UUID) -> None:
        """Delete a trigger execution.

        Args:
            trigger_execution_id: The ID of the trigger execution to delete.

        Raises:
            KeyError: If the trigger execution doesn't exist.
        """

    # -------------------- Users --------------------

    @abstractmethod
    def create_user(self, user: UserRequest) -> UserResponse:
        """Creates a new user.

        Args:
            user: User to be created.

        Returns:
            The newly created user.

        Raises:
            EntityExistsError: If a user with the given name already exists.
        """

    @abstractmethod
    def get_user(
        self,
        user_name_or_id: Optional[Union[str, UUID]] = None,
        include_private: bool = False,
        hydrate: bool = True,
    ) -> UserResponse:
        """Gets a specific user, when no id is specified the active user is returned.

        Args:
            user_name_or_id: The name or ID of the user to get.
            include_private: Whether to include private user information.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested user, if it was found.

        Raises:
            KeyError: If no user with the given name or ID exists.
        """

    @abstractmethod
    def list_users(
        self,
        user_filter_model: UserFilter,
        hydrate: bool = False,
    ) -> Page[UserResponse]:
        """List all users.

        Args:
            user_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all users.
        """

    @abstractmethod
    def update_user(
        self, user_id: UUID, user_update: UserUpdate
    ) -> UserResponse:
        """Updates an existing user.

        Args:
            user_id: The id of the user to update.
            user_update: The update to be applied to the user.

        Returns:
            The updated user.

        Raises:
            KeyError: If no user with the given name exists.
        """

    @abstractmethod
    def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
        """Deletes a user.

        Args:
            user_name_or_id: The name or ID of the user to delete.

        Raises:
            KeyError: If no user with the given ID exists.
        """

    # -------------------- Workspaces --------------------

    @abstractmethod
    def create_workspace(
        self, workspace: WorkspaceRequest
    ) -> WorkspaceResponse:
        """Creates a new workspace.

        Args:
            workspace: The workspace to create.

        Returns:
            The newly created workspace.

        Raises:
            EntityExistsError: If a workspace with the given name already exists.
        """

    @abstractmethod
    def get_workspace(
        self, workspace_name_or_id: Union[UUID, str], hydrate: bool = True
    ) -> WorkspaceResponse:
        """Get an existing workspace by name or ID.

        Args:
            workspace_name_or_id: Name or ID of the workspace to get.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The requested workspace.

        Raises:
            KeyError: If there is no such workspace.
        """

    @abstractmethod
    def list_workspaces(
        self,
        workspace_filter_model: WorkspaceFilter,
        hydrate: bool = False,
    ) -> Page[WorkspaceResponse]:
        """List all workspace matching the given filter criteria.

        Args:
            workspace_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A list of all workspace matching the filter criteria.
        """

    @abstractmethod
    def update_workspace(
        self, workspace_id: UUID, workspace_update: WorkspaceUpdate
    ) -> WorkspaceResponse:
        """Update an existing workspace.

        Args:
            workspace_id: The ID of the workspace to be updated.
            workspace_update: The update to be applied to the workspace.

        Returns:
            The updated workspace.

        Raises:
            KeyError: if the workspace does not exist.
        """

    @abstractmethod
    def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
        """Deletes a workspace.

        Args:
            workspace_name_or_id: Name or ID of the workspace to delete.

        Raises:
            KeyError: If no workspace with the given name exists.
        """

    # -------------------- Models --------------------

    @abstractmethod
    def create_model(self, model: ModelRequest) -> ModelResponse:
        """Creates a new model.

        Args:
            model: the Model to be created.

        Returns:
            The newly created model.

        Raises:
            EntityExistsError: If a model with the given name already exists.
        """

    @abstractmethod
    def delete_model(self, model_name_or_id: Union[str, UUID]) -> None:
        """Deletes a model.

        Args:
            model_name_or_id: name or id of the model to be deleted.

        Raises:
            KeyError: specified ID or name not found.
        """

    @abstractmethod
    def update_model(
        self,
        model_id: UUID,
        model_update: ModelUpdate,
    ) -> ModelResponse:
        """Updates an existing model.

        Args:
            model_id: UUID of the model to be updated.
            model_update: the Model to be updated.

        Returns:
            The updated model.
        """

    @abstractmethod
    def get_model(
        self, model_name_or_id: Union[str, UUID], hydrate: bool = True
    ) -> ModelResponse:
        """Get an existing model.

        Args:
            model_name_or_id: name or id of the model to be retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The model of interest.

        Raises:
            KeyError: specified ID or name not found.
        """

    @abstractmethod
    def list_models(
        self,
        model_filter_model: ModelFilter,
        hydrate: bool = False,
    ) -> Page[ModelResponse]:
        """Get all models by filter.

        Args:
            model_filter_model: All filter parameters including pagination
                params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all models.
        """

    # -------------------- Model versions --------------------

    @abstractmethod
    def create_model_version(
        self, model_version: ModelVersionRequest
    ) -> ModelVersionResponse:
        """Creates a new model version.

        Args:
            model_version: the Model Version to be created.

        Returns:
            The newly created model version.

        Raises:
            ValueError: If `number` is not None during model version creation.
            EntityExistsError: If a model version with the given name already
                exists.
        """

    @abstractmethod
    def delete_model_version(
        self,
        model_version_id: UUID,
    ) -> None:
        """Deletes a model version.

        Args:
            model_version_id: id of the model version to be deleted.

        Raises:
            KeyError: specified ID or name not found.
        """

    @abstractmethod
    def get_model_version(
        self, model_version_id: UUID, hydrate: bool = True
    ) -> ModelVersionResponse:
        """Get an existing model version.

        Args:
            model_version_id: name, id, stage or number of the model version to
                be retrieved. If skipped - latest is retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.


        Returns:
            The model version of interest.

        Raises:
            KeyError: specified ID or name not found.
        """

    @abstractmethod
    def list_model_versions(
        self,
        model_version_filter_model: ModelVersionFilter,
        model_name_or_id: Optional[Union[str, UUID]] = None,
        hydrate: bool = False,
    ) -> Page[ModelVersionResponse]:
        """Get all model versions by filter.

        Args:
            model_name_or_id: name or id of the model containing the model
                versions.
            model_version_filter_model: All filter parameters including
                pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model versions.
        """

    @abstractmethod
    def update_model_version(
        self,
        model_version_id: UUID,
        model_version_update_model: ModelVersionUpdate,
    ) -> ModelVersionResponse:
        """Get all model versions by filter.

        Args:
            model_version_id: The ID of model version to be updated.
            model_version_update_model: The model version to be updated.

        Returns:
            An updated model version.

        Raises:
            KeyError: If the model version not found
            RuntimeError: If there is a model version with target stage,
                but `force` flag is off
        """

    # -------------------- Model Versions Artifacts --------------------

    @abstractmethod
    def create_model_version_artifact_link(
        self, model_version_artifact_link: ModelVersionArtifactRequest
    ) -> ModelVersionArtifactResponse:
        """Creates a new model version link.

        Args:
            model_version_artifact_link: the Model Version to Artifact Link
                to be created.

        Returns:
            The newly created model version to artifact link.

        Raises:
            EntityExistsError: If a link with the given name already exists.
        """

    @abstractmethod
    def list_model_version_artifact_links(
        self,
        model_version_artifact_link_filter_model: ModelVersionArtifactFilter,
        hydrate: bool = False,
    ) -> Page[ModelVersionArtifactResponse]:
        """Get all model version to artifact links by filter.

        Args:
            model_version_artifact_link_filter_model: All filter parameters
                including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model version to artifact links.
        """

    @abstractmethod
    def delete_model_version_artifact_link(
        self,
        model_version_id: UUID,
        model_version_artifact_link_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a model version to artifact link.

        Args:
            model_version_id: ID of the model version containing the link.
            model_version_artifact_link_name_or_id: name or ID of the model
                version to artifact link to be deleted.

        Raises:
            KeyError: specified ID or name not found.
        """

    @abstractmethod
    def delete_all_model_version_artifact_links(
        self,
        model_version_id: UUID,
        only_links: bool = True,
    ) -> None:
        """Deletes all model version to artifact links.

        Args:
            model_version_id: ID of the model version containing the link.
            only_links: Flag deciding whether to delete only links or all.
        """

    # -------------------- Model Versions Pipeline Runs --------------------

    @abstractmethod
    def create_model_version_pipeline_run_link(
        self,
        model_version_pipeline_run_link: ModelVersionPipelineRunRequest,
    ) -> ModelVersionPipelineRunResponse:
        """Creates a new model version to pipeline run link.

        Args:
            model_version_pipeline_run_link: the Model Version to Pipeline Run
                Link to be created.

        Returns:
            - If Model Version to Pipeline Run Link already exists - returns
                the existing link.
            - Otherwise, returns the newly created model version to pipeline
                run link.
        """

    @abstractmethod
    def list_model_version_pipeline_run_links(
        self,
        model_version_pipeline_run_link_filter_model: ModelVersionPipelineRunFilter,
        hydrate: bool = False,
    ) -> Page[ModelVersionPipelineRunResponse]:
        """Get all model version to pipeline run links by filter.

        Args:
            model_version_pipeline_run_link_filter_model: All filter parameters
                including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all model version to pipeline run links.
        """

    @abstractmethod
    def delete_model_version_pipeline_run_link(
        self,
        model_version_id: UUID,
        model_version_pipeline_run_link_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a model version to pipeline run link.

        Args:
            model_version_id: ID of the model version containing the link.
            model_version_pipeline_run_link_name_or_id: name or ID of the model
                version to pipeline run link to be deleted.

        Raises:
            KeyError: specified ID not found.
        """

    #################
    # Tags
    #################

    @abstractmethod
    def create_tag(self, tag: TagRequest) -> TagResponse:
        """Creates a new tag.

        Args:
            tag: the tag to be created.

        Returns:
            The newly created tag.

        Raises:
            EntityExistsError: If a tag with the given name already exists.
        """

    @abstractmethod
    def delete_tag(
        self,
        tag_name_or_id: Union[str, UUID],
    ) -> None:
        """Deletes a tag.

        Args:
            tag_name_or_id: name or id of the tag to delete.

        Raises:
            KeyError: specified ID or name not found.
        """

    @abstractmethod
    def get_tag(
        self, tag_name_or_id: Union[str, UUID], hydrate: bool = True
    ) -> TagResponse:
        """Get an existing tag.

        Args:
            tag_name_or_id: name or id of the tag to be retrieved.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            The tag of interest.

        Raises:
            KeyError: specified ID or name not found.
        """

    @abstractmethod
    def list_tags(
        self,
        tag_filter_model: TagFilter,
        hydrate: bool = False,
    ) -> Page[TagResponse]:
        """Get all tags by filter.

        Args:
            tag_filter_model: All filter parameters including pagination params.
            hydrate: Flag deciding whether to hydrate the output model(s)
                by including metadata fields in the response.

        Returns:
            A page of all tags.
        """

    @abstractmethod
    def update_tag(
        self,
        tag_name_or_id: Union[str, UUID],
        tag_update_model: TagUpdate,
    ) -> TagResponse:
        """Update tag.

        Args:
            tag_name_or_id: name or id of the tag to be updated.
            tag_update_model: Tag to use for the update.

        Returns:
            An updated tag.

        Raises:
            KeyError: If the tag is not found
        """
backup_secrets(self, ignore_errors=True, delete_secrets=False)

Backs up all secrets to the configured backup secrets store.

Parameters:

Name Type Description Default
ignore_errors bool

Whether to ignore individual errors during the backup process and attempt to backup all secrets.

True
delete_secrets bool

Whether to delete the secrets that have been successfully backed up from the primary secrets store. Setting this flag effectively moves all secrets from the primary secrets store to the backup secrets store.

False

Exceptions:

Type Description
BackupSecretsStoreNotConfiguredError

if no backup secrets store is configured.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def backup_secrets(
    self, ignore_errors: bool = True, delete_secrets: bool = False
) -> None:
    """Backs up all secrets to the configured backup secrets store.

    Args:
        ignore_errors: Whether to ignore individual errors during the backup
            process and attempt to backup all secrets.
        delete_secrets: Whether to delete the secrets that have been
            successfully backed up from the primary secrets store. Setting
            this flag effectively moves all secrets from the primary secrets
            store to the backup secrets store.

    Raises:
        BackupSecretsStoreNotConfiguredError: if no backup secrets store is
            configured.
    """
create_api_key(self, service_account_id, api_key)

Create a new API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to create the API key.

required
api_key APIKeyRequest

The API key to create.

required

Returns:

Type Description
APIKeyResponse

The created API key.

Exceptions:

Type Description
KeyError

If the service account doesn't exist.

EntityExistsError

If an API key with the same name is already configured for the same service account.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_api_key(
    self, service_account_id: UUID, api_key: APIKeyRequest
) -> APIKeyResponse:
    """Create a new API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            create the API key.
        api_key: The API key to create.

    Returns:
        The created API key.

    Raises:
        KeyError: If the service account doesn't exist.
        EntityExistsError: If an API key with the same name is already
            configured for the same service account.
    """
create_artifact(self, artifact)

Creates a new artifact.

Parameters:

Name Type Description Default
artifact ArtifactRequest

The artifact to create.

required

Returns:

Type Description
ArtifactResponse

The newly created artifact.

Exceptions:

Type Description
EntityExistsError

If an artifact with the same name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_artifact(self, artifact: ArtifactRequest) -> ArtifactResponse:
    """Creates a new artifact.

    Args:
        artifact: The artifact to create.

    Returns:
        The newly created artifact.

    Raises:
        EntityExistsError: If an artifact with the same name already exists.
    """
create_artifact_version(self, artifact_version)

Creates an artifact version.

Parameters:

Name Type Description Default
artifact_version ArtifactVersionRequest

The artifact version to create.

required

Returns:

Type Description
ArtifactVersionResponse

The created artifact version.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_artifact_version(
    self, artifact_version: ArtifactVersionRequest
) -> ArtifactVersionResponse:
    """Creates an artifact version.

    Args:
        artifact_version: The artifact version to create.

    Returns:
        The created artifact version.
    """
create_build(self, build)

Creates a new build in a workspace.

Parameters:

Name Type Description Default
build PipelineBuildRequest

The build to create.

required

Returns:

Type Description
PipelineBuildResponse

The newly created build.

Exceptions:

Type Description
KeyError

If the workspace does not exist.

EntityExistsError

If an identical build already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_build(
    self,
    build: PipelineBuildRequest,
) -> PipelineBuildResponse:
    """Creates a new build in a workspace.

    Args:
        build: The build to create.

    Returns:
        The newly created build.

    Raises:
        KeyError: If the workspace does not exist.
        EntityExistsError: If an identical build already exists.
    """
create_code_repository(self, code_repository)

Creates a new code repository.

Parameters:

Name Type Description Default
code_repository CodeRepositoryRequest

Code repository to be created.

required

Returns:

Type Description
CodeRepositoryResponse

The newly created code repository.

Exceptions:

Type Description
EntityExistsError

If a code repository with the given name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_code_repository(
    self, code_repository: CodeRepositoryRequest
) -> CodeRepositoryResponse:
    """Creates a new code repository.

    Args:
        code_repository: Code repository to be created.

    Returns:
        The newly created code repository.

    Raises:
        EntityExistsError: If a code repository with the given name already
            exists.
    """
create_deployment(self, deployment)

Creates a new deployment in a workspace.

Parameters:

Name Type Description Default
deployment PipelineDeploymentRequest

The deployment to create.

required

Returns:

Type Description
PipelineDeploymentResponse

The newly created deployment.

Exceptions:

Type Description
KeyError

If the workspace does not exist.

EntityExistsError

If an identical deployment already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_deployment(
    self,
    deployment: PipelineDeploymentRequest,
) -> PipelineDeploymentResponse:
    """Creates a new deployment in a workspace.

    Args:
        deployment: The deployment to create.

    Returns:
        The newly created deployment.

    Raises:
        KeyError: If the workspace does not exist.
        EntityExistsError: If an identical deployment already exists.
    """
create_event_source(self, event_source)

Create an event_source.

Parameters:

Name Type Description Default
event_source EventSourceRequest

The event_source to create.

required

Returns:

Type Description
EventSourceResponse

The created event_source.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_event_source(
    self, event_source: EventSourceRequest
) -> EventSourceResponse:
    """Create an event_source.

    Args:
        event_source: The event_source to create.

    Returns:
        The created event_source.
    """
create_flavor(self, flavor)

Creates a new stack component flavor.

Parameters:

Name Type Description Default
flavor FlavorRequest

The stack component flavor to create.

required

Returns:

Type Description
FlavorResponse

The newly created flavor.

Exceptions:

Type Description
EntityExistsError

If a flavor with the same name and type is already owned by this user in this workspace.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_flavor(
    self,
    flavor: FlavorRequest,
) -> FlavorResponse:
    """Creates a new stack component flavor.

    Args:
        flavor: The stack component flavor to create.

    Returns:
        The newly created flavor.

    Raises:
        EntityExistsError: If a flavor with the same name and type
            is already owned by this user in this workspace.
    """
create_model(self, model)

Creates a new model.

Parameters:

Name Type Description Default
model ModelRequest

the Model to be created.

required

Returns:

Type Description
ModelResponse

The newly created model.

Exceptions:

Type Description
EntityExistsError

If a model with the given name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_model(self, model: ModelRequest) -> ModelResponse:
    """Creates a new model.

    Args:
        model: the Model to be created.

    Returns:
        The newly created model.

    Raises:
        EntityExistsError: If a model with the given name already exists.
    """
create_model_version(self, model_version)

Creates a new model version.

Parameters:

Name Type Description Default
model_version ModelVersionRequest

the Model Version to be created.

required

Returns:

Type Description
ModelVersionResponse

The newly created model version.

Exceptions:

Type Description
ValueError

If number is not None during model version creation.

EntityExistsError

If a model version with the given name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_model_version(
    self, model_version: ModelVersionRequest
) -> ModelVersionResponse:
    """Creates a new model version.

    Args:
        model_version: the Model Version to be created.

    Returns:
        The newly created model version.

    Raises:
        ValueError: If `number` is not None during model version creation.
        EntityExistsError: If a model version with the given name already
            exists.
    """

Creates a new model version link.

Parameters:

Name Type Description Default
model_version_artifact_link ModelVersionArtifactRequest

the Model Version to Artifact Link to be created.

required

Returns:

Type Description
ModelVersionArtifactResponse

The newly created model version to artifact link.

Exceptions:

Type Description
EntityExistsError

If a link with the given name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_model_version_artifact_link(
    self, model_version_artifact_link: ModelVersionArtifactRequest
) -> ModelVersionArtifactResponse:
    """Creates a new model version link.

    Args:
        model_version_artifact_link: the Model Version to Artifact Link
            to be created.

    Returns:
        The newly created model version to artifact link.

    Raises:
        EntityExistsError: If a link with the given name already exists.
    """

Creates a new model version to pipeline run link.

Parameters:

Name Type Description Default
model_version_pipeline_run_link ModelVersionPipelineRunRequest

the Model Version to Pipeline Run Link to be created.

required

Returns:

Type Description
ModelVersionPipelineRunResponse
  • If Model Version to Pipeline Run Link already exists - returns the existing link.
  • Otherwise, returns the newly created model version to pipeline run link.
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_model_version_pipeline_run_link(
    self,
    model_version_pipeline_run_link: ModelVersionPipelineRunRequest,
) -> ModelVersionPipelineRunResponse:
    """Creates a new model version to pipeline run link.

    Args:
        model_version_pipeline_run_link: the Model Version to Pipeline Run
            Link to be created.

    Returns:
        - If Model Version to Pipeline Run Link already exists - returns
            the existing link.
        - Otherwise, returns the newly created model version to pipeline
            run link.
    """
create_pipeline(self, pipeline)

Creates a new pipeline in a workspace.

Parameters:

Name Type Description Default
pipeline PipelineRequest

The pipeline to create.

required

Returns:

Type Description
PipelineResponse

The newly created pipeline.

Exceptions:

Type Description
KeyError

if the workspace does not exist.

EntityExistsError

If an identical pipeline already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_pipeline(
    self,
    pipeline: PipelineRequest,
) -> PipelineResponse:
    """Creates a new pipeline in a workspace.

    Args:
        pipeline: The pipeline to create.

    Returns:
        The newly created pipeline.

    Raises:
        KeyError: if the workspace does not exist.
        EntityExistsError: If an identical pipeline already exists.
    """
create_run(self, pipeline_run)

Creates a pipeline run.

Parameters:

Name Type Description Default
pipeline_run PipelineRunRequest

The pipeline run to create.

required

Returns:

Type Description
PipelineRunResponse

The created pipeline run.

Exceptions:

Type Description
EntityExistsError

If an identical pipeline run already exists.

KeyError

If the pipeline does not exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_run(
    self, pipeline_run: PipelineRunRequest
) -> PipelineRunResponse:
    """Creates a pipeline run.

    Args:
        pipeline_run: The pipeline run to create.

    Returns:
        The created pipeline run.

    Raises:
        EntityExistsError: If an identical pipeline run already exists.
        KeyError: If the pipeline does not exist.
    """
create_run_metadata(self, run_metadata)

Creates run metadata.

Parameters:

Name Type Description Default
run_metadata RunMetadataRequest

The run metadata to create.

required

Returns:

Type Description
List[zenml.models.v2.core.run_metadata.RunMetadataResponse]

The created run metadata.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_run_metadata(
    self, run_metadata: RunMetadataRequest
) -> List[RunMetadataResponse]:
    """Creates run metadata.

    Args:
        run_metadata: The run metadata to create.

    Returns:
        The created run metadata.
    """
create_run_step(self, step_run)

Creates a step run.

Parameters:

Name Type Description Default
step_run StepRunRequest

The step run to create.

required

Returns:

Type Description
StepRunResponse

The created step run.

Exceptions:

Type Description
EntityExistsError

if the step run already exists.

KeyError

if the pipeline run doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_run_step(self, step_run: StepRunRequest) -> StepRunResponse:
    """Creates a step run.

    Args:
        step_run: The step run to create.

    Returns:
        The created step run.

    Raises:
        EntityExistsError: if the step run already exists.
        KeyError: if the pipeline run doesn't exist.
    """
create_schedule(self, schedule)

Creates a new schedule.

Parameters:

Name Type Description Default
schedule ScheduleRequest

The schedule to create.

required

Returns:

Type Description
ScheduleResponse

The newly created schedule.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_schedule(self, schedule: ScheduleRequest) -> ScheduleResponse:
    """Creates a new schedule.

    Args:
        schedule: The schedule to create.

    Returns:
        The newly created schedule.
    """
create_secret(self, secret)

Creates a new secret.

The new secret is also validated against the scoping rules enforced in the secrets store:

  • only one workspace-scoped secret with the given name can exist in the target workspace.
  • only one user-scoped secret with the given name can exist in the target workspace for the target user.

Parameters:

Name Type Description Default
secret SecretRequest

The secret to create.

required

Returns:

Type Description
SecretResponse

The newly created secret.

Exceptions:

Type Description
KeyError

if the user or workspace does not exist.

EntityExistsError

If a secret with the same name already exists in the same scope.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_secret(
    self,
    secret: SecretRequest,
) -> SecretResponse:
    """Creates a new secret.

    The new secret is also validated against the scoping rules enforced in
    the secrets store:

      - only one workspace-scoped secret with the given name can exist
        in the target workspace.
      - only one user-scoped secret with the given name can exist in the
        target workspace for the target user.

    Args:
        secret: The secret to create.

    Returns:
        The newly created secret.

    Raises:
        KeyError: if the user or workspace does not exist.
        EntityExistsError: If a secret with the same name already exists in
            the same scope.
    """
create_service_account(self, service_account)

Creates a new service account.

Parameters:

Name Type Description Default
service_account ServiceAccountRequest

Service account to be created.

required

Returns:

Type Description
ServiceAccountResponse

The newly created service account.

Exceptions:

Type Description
EntityExistsError

If a user or service account with the given name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_service_account(
    self, service_account: ServiceAccountRequest
) -> ServiceAccountResponse:
    """Creates a new service account.

    Args:
        service_account: Service account to be created.

    Returns:
        The newly created service account.

    Raises:
        EntityExistsError: If a user or service account with the given name
            already exists.
    """
create_service_connector(self, service_connector)

Creates a new service connector.

Parameters:

Name Type Description Default
service_connector ServiceConnectorRequest

Service connector to be created.

required

Returns:

Type Description
ServiceConnectorResponse

The newly created service connector.

Exceptions:

Type Description
EntityExistsError

If a service connector with the given name is already owned by this user in this workspace.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_service_connector(
    self,
    service_connector: ServiceConnectorRequest,
) -> ServiceConnectorResponse:
    """Creates a new service connector.

    Args:
        service_connector: Service connector to be created.

    Returns:
        The newly created service connector.

    Raises:
        EntityExistsError: If a service connector with the given name
            is already owned by this user in this workspace.
    """
create_stack(self, stack)

Create a new stack.

Parameters:

Name Type Description Default
stack StackRequest

The stack to create.

required

Returns:

Type Description
StackResponse

The created stack.

Exceptions:

Type Description
StackExistsError

If a stack with the same name is already owned by this user in this workspace.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_stack(self, stack: StackRequest) -> StackResponse:
    """Create a new stack.

    Args:
        stack: The stack to create.

    Returns:
        The created stack.

    Raises:
        StackExistsError: If a stack with the same name is already owned
            by this user in this workspace.
    """
create_stack_component(self, component)

Create a stack component.

Parameters:

Name Type Description Default
component ComponentRequest

The stack component to create.

required

Returns:

Type Description
ComponentResponse

The created stack component.

Exceptions:

Type Description
StackComponentExistsError

If a stack component with the same name and type is already owned by this user in this workspace.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_stack_component(
    self, component: ComponentRequest
) -> ComponentResponse:
    """Create a stack component.

    Args:
        component: The stack component to create.

    Returns:
        The created stack component.

    Raises:
        StackComponentExistsError: If a stack component with the same name
            and type is already owned by this user in this workspace.
    """
create_tag(self, tag)

Creates a new tag.

Parameters:

Name Type Description Default
tag TagRequest

the tag to be created.

required

Returns:

Type Description
TagResponse

The newly created tag.

Exceptions:

Type Description
EntityExistsError

If a tag with the given name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_tag(self, tag: TagRequest) -> TagResponse:
    """Creates a new tag.

    Args:
        tag: the tag to be created.

    Returns:
        The newly created tag.

    Raises:
        EntityExistsError: If a tag with the given name already exists.
    """
create_trigger(self, trigger)

Create an trigger.

Parameters:

Name Type Description Default
trigger TriggerRequest

The trigger to create.

required

Returns:

Type Description
TriggerResponse

The created trigger.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_trigger(self, trigger: TriggerRequest) -> TriggerResponse:
    """Create an trigger.

    Args:
        trigger: The trigger to create.

    Returns:
        The created trigger.
    """
create_user(self, user)

Creates a new user.

Parameters:

Name Type Description Default
user UserRequest

User to be created.

required

Returns:

Type Description
UserResponse

The newly created user.

Exceptions:

Type Description
EntityExistsError

If a user with the given name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_user(self, user: UserRequest) -> UserResponse:
    """Creates a new user.

    Args:
        user: User to be created.

    Returns:
        The newly created user.

    Raises:
        EntityExistsError: If a user with the given name already exists.
    """
create_workspace(self, workspace)

Creates a new workspace.

Parameters:

Name Type Description Default
workspace WorkspaceRequest

The workspace to create.

required

Returns:

Type Description
WorkspaceResponse

The newly created workspace.

Exceptions:

Type Description
EntityExistsError

If a workspace with the given name already exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_workspace(
    self, workspace: WorkspaceRequest
) -> WorkspaceResponse:
    """Creates a new workspace.

    Args:
        workspace: The workspace to create.

    Returns:
        The newly created workspace.

    Raises:
        EntityExistsError: If a workspace with the given name already exists.
    """

Deletes all model version to artifact links.

Parameters:

Name Type Description Default
model_version_id UUID

ID of the model version containing the link.

required
only_links bool

Flag deciding whether to delete only links or all.

True
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_all_model_version_artifact_links(
    self,
    model_version_id: UUID,
    only_links: bool = True,
) -> None:
    """Deletes all model version to artifact links.

    Args:
        model_version_id: ID of the model version containing the link.
        only_links: Flag deciding whether to delete only links or all.
    """
delete_api_key(self, service_account_id, api_key_name_or_id)

Delete an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to delete the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to delete.

required

Exceptions:

Type Description
KeyError

if an API key with the given name or ID is not configured for the given service account.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
) -> None:
    """Delete an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            delete the API key.
        api_key_name_or_id: The name or ID of the API key to delete.

    Raises:
        KeyError: if an API key with the given name or ID is not configured
            for the given service account.
    """
delete_artifact(self, artifact_id)

Deletes an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to delete.

required

Exceptions:

Type Description
KeyError

if the artifact doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_artifact(self, artifact_id: UUID) -> None:
    """Deletes an artifact.

    Args:
        artifact_id: The ID of the artifact to delete.

    Raises:
        KeyError: if the artifact doesn't exist.
    """
delete_artifact_version(self, artifact_version_id)

Deletes an artifact version.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to delete.

required

Exceptions:

Type Description
KeyError

if the artifact version doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_artifact_version(self, artifact_version_id: UUID) -> None:
    """Deletes an artifact version.

    Args:
        artifact_version_id: The ID of the artifact version to delete.

    Raises:
        KeyError: if the artifact version doesn't exist.
    """
delete_authorized_device(self, device_id)

Deletes an OAuth 2.0 authorized device.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to delete.

required

Exceptions:

Type Description
KeyError

If no device with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_authorized_device(self, device_id: UUID) -> None:
    """Deletes an OAuth 2.0 authorized device.

    Args:
        device_id: The ID of the device to delete.

    Raises:
        KeyError: If no device with the given ID exists.
    """
delete_build(self, build_id)

Deletes a build.

Parameters:

Name Type Description Default
build_id UUID

The ID of the build to delete.

required

Exceptions:

Type Description
KeyError

if the build doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_build(self, build_id: UUID) -> None:
    """Deletes a build.

    Args:
        build_id: The ID of the build to delete.

    Raises:
        KeyError: if the build doesn't exist.
    """
delete_code_repository(self, code_repository_id)

Deletes a code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to delete.

required

Exceptions:

Type Description
KeyError

If no code repository with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_code_repository(self, code_repository_id: UUID) -> None:
    """Deletes a code repository.

    Args:
        code_repository_id: The ID of the code repository to delete.

    Raises:
        KeyError: If no code repository with the given ID exists.
    """
delete_deployment(self, deployment_id)

Deletes a deployment.

Parameters:

Name Type Description Default
deployment_id UUID

The ID of the deployment to delete.

required

Exceptions:

Type Description
KeyError

If the deployment doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_deployment(self, deployment_id: UUID) -> None:
    """Deletes a deployment.

    Args:
        deployment_id: The ID of the deployment to delete.

    Raises:
        KeyError: If the deployment doesn't exist.
    """
delete_event_source(self, event_source_id)

Delete an event_source.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to delete.

required

Exceptions:

Type Description
KeyError

if the event_source doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_event_source(self, event_source_id: UUID) -> None:
    """Delete an event_source.

    Args:
        event_source_id: The ID of the event_source to delete.

    Raises:
        KeyError: if the event_source doesn't exist.
    """
delete_flavor(self, flavor_id)

Delete a stack component flavor.

Parameters:

Name Type Description Default
flavor_id UUID

The ID of the stack component flavor to delete.

required

Exceptions:

Type Description
KeyError

if the stack component flavor doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_flavor(self, flavor_id: UUID) -> None:
    """Delete a stack component flavor.

    Args:
        flavor_id: The ID of the stack component flavor to delete.

    Raises:
        KeyError: if the stack component flavor doesn't exist.
    """
delete_model(self, model_name_or_id)

Deletes a model.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model to be deleted.

required

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_model(self, model_name_or_id: Union[str, UUID]) -> None:
    """Deletes a model.

    Args:
        model_name_or_id: name or id of the model to be deleted.

    Raises:
        KeyError: specified ID or name not found.
    """
delete_model_version(self, model_version_id)

Deletes a model version.

Parameters:

Name Type Description Default
model_version_id UUID

id of the model version to be deleted.

required

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_model_version(
    self,
    model_version_id: UUID,
) -> None:
    """Deletes a model version.

    Args:
        model_version_id: id of the model version to be deleted.

    Raises:
        KeyError: specified ID or name not found.
    """

Deletes a model version to artifact link.

Parameters:

Name Type Description Default
model_version_id UUID

ID of the model version containing the link.

required
model_version_artifact_link_name_or_id Union[str, uuid.UUID]

name or ID of the model version to artifact link to be deleted.

required

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_model_version_artifact_link(
    self,
    model_version_id: UUID,
    model_version_artifact_link_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a model version to artifact link.

    Args:
        model_version_id: ID of the model version containing the link.
        model_version_artifact_link_name_or_id: name or ID of the model
            version to artifact link to be deleted.

    Raises:
        KeyError: specified ID or name not found.
    """

Deletes a model version to pipeline run link.

Parameters:

Name Type Description Default
model_version_id UUID

ID of the model version containing the link.

required
model_version_pipeline_run_link_name_or_id Union[str, uuid.UUID]

name or ID of the model version to pipeline run link to be deleted.

required

Exceptions:

Type Description
KeyError

specified ID not found.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_model_version_pipeline_run_link(
    self,
    model_version_id: UUID,
    model_version_pipeline_run_link_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a model version to pipeline run link.

    Args:
        model_version_id: ID of the model version containing the link.
        model_version_pipeline_run_link_name_or_id: name or ID of the model
            version to pipeline run link to be deleted.

    Raises:
        KeyError: specified ID not found.
    """
delete_pipeline(self, pipeline_id)

Deletes a pipeline.

Parameters:

Name Type Description Default
pipeline_id UUID

The ID of the pipeline to delete.

required

Exceptions:

Type Description
KeyError

if the pipeline doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_pipeline(self, pipeline_id: UUID) -> None:
    """Deletes a pipeline.

    Args:
        pipeline_id: The ID of the pipeline to delete.

    Raises:
        KeyError: if the pipeline doesn't exist.
    """
delete_run(self, run_id)

Deletes a pipeline run.

Parameters:

Name Type Description Default
run_id UUID

The ID of the pipeline run to delete.

required

Exceptions:

Type Description
KeyError

if the pipeline run doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_run(self, run_id: UUID) -> None:
    """Deletes a pipeline run.

    Args:
        run_id: The ID of the pipeline run to delete.

    Raises:
        KeyError: if the pipeline run doesn't exist.
    """
delete_schedule(self, schedule_id)

Deletes a schedule.

Parameters:

Name Type Description Default
schedule_id UUID

The ID of the schedule to delete.

required

Exceptions:

Type Description
KeyError

if the schedule doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_schedule(self, schedule_id: UUID) -> None:
    """Deletes a schedule.

    Args:
        schedule_id: The ID of the schedule to delete.

    Raises:
        KeyError: if the schedule doesn't exist.
    """
delete_secret(self, secret_id)

Deletes a secret.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to delete.

required

Exceptions:

Type Description
KeyError

if the secret doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_secret(self, secret_id: UUID) -> None:
    """Deletes a secret.

    Args:
        secret_id: The ID of the secret to delete.

    Raises:
        KeyError: if the secret doesn't exist.
    """
delete_service_account(self, service_account_name_or_id)

Delete a service account.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or the ID of the service account to delete.

required

Exceptions:

Type Description
IllegalOperationError

if the service account has already been used to create other resources.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
) -> None:
    """Delete a service account.

    Args:
        service_account_name_or_id: The name or the ID of the service
            account to delete.

    Raises:
        IllegalOperationError: if the service account has already been used
            to create other resources.
    """
delete_service_connector(self, service_connector_id)

Deletes a service connector.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to delete.

required

Exceptions:

Type Description
KeyError

If no service connector with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_service_connector(self, service_connector_id: UUID) -> None:
    """Deletes a service connector.

    Args:
        service_connector_id: The ID of the service connector to delete.

    Raises:
        KeyError: If no service connector with the given ID exists.
    """
delete_stack(self, stack_id)

Delete a stack.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack to delete.

required

Exceptions:

Type Description
KeyError

if the stack doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_stack(self, stack_id: UUID) -> None:
    """Delete a stack.

    Args:
        stack_id: The ID of the stack to delete.

    Raises:
        KeyError: if the stack doesn't exist.
    """
delete_stack_component(self, component_id)

Delete a stack component.

Parameters:

Name Type Description Default
component_id UUID

The ID of the stack component to delete.

required

Exceptions:

Type Description
KeyError

if the stack component doesn't exist.

ValueError

if the stack component is part of one or more stacks.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_stack_component(self, component_id: UUID) -> None:
    """Delete a stack component.

    Args:
        component_id: The ID of the stack component to delete.

    Raises:
        KeyError: if the stack component doesn't exist.
        ValueError: if the stack component is part of one or more stacks.
    """
delete_tag(self, tag_name_or_id)

Deletes a tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to delete.

required

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_tag(
    self,
    tag_name_or_id: Union[str, UUID],
) -> None:
    """Deletes a tag.

    Args:
        tag_name_or_id: name or id of the tag to delete.

    Raises:
        KeyError: specified ID or name not found.
    """
delete_trigger(self, trigger_id)

Delete an trigger.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger to delete.

required

Exceptions:

Type Description
KeyError

if the trigger doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_trigger(self, trigger_id: UUID) -> None:
    """Delete an trigger.

    Args:
        trigger_id: The ID of the trigger to delete.

    Raises:
        KeyError: if the trigger doesn't exist.
    """
delete_trigger_execution(self, trigger_execution_id)

Delete a trigger execution.

Parameters:

Name Type Description Default
trigger_execution_id UUID

The ID of the trigger execution to delete.

required

Exceptions:

Type Description
KeyError

If the trigger execution doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_trigger_execution(self, trigger_execution_id: UUID) -> None:
    """Delete a trigger execution.

    Args:
        trigger_execution_id: The ID of the trigger execution to delete.

    Raises:
        KeyError: If the trigger execution doesn't exist.
    """
delete_user(self, user_name_or_id)

Deletes a user.

Parameters:

Name Type Description Default
user_name_or_id Union[str, uuid.UUID]

The name or ID of the user to delete.

required

Exceptions:

Type Description
KeyError

If no user with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
    """Deletes a user.

    Args:
        user_name_or_id: The name or ID of the user to delete.

    Raises:
        KeyError: If no user with the given ID exists.
    """
delete_workspace(self, workspace_name_or_id)

Deletes a workspace.

Parameters:

Name Type Description Default
workspace_name_or_id Union[str, uuid.UUID]

Name or ID of the workspace to delete.

required

Exceptions:

Type Description
KeyError

If no workspace with the given name exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
    """Deletes a workspace.

    Args:
        workspace_name_or_id: Name or ID of the workspace to delete.

    Raises:
        KeyError: If no workspace with the given name exists.
    """
get_api_key(self, service_account_id, api_key_name_or_id, hydrate=True)

Get an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to fetch the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
APIKeyResponse

The API key with the given ID.

Exceptions:

Type Description
KeyError

if an API key with the given name or ID is not configured for the given service account.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    hydrate: bool = True,
) -> APIKeyResponse:
    """Get an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to fetch
            the API key.
        api_key_name_or_id: The name or ID of the API key to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The API key with the given ID.

    Raises:
        KeyError: if an API key with the given name or ID is not configured
            for the given service account.
    """
get_artifact(self, artifact_id, hydrate=True)

Gets an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactResponse

The artifact.

Exceptions:

Type Description
KeyError

if the artifact doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_artifact(
    self, artifact_id: UUID, hydrate: bool = True
) -> ArtifactResponse:
    """Gets an artifact.

    Args:
        artifact_id: The ID of the artifact to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact.

    Raises:
        KeyError: if the artifact doesn't exist.
    """
get_artifact_version(self, artifact_version_id, hydrate=True)

Gets an artifact version.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactVersionResponse

The artifact version.

Exceptions:

Type Description
KeyError

if the artifact version doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_artifact_version(
    self, artifact_version_id: UUID, hydrate: bool = True
) -> ArtifactVersionResponse:
    """Gets an artifact version.

    Args:
        artifact_version_id: The ID of the artifact version to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact version.

    Raises:
        KeyError: if the artifact version doesn't exist.
    """
get_artifact_visualization(self, artifact_visualization_id, hydrate=True)

Gets an artifact visualization.

Parameters:

Name Type Description Default
artifact_visualization_id UUID

The ID of the artifact visualization to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ArtifactVisualizationResponse

The artifact visualization.

Exceptions:

Type Description
KeyError

if the artifact visualization doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_artifact_visualization(
    self, artifact_visualization_id: UUID, hydrate: bool = True
) -> ArtifactVisualizationResponse:
    """Gets an artifact visualization.

    Args:
        artifact_visualization_id: The ID of the artifact visualization
            to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The artifact visualization.

    Raises:
        KeyError: if the artifact visualization doesn't exist.
    """
get_authorized_device(self, device_id, hydrate=True)

Gets a specific OAuth 2.0 authorized device.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
OAuthDeviceResponse

The requested device, if it was found.

Exceptions:

Type Description
KeyError

If no device with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_authorized_device(
    self, device_id: UUID, hydrate: bool = True
) -> OAuthDeviceResponse:
    """Gets a specific OAuth 2.0 authorized device.

    Args:
        device_id: The ID of the device to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested device, if it was found.

    Raises:
        KeyError: If no device with the given ID exists.
    """
get_build(self, build_id, hydrate=True)

Get a build with a given ID.

Parameters:

Name Type Description Default
build_id UUID

ID of the build.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineBuildResponse

The build.

Exceptions:

Type Description
KeyError

If the build does not exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_build(
    self, build_id: UUID, hydrate: bool = True
) -> PipelineBuildResponse:
    """Get a build with a given ID.

    Args:
        build_id: ID of the build.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The build.

    Raises:
        KeyError: If the build does not exist.
    """
get_code_reference(self, code_reference_id, hydrate=True)

Gets a specific code reference.

Parameters:

Name Type Description Default
code_reference_id UUID

The ID of the code reference to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
CodeReferenceResponse

The requested code reference, if it was found.

Exceptions:

Type Description
KeyError

If no code reference with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_code_reference(
    self, code_reference_id: UUID, hydrate: bool = True
) -> CodeReferenceResponse:
    """Gets a specific code reference.

    Args:
        code_reference_id: The ID of the code reference to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested code reference, if it was found.

    Raises:
        KeyError: If no code reference with the given ID exists.
    """
get_code_repository(self, code_repository_id, hydrate=True)

Gets a specific code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
CodeRepositoryResponse

The requested code repository, if it was found.

Exceptions:

Type Description
KeyError

If no code repository with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_code_repository(
    self, code_repository_id: UUID, hydrate: bool = True
) -> CodeRepositoryResponse:
    """Gets a specific code repository.

    Args:
        code_repository_id: The ID of the code repository to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested code repository, if it was found.

    Raises:
        KeyError: If no code repository with the given ID exists.
    """
get_deployment(self, deployment_id, hydrate=True)

Get a deployment with a given ID.

Parameters:

Name Type Description Default
deployment_id UUID

ID of the deployment.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineDeploymentResponse

The deployment.

Exceptions:

Type Description
KeyError

If the deployment does not exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_deployment(
    self, deployment_id: UUID, hydrate: bool = True
) -> PipelineDeploymentResponse:
    """Get a deployment with a given ID.

    Args:
        deployment_id: ID of the deployment.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The deployment.

    Raises:
        KeyError: If the deployment does not exist.
    """
get_deployment_id(self)

Get the ID of the deployment.

Returns:

Type Description
UUID

The ID of the deployment.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_deployment_id(self) -> UUID:
    """Get the ID of the deployment.

    Returns:
        The ID of the deployment.
    """
get_event_source(self, event_source_id, hydrate=True)

Get an event_source by ID.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
EventSourceResponse

The event_source.

Exceptions:

Type Description
KeyError

if the stack event_source doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_event_source(
    self,
    event_source_id: UUID,
    hydrate: bool = True,
) -> EventSourceResponse:
    """Get an event_source by ID.

    Args:
        event_source_id: The ID of the event_source to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The event_source.

    Raises:
        KeyError: if the stack event_source doesn't exist.
    """
get_flavor(self, flavor_id, hydrate=True)

Get a stack component flavor by ID.

Parameters:

Name Type Description Default
flavor_id UUID

The ID of the flavor to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
FlavorResponse

The stack component flavor.

Exceptions:

Type Description
KeyError

if the stack component flavor doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_flavor(
    self, flavor_id: UUID, hydrate: bool = True
) -> FlavorResponse:
    """Get a stack component flavor by ID.

    Args:
        flavor_id: The ID of the flavor to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack component flavor.

    Raises:
        KeyError: if the stack component flavor doesn't exist.
    """
get_logs(self, logs_id, hydrate=True)

Get logs by its unique ID.

Parameters:

Name Type Description Default
logs_id UUID

The ID of the logs to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
LogsResponse

The logs with the given ID.

Exceptions:

Type Description
KeyError

if the logs doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_logs(self, logs_id: UUID, hydrate: bool = True) -> LogsResponse:
    """Get logs by its unique ID.

    Args:
        logs_id: The ID of the logs to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The logs with the given ID.

    Raises:
        KeyError: if the logs doesn't exist.
    """
get_model(self, model_name_or_id, hydrate=True)

Get an existing model.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model to be retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ModelResponse

The model of interest.

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_model(
    self, model_name_or_id: Union[str, UUID], hydrate: bool = True
) -> ModelResponse:
    """Get an existing model.

    Args:
        model_name_or_id: name or id of the model to be retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The model of interest.

    Raises:
        KeyError: specified ID or name not found.
    """
get_model_version(self, model_version_id, hydrate=True)

Get an existing model version.

Parameters:

Name Type Description Default
model_version_id UUID

name, id, stage or number of the model version to be retrieved. If skipped - latest is retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ModelVersionResponse

The model version of interest.

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_model_version(
    self, model_version_id: UUID, hydrate: bool = True
) -> ModelVersionResponse:
    """Get an existing model version.

    Args:
        model_version_id: name, id, stage or number of the model version to
            be retrieved. If skipped - latest is retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.


    Returns:
        The model version of interest.

    Raises:
        KeyError: specified ID or name not found.
    """
get_or_create_run(self, pipeline_run)

Gets or creates a pipeline run.

If a run with the same ID or name already exists, it is returned. Otherwise, a new run is created.

Parameters:

Name Type Description Default
pipeline_run PipelineRunRequest

The pipeline run to get or create.

required

Returns:

Type Description
Tuple[zenml.models.v2.core.pipeline_run.PipelineRunResponse, bool]

The pipeline run, and a boolean indicating whether the run was created or not.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_or_create_run(
    self, pipeline_run: PipelineRunRequest
) -> Tuple[PipelineRunResponse, bool]:
    """Gets or creates a pipeline run.

    If a run with the same ID or name already exists, it is returned.
    Otherwise, a new run is created.

    Args:
        pipeline_run: The pipeline run to get or create.

    Returns:
        The pipeline run, and a boolean indicating whether the run was
        created or not.
    """
get_pipeline(self, pipeline_id, hydrate=True)

Get a pipeline with a given ID.

Parameters:

Name Type Description Default
pipeline_id UUID

ID of the pipeline.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineResponse

The pipeline.

Exceptions:

Type Description
KeyError

if the pipeline does not exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_pipeline(
    self, pipeline_id: UUID, hydrate: bool = True
) -> PipelineResponse:
    """Get a pipeline with a given ID.

    Args:
        pipeline_id: ID of the pipeline.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The pipeline.

    Raises:
        KeyError: if the pipeline does not exist.
    """
get_run(self, run_name_or_id, hydrate=True)

Gets a pipeline run.

Parameters:

Name Type Description Default
run_name_or_id Union[str, uuid.UUID]

The name or ID of the pipeline run to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
PipelineRunResponse

The pipeline run.

Exceptions:

Type Description
KeyError

if the pipeline run doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run(
    self, run_name_or_id: Union[str, UUID], hydrate: bool = True
) -> PipelineRunResponse:
    """Gets a pipeline run.

    Args:
        run_name_or_id: The name or ID of the pipeline run to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The pipeline run.

    Raises:
        KeyError: if the pipeline run doesn't exist.
    """
get_run_metadata(self, run_metadata_id, hydrate=True)

Get run metadata by its unique ID.

Parameters:

Name Type Description Default
run_metadata_id UUID

The ID of the run metadata to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
RunMetadataResponse

The run metadata with the given ID.

Exceptions:

Type Description
KeyError

if the run metadata doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_metadata(
    self, run_metadata_id: UUID, hydrate: bool = True
) -> RunMetadataResponse:
    """Get run metadata by its unique ID.

    Args:
        run_metadata_id: The ID of the run metadata to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The run metadata with the given ID.

    Raises:
        KeyError: if the run metadata doesn't exist.
    """
get_run_step(self, step_run_id, hydrate=True)

Get a step run by ID.

Parameters:

Name Type Description Default
step_run_id UUID

The ID of the step run to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
StepRunResponse

The step run.

Exceptions:

Type Description
KeyError

if the step run doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_step(
    self, step_run_id: UUID, hydrate: bool = True
) -> StepRunResponse:
    """Get a step run by ID.

    Args:
        step_run_id: The ID of the step run to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The step run.

    Raises:
        KeyError: if the step run doesn't exist.
    """
get_schedule(self, schedule_id, hydrate=True)

Get a schedule with a given ID.

Parameters:

Name Type Description Default
schedule_id UUID

ID of the schedule.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ScheduleResponse

The schedule.

Exceptions:

Type Description
KeyError

if the schedule does not exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_schedule(
    self, schedule_id: UUID, hydrate: bool = True
) -> ScheduleResponse:
    """Get a schedule with a given ID.

    Args:
        schedule_id: ID of the schedule.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The schedule.

    Raises:
        KeyError: if the schedule does not exist.
    """
get_secret(self, secret_id, hydrate=True)

Get a secret with a given name.

Parameters:

Name Type Description Default
secret_id UUID

ID of the secret.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
SecretResponse

The secret.

Exceptions:

Type Description
KeyError

if the secret does not exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_secret(
    self, secret_id: UUID, hydrate: bool = True
) -> SecretResponse:
    """Get a secret with a given name.

    Args:
        secret_id: ID of the secret.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The secret.

    Raises:
        KeyError: if the secret does not exist.
    """
get_service_account(self, service_account_name_or_id, hydrate=True)

Gets a specific service account.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or ID of the service account to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ServiceAccountResponse

The requested service account, if it was found.

Exceptions:

Type Description
KeyError

If no service account with the given name or ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
    hydrate: bool = True,
) -> ServiceAccountResponse:
    """Gets a specific service account.

    Args:
        service_account_name_or_id: The name or ID of the service account to
            get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested service account, if it was found.

    Raises:
        KeyError: If no service account with the given name or ID exists.
    """
get_service_connector(self, service_connector_id, hydrate=True)

Gets a specific service connector.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ServiceConnectorResponse

The requested service connector, if it was found.

Exceptions:

Type Description
KeyError

If no service connector with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_service_connector(
    self, service_connector_id: UUID, hydrate: bool = True
) -> ServiceConnectorResponse:
    """Gets a specific service connector.

    Args:
        service_connector_id: The ID of the service connector to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested service connector, if it was found.

    Raises:
        KeyError: If no service connector with the given ID exists.
    """
get_service_connector_client(self, service_connector_id, resource_type=None, resource_id=None)

Get a service connector client for a service connector and given resource.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the base service connector to use.

required
resource_type Optional[str]

The type of resource to get a client for.

None
resource_id Optional[str]

The ID of the resource to get a client for.

None

Returns:

Type Description
ServiceConnectorResponse

A service connector client that can be used to access the given resource.

Exceptions:

Type Description
KeyError

If no service connector with the given name exists.

NotImplementError

If the service connector cannot be instantiated on the store e.g. due to missing package dependencies.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_service_connector_client(
    self,
    service_connector_id: UUID,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
) -> ServiceConnectorResponse:
    """Get a service connector client for a service connector and given resource.

    Args:
        service_connector_id: The ID of the base service connector to use.
        resource_type: The type of resource to get a client for.
        resource_id: The ID of the resource to get a client for.

    Returns:
        A service connector client that can be used to access the given
        resource.

    Raises:
        KeyError: If no service connector with the given name exists.
        NotImplementError: If the service connector cannot be instantiated
            on the store e.g. due to missing package dependencies.
    """
get_service_connector_type(self, connector_type)

Returns the requested service connector type.

Parameters:

Name Type Description Default
connector_type str

the service connector type identifier.

required

Returns:

Type Description
ServiceConnectorTypeModel

The requested service connector type.

Exceptions:

Type Description
KeyError

If no service connector type with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_service_connector_type(
    self,
    connector_type: str,
) -> ServiceConnectorTypeModel:
    """Returns the requested service connector type.

    Args:
        connector_type: the service connector type identifier.

    Returns:
        The requested service connector type.

    Raises:
        KeyError: If no service connector type with the given ID exists.
    """
get_stack(self, stack_id, hydrate=True)

Get a stack by its unique ID.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
StackResponse

The stack with the given ID.

Exceptions:

Type Description
KeyError

if the stack doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_stack(self, stack_id: UUID, hydrate: bool = True) -> StackResponse:
    """Get a stack by its unique ID.

    Args:
        stack_id: The ID of the stack to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack with the given ID.

    Raises:
        KeyError: if the stack doesn't exist.
    """
get_stack_component(self, component_id, hydrate=True)

Get a stack component by ID.

Parameters:

Name Type Description Default
component_id UUID

The ID of the stack component to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
ComponentResponse

The stack component.

Exceptions:

Type Description
KeyError

if the stack component doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_stack_component(
    self,
    component_id: UUID,
    hydrate: bool = True,
) -> ComponentResponse:
    """Get a stack component by ID.

    Args:
        component_id: The ID of the stack component to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The stack component.

    Raises:
        KeyError: if the stack component doesn't exist.
    """
get_store_info(self)

Get information about the store.

Returns:

Type Description
ServerModel

Information about the store.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_store_info(self) -> ServerModel:
    """Get information about the store.

    Returns:
        Information about the store.
    """
get_tag(self, tag_name_or_id, hydrate=True)

Get an existing tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to be retrieved.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TagResponse

The tag of interest.

Exceptions:

Type Description
KeyError

specified ID or name not found.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_tag(
    self, tag_name_or_id: Union[str, UUID], hydrate: bool = True
) -> TagResponse:
    """Get an existing tag.

    Args:
        tag_name_or_id: name or id of the tag to be retrieved.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The tag of interest.

    Raises:
        KeyError: specified ID or name not found.
    """
get_trigger(self, trigger_id, hydrate=True)

Get an trigger by ID.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TriggerResponse

The trigger.

Exceptions:

Type Description
KeyError

if the stack trigger doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_trigger(
    self,
    trigger_id: UUID,
    hydrate: bool = True,
) -> TriggerResponse:
    """Get an trigger by ID.

    Args:
        trigger_id: The ID of the trigger to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The trigger.

    Raises:
        KeyError: if the stack trigger doesn't exist.
    """
get_trigger_execution(self, trigger_execution_id, hydrate=True)

Get an trigger execution by ID.

Parameters:

Name Type Description Default
trigger_execution_id UUID

The ID of the trigger execution to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
TriggerExecutionResponse

The trigger execution.

Exceptions:

Type Description
KeyError

If the trigger execution doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_trigger_execution(
    self,
    trigger_execution_id: UUID,
    hydrate: bool = True,
) -> TriggerExecutionResponse:
    """Get an trigger execution by ID.

    Args:
        trigger_execution_id: The ID of the trigger execution to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The trigger execution.

    Raises:
        KeyError: If the trigger execution doesn't exist.
    """
get_user(self, user_name_or_id=None, include_private=False, hydrate=True)

Gets a specific user, when no id is specified the active user is returned.

Parameters:

Name Type Description Default
user_name_or_id Union[str, uuid.UUID]

The name or ID of the user to get.

None
include_private bool

Whether to include private user information.

False
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
UserResponse

The requested user, if it was found.

Exceptions:

Type Description
KeyError

If no user with the given name or ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_user(
    self,
    user_name_or_id: Optional[Union[str, UUID]] = None,
    include_private: bool = False,
    hydrate: bool = True,
) -> UserResponse:
    """Gets a specific user, when no id is specified the active user is returned.

    Args:
        user_name_or_id: The name or ID of the user to get.
        include_private: Whether to include private user information.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested user, if it was found.

    Raises:
        KeyError: If no user with the given name or ID exists.
    """
get_workspace(self, workspace_name_or_id, hydrate=True)

Get an existing workspace by name or ID.

Parameters:

Name Type Description Default
workspace_name_or_id Union[uuid.UUID, str]

Name or ID of the workspace to get.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

True

Returns:

Type Description
WorkspaceResponse

The requested workspace.

Exceptions:

Type Description
KeyError

If there is no such workspace.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_workspace(
    self, workspace_name_or_id: Union[UUID, str], hydrate: bool = True
) -> WorkspaceResponse:
    """Get an existing workspace by name or ID.

    Args:
        workspace_name_or_id: Name or ID of the workspace to get.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The requested workspace.

    Raises:
        KeyError: If there is no such workspace.
    """
list_api_keys(self, service_account_id, filter_model, hydrate=False)

List all API keys for a service account matching the given filter criteria.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to list the API keys.

required
filter_model APIKeyFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[APIKeyResponse]

A list of all API keys matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_api_keys(
    self,
    service_account_id: UUID,
    filter_model: APIKeyFilter,
    hydrate: bool = False,
) -> Page[APIKeyResponse]:
    """List all API keys for a service account matching the given filter criteria.

    Args:
        service_account_id: The ID of the service account for which to list
            the API keys.
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all API keys matching the filter criteria.
    """
list_artifact_versions(self, artifact_version_filter_model, hydrate=False)

List all artifact versions matching the given filter criteria.

Parameters:

Name Type Description Default
artifact_version_filter_model ArtifactVersionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ArtifactVersionResponse]

A list of all artifact versions matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_artifact_versions(
    self,
    artifact_version_filter_model: ArtifactVersionFilter,
    hydrate: bool = False,
) -> Page[ArtifactVersionResponse]:
    """List all artifact versions matching the given filter criteria.

    Args:
        artifact_version_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all artifact versions matching the filter criteria.
    """
list_artifacts(self, filter_model, hydrate=False)

List all artifacts matching the given filter criteria.

Parameters:

Name Type Description Default
filter_model ArtifactFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ArtifactResponse]

A list of all artifacts matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_artifacts(
    self, filter_model: ArtifactFilter, hydrate: bool = False
) -> Page[ArtifactResponse]:
    """List all artifacts matching the given filter criteria.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all artifacts matching the filter criteria.
    """
list_authorized_devices(self, filter_model, hydrate=False)

List all OAuth 2.0 authorized devices for a user.

Parameters:

Name Type Description Default
filter_model OAuthDeviceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[OAuthDeviceResponse]

A page of all matching OAuth 2.0 authorized devices.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_authorized_devices(
    self, filter_model: OAuthDeviceFilter, hydrate: bool = False
) -> Page[OAuthDeviceResponse]:
    """List all OAuth 2.0 authorized devices for a user.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all matching OAuth 2.0 authorized devices.
    """
list_builds(self, build_filter_model, hydrate=False)

List all builds matching the given filter criteria.

Parameters:

Name Type Description Default
build_filter_model PipelineBuildFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineBuildResponse]

A page of all builds matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_builds(
    self,
    build_filter_model: PipelineBuildFilter,
    hydrate: bool = False,
) -> Page[PipelineBuildResponse]:
    """List all builds matching the given filter criteria.

    Args:
        build_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all builds matching the filter criteria.
    """
list_code_repositories(self, filter_model, hydrate=False)

List all code repositories.

Parameters:

Name Type Description Default
filter_model CodeRepositoryFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[CodeRepositoryResponse]

A page of all code repositories.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_code_repositories(
    self, filter_model: CodeRepositoryFilter, hydrate: bool = False
) -> Page[CodeRepositoryResponse]:
    """List all code repositories.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all code repositories.
    """
list_deployments(self, deployment_filter_model, hydrate=False)

List all deployments matching the given filter criteria.

Parameters:

Name Type Description Default
deployment_filter_model PipelineDeploymentFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineDeploymentResponse]

A page of all deployments matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_deployments(
    self,
    deployment_filter_model: PipelineDeploymentFilter,
    hydrate: bool = False,
) -> Page[PipelineDeploymentResponse]:
    """List all deployments matching the given filter criteria.

    Args:
        deployment_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all deployments matching the filter criteria.
    """
list_event_sources(self, event_source_filter_model, hydrate=False)

List all event_sources matching the given filter criteria.

Parameters:

Name Type Description Default
event_source_filter_model EventSourceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[EventSourceResponse]

A list of all event_sources matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_event_sources(
    self,
    event_source_filter_model: EventSourceFilter,
    hydrate: bool = False,
) -> Page[EventSourceResponse]:
    """List all event_sources matching the given filter criteria.

    Args:
        event_source_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all event_sources matching the filter criteria.
    """
list_flavors(self, flavor_filter_model, hydrate=False)

List all stack component flavors matching the given filter criteria.

Parameters:

Name Type Description Default
flavor_filter_model FlavorFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[FlavorResponse]

List of all the stack component flavors matching the given criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_flavors(
    self,
    flavor_filter_model: FlavorFilter,
    hydrate: bool = False,
) -> Page[FlavorResponse]:
    """List all stack component flavors matching the given filter criteria.

    Args:
        flavor_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        List of all the stack component flavors matching the given criteria.
    """

Get all model version to artifact links by filter.

Parameters:

Name Type Description Default
model_version_artifact_link_filter_model ModelVersionArtifactFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionArtifactResponse]

A page of all model version to artifact links.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_model_version_artifact_links(
    self,
    model_version_artifact_link_filter_model: ModelVersionArtifactFilter,
    hydrate: bool = False,
) -> Page[ModelVersionArtifactResponse]:
    """Get all model version to artifact links by filter.

    Args:
        model_version_artifact_link_filter_model: All filter parameters
            including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model version to artifact links.
    """

Get all model version to pipeline run links by filter.

Parameters:

Name Type Description Default
model_version_pipeline_run_link_filter_model ModelVersionPipelineRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionPipelineRunResponse]

A page of all model version to pipeline run links.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_model_version_pipeline_run_links(
    self,
    model_version_pipeline_run_link_filter_model: ModelVersionPipelineRunFilter,
    hydrate: bool = False,
) -> Page[ModelVersionPipelineRunResponse]:
    """Get all model version to pipeline run links by filter.

    Args:
        model_version_pipeline_run_link_filter_model: All filter parameters
            including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model version to pipeline run links.
    """
list_model_versions(self, model_version_filter_model, model_name_or_id=None, hydrate=False)

Get all model versions by filter.

Parameters:

Name Type Description Default
model_name_or_id Union[str, uuid.UUID]

name or id of the model containing the model versions.

None
model_version_filter_model ModelVersionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelVersionResponse]

A page of all model versions.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_model_versions(
    self,
    model_version_filter_model: ModelVersionFilter,
    model_name_or_id: Optional[Union[str, UUID]] = None,
    hydrate: bool = False,
) -> Page[ModelVersionResponse]:
    """Get all model versions by filter.

    Args:
        model_name_or_id: name or id of the model containing the model
            versions.
        model_version_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all model versions.
    """
list_models(self, model_filter_model, hydrate=False)

Get all models by filter.

Parameters:

Name Type Description Default
model_filter_model ModelFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ModelResponse]

A page of all models.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_models(
    self,
    model_filter_model: ModelFilter,
    hydrate: bool = False,
) -> Page[ModelResponse]:
    """Get all models by filter.

    Args:
        model_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all models.
    """
list_pipelines(self, pipeline_filter_model, hydrate=False)

List all pipelines matching the given filter criteria.

Parameters:

Name Type Description Default
pipeline_filter_model PipelineFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineResponse]

A list of all pipelines matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_pipelines(
    self,
    pipeline_filter_model: PipelineFilter,
    hydrate: bool = False,
) -> Page[PipelineResponse]:
    """List all pipelines matching the given filter criteria.

    Args:
        pipeline_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all pipelines matching the filter criteria.
    """
list_run_metadata(self, run_metadata_filter_model, hydrate=False)

List run metadata.

Parameters:

Name Type Description Default
run_metadata_filter_model RunMetadataFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[RunMetadataResponse]

The run metadata.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_run_metadata(
    self,
    run_metadata_filter_model: RunMetadataFilter,
    hydrate: bool = False,
) -> Page[RunMetadataResponse]:
    """List run metadata.

    Args:
        run_metadata_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        The run metadata.
    """
list_run_steps(self, step_run_filter_model, hydrate=False)

List all step runs matching the given filter criteria.

Parameters:

Name Type Description Default
step_run_filter_model StepRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[StepRunResponse]

A list of all step runs matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_run_steps(
    self,
    step_run_filter_model: StepRunFilter,
    hydrate: bool = False,
) -> Page[StepRunResponse]:
    """List all step runs matching the given filter criteria.

    Args:
        step_run_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all step runs matching the filter criteria.
    """
list_runs(self, runs_filter_model, hydrate=False)

List all pipeline runs matching the given filter criteria.

Parameters:

Name Type Description Default
runs_filter_model PipelineRunFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[PipelineRunResponse]

A list of all pipeline runs matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_runs(
    self,
    runs_filter_model: PipelineRunFilter,
    hydrate: bool = False,
) -> Page[PipelineRunResponse]:
    """List all pipeline runs matching the given filter criteria.

    Args:
        runs_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all pipeline runs matching the filter criteria.
    """
list_schedules(self, schedule_filter_model, hydrate=False)

List all schedules in the workspace.

Parameters:

Name Type Description Default
schedule_filter_model ScheduleFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ScheduleResponse]

A list of schedules.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_schedules(
    self,
    schedule_filter_model: ScheduleFilter,
    hydrate: bool = False,
) -> Page[ScheduleResponse]:
    """List all schedules in the workspace.

    Args:
        schedule_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of schedules.
    """
list_secrets(self, secret_filter_model, hydrate=False)

List all secrets matching the given filter criteria.

Note that returned secrets do not include any secret values. To fetch the secret values, use get_secret.

Parameters:

Name Type Description Default
secret_filter_model SecretFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[SecretResponse]

A list of all secrets matching the filter criteria, with pagination information and sorted according to the filter criteria. The returned secrets do not include any secret values, only metadata. To fetch the secret values, use get_secret individually with each secret.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_secrets(
    self, secret_filter_model: SecretFilter, hydrate: bool = False
) -> Page[SecretResponse]:
    """List all secrets matching the given filter criteria.

    Note that returned secrets do not include any secret values. To fetch
    the secret values, use `get_secret`.

    Args:
        secret_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all secrets matching the filter criteria, with pagination
        information and sorted according to the filter criteria. The
        returned secrets do not include any secret values, only metadata. To
        fetch the secret values, use `get_secret` individually with each
        secret.
    """
list_service_accounts(self, filter_model, hydrate=False)

List all service accounts.

Parameters:

Name Type Description Default
filter_model ServiceAccountFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ServiceAccountResponse]

A list of filtered service accounts.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_service_accounts(
    self,
    filter_model: ServiceAccountFilter,
    hydrate: bool = False,
) -> Page[ServiceAccountResponse]:
    """List all service accounts.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of filtered service accounts.
    """
list_service_connector_resources(self, workspace_name_or_id, connector_type=None, resource_type=None, resource_id=None)

List resources that can be accessed by service connectors.

Parameters:

Name Type Description Default
workspace_name_or_id Union[str, uuid.UUID]

The name or ID of the workspace to scope to.

required
connector_type Optional[str]

The type of service connector to scope to.

None
resource_type Optional[str]

The type of resource to scope to.

None
resource_id Optional[str]

The ID of the resource to scope to.

None

Returns:

Type Description
List[zenml.models.v2.misc.service_connector_type.ServiceConnectorResourcesModel]

The matching list of resources that available service connectors have access to.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_service_connector_resources(
    self,
    workspace_name_or_id: Union[str, UUID],
    connector_type: Optional[str] = None,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
) -> List[ServiceConnectorResourcesModel]:
    """List resources that can be accessed by service connectors.

    Args:
        workspace_name_or_id: The name or ID of the workspace to scope to.
        connector_type: The type of service connector to scope to.
        resource_type: The type of resource to scope to.
        resource_id: The ID of the resource to scope to.

    Returns:
        The matching list of resources that available service
        connectors have access to.
    """
list_service_connector_types(self, connector_type=None, resource_type=None, auth_method=None)

Get a list of service connector types.

Parameters:

Name Type Description Default
connector_type Optional[str]

Filter by connector type.

None
resource_type Optional[str]

Filter by resource type.

None
auth_method Optional[str]

Filter by authentication method.

None

Returns:

Type Description
List[zenml.models.v2.misc.service_connector_type.ServiceConnectorTypeModel]

List of service connector types.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_service_connector_types(
    self,
    connector_type: Optional[str] = None,
    resource_type: Optional[str] = None,
    auth_method: Optional[str] = None,
) -> List[ServiceConnectorTypeModel]:
    """Get a list of service connector types.

    Args:
        connector_type: Filter by connector type.
        resource_type: Filter by resource type.
        auth_method: Filter by authentication method.

    Returns:
        List of service connector types.
    """
list_service_connectors(self, filter_model, hydrate=False)

List all service connectors.

Parameters:

Name Type Description Default
filter_model ServiceConnectorFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ServiceConnectorResponse]

A page of all service connectors.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_service_connectors(
    self,
    filter_model: ServiceConnectorFilter,
    hydrate: bool = False,
) -> Page[ServiceConnectorResponse]:
    """List all service connectors.

    Args:
        filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all service connectors.
    """
list_stack_components(self, component_filter_model, hydrate=False)

List all stack components matching the given filter criteria.

Parameters:

Name Type Description Default
component_filter_model ComponentFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[ComponentResponse]

A list of all stack components matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_stack_components(
    self,
    component_filter_model: ComponentFilter,
    hydrate: bool = False,
) -> Page[ComponentResponse]:
    """List all stack components matching the given filter criteria.

    Args:
        component_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all stack components matching the filter criteria.
    """
list_stacks(self, stack_filter_model, hydrate=False)

List all stacks matching the given filter criteria.

Parameters:

Name Type Description Default
stack_filter_model StackFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[StackResponse]

A list of all stacks matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_stacks(
    self,
    stack_filter_model: StackFilter,
    hydrate: bool = False,
) -> Page[StackResponse]:
    """List all stacks matching the given filter criteria.

    Args:
        stack_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all stacks matching the filter criteria.
    """
list_tags(self, tag_filter_model, hydrate=False)

Get all tags by filter.

Parameters:

Name Type Description Default
tag_filter_model TagFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TagResponse]

A page of all tags.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_tags(
    self,
    tag_filter_model: TagFilter,
    hydrate: bool = False,
) -> Page[TagResponse]:
    """Get all tags by filter.

    Args:
        tag_filter_model: All filter parameters including pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A page of all tags.
    """
list_trigger_executions(self, trigger_execution_filter_model, hydrate=False)

List all trigger executions matching the given filter criteria.

Parameters:

Name Type Description Default
trigger_execution_filter_model TriggerExecutionFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TriggerExecutionResponse]

A list of all trigger executions matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_trigger_executions(
    self,
    trigger_execution_filter_model: TriggerExecutionFilter,
    hydrate: bool = False,
) -> Page[TriggerExecutionResponse]:
    """List all trigger executions matching the given filter criteria.

    Args:
        trigger_execution_filter_model: All filter parameters including
            pagination params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all trigger executions matching the filter criteria.
    """
list_triggers(self, trigger_filter_model, hydrate=False)

List all triggers matching the given filter criteria.

Parameters:

Name Type Description Default
trigger_filter_model TriggerFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[TriggerResponse]

A list of all triggers matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_triggers(
    self,
    trigger_filter_model: TriggerFilter,
    hydrate: bool = False,
) -> Page[TriggerResponse]:
    """List all triggers matching the given filter criteria.

    Args:
        trigger_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all triggers matching the filter criteria.
    """
list_users(self, user_filter_model, hydrate=False)

List all users.

Parameters:

Name Type Description Default
user_filter_model UserFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[UserResponse]

A list of all users.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_users(
    self,
    user_filter_model: UserFilter,
    hydrate: bool = False,
) -> Page[UserResponse]:
    """List all users.

    Args:
        user_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all users.
    """
list_workspaces(self, workspace_filter_model, hydrate=False)

List all workspace matching the given filter criteria.

Parameters:

Name Type Description Default
workspace_filter_model WorkspaceFilter

All filter parameters including pagination params.

required
hydrate bool

Flag deciding whether to hydrate the output model(s) by including metadata fields in the response.

False

Returns:

Type Description
Page[WorkspaceResponse]

A list of all workspace matching the filter criteria.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_workspaces(
    self,
    workspace_filter_model: WorkspaceFilter,
    hydrate: bool = False,
) -> Page[WorkspaceResponse]:
    """List all workspace matching the given filter criteria.

    Args:
        workspace_filter_model: All filter parameters including pagination
            params.
        hydrate: Flag deciding whether to hydrate the output model(s)
            by including metadata fields in the response.

    Returns:
        A list of all workspace matching the filter criteria.
    """
prune_artifact_versions(self, only_versions=True)

Prunes unused artifact versions and their artifacts.

Parameters:

Name Type Description Default
only_versions bool

Only delete artifact versions, keeping artifacts

True
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def prune_artifact_versions(
    self,
    only_versions: bool = True,
) -> None:
    """Prunes unused artifact versions and their artifacts.

    Args:
        only_versions: Only delete artifact versions, keeping artifacts
    """
restore_secrets(self, ignore_errors=False, delete_secrets=False)

Restore all secrets from the configured backup secrets store.

Parameters:

Name Type Description Default
ignore_errors bool

Whether to ignore individual errors during the restore process and attempt to restore all secrets.

False
delete_secrets bool

Whether to delete the secrets that have been successfully restored from the backup secrets store. Setting this flag effectively moves all secrets from the backup secrets store to the primary secrets store.

False

Exceptions:

Type Description
BackupSecretsStoreNotConfiguredError

if no backup secrets store is configured.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def restore_secrets(
    self, ignore_errors: bool = False, delete_secrets: bool = False
) -> None:
    """Restore all secrets from the configured backup secrets store.

    Args:
        ignore_errors: Whether to ignore individual errors during the
            restore process and attempt to restore all secrets.
        delete_secrets: Whether to delete the secrets that have been
            successfully restored from the backup secrets store. Setting
            this flag effectively moves all secrets from the backup secrets
            store to the primary secrets store.

    Raises:
        BackupSecretsStoreNotConfiguredError: if no backup secrets store is
            configured.
    """
rotate_api_key(self, service_account_id, api_key_name_or_id, rotate_request)

Rotate an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to rotate the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to rotate.

required
rotate_request APIKeyRotateRequest

The rotate request on the API key.

required

Returns:

Type Description
APIKeyResponse

The updated API key.

Exceptions:

Type Description
KeyError

if an API key with the given name or ID is not configured for the given service account.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def rotate_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    rotate_request: APIKeyRotateRequest,
) -> APIKeyResponse:
    """Rotate an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to
            rotate the API key.
        api_key_name_or_id: The name or ID of the API key to rotate.
        rotate_request: The rotate request on the API key.

    Returns:
        The updated API key.

    Raises:
        KeyError: if an API key with the given name or ID is not configured
            for the given service account.
    """
update_api_key(self, service_account_id, api_key_name_or_id, api_key_update)

Update an API key for a service account.

Parameters:

Name Type Description Default
service_account_id UUID

The ID of the service account for which to update the API key.

required
api_key_name_or_id Union[str, uuid.UUID]

The name or ID of the API key to update.

required
api_key_update APIKeyUpdate

The update request on the API key.

required

Returns:

Type Description
APIKeyResponse

The updated API key.

Exceptions:

Type Description
KeyError

if an API key with the given name or ID is not configured for the given service account.

EntityExistsError

if the API key update would result in a name conflict with an existing API key for the same service account.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_api_key(
    self,
    service_account_id: UUID,
    api_key_name_or_id: Union[str, UUID],
    api_key_update: APIKeyUpdate,
) -> APIKeyResponse:
    """Update an API key for a service account.

    Args:
        service_account_id: The ID of the service account for which to update
            the API key.
        api_key_name_or_id: The name or ID of the API key to update.
        api_key_update: The update request on the API key.

    Returns:
        The updated API key.

    Raises:
        KeyError: if an API key with the given name or ID is not configured
            for the given service account.
        EntityExistsError: if the API key update would result in a name
            conflict with an existing API key for the same service account.
    """
update_artifact(self, artifact_id, artifact_update)

Updates an artifact.

Parameters:

Name Type Description Default
artifact_id UUID

The ID of the artifact to update.

required
artifact_update ArtifactUpdate

The update to be applied to the artifact.

required

Returns:

Type Description
ArtifactResponse

The updated artifact.

Exceptions:

Type Description
KeyError

if the artifact doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_artifact(
    self, artifact_id: UUID, artifact_update: ArtifactUpdate
) -> ArtifactResponse:
    """Updates an artifact.

    Args:
        artifact_id: The ID of the artifact to update.
        artifact_update: The update to be applied to the artifact.

    Returns:
        The updated artifact.

    Raises:
        KeyError: if the artifact doesn't exist.
    """
update_artifact_version(self, artifact_version_id, artifact_version_update)

Updates an artifact version.

Parameters:

Name Type Description Default
artifact_version_id UUID

The ID of the artifact version to update.

required
artifact_version_update ArtifactVersionUpdate

The update to be applied to the artifact version.

required

Returns:

Type Description
ArtifactVersionResponse

The updated artifact version.

Exceptions:

Type Description
KeyError

if the artifact version doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_artifact_version(
    self,
    artifact_version_id: UUID,
    artifact_version_update: ArtifactVersionUpdate,
) -> ArtifactVersionResponse:
    """Updates an artifact version.

    Args:
        artifact_version_id: The ID of the artifact version to update.
        artifact_version_update: The update to be applied to the artifact
            version.

    Returns:
        The updated artifact version.

    Raises:
        KeyError: if the artifact version doesn't exist.
    """
update_authorized_device(self, device_id, update)

Updates an existing OAuth 2.0 authorized device for internal use.

Parameters:

Name Type Description Default
device_id UUID

The ID of the device to update.

required
update OAuthDeviceUpdate

The update to be applied to the device.

required

Returns:

Type Description
OAuthDeviceResponse

The updated OAuth 2.0 authorized device.

Exceptions:

Type Description
KeyError

If no device with the given ID exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_authorized_device(
    self, device_id: UUID, update: OAuthDeviceUpdate
) -> OAuthDeviceResponse:
    """Updates an existing OAuth 2.0 authorized device for internal use.

    Args:
        device_id: The ID of the device to update.
        update: The update to be applied to the device.

    Returns:
        The updated OAuth 2.0 authorized device.

    Raises:
        KeyError: If no device with the given ID exists.
    """
update_code_repository(self, code_repository_id, update)

Updates an existing code repository.

Parameters:

Name Type Description Default
code_repository_id UUID

The ID of the code repository to update.

required
update CodeRepositoryUpdate

The update to be applied to the code repository.

required

Returns:

Type Description
CodeRepositoryResponse

The updated code repository.

Exceptions:

Type Description
KeyError

If no code repository with the given name exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_code_repository(
    self, code_repository_id: UUID, update: CodeRepositoryUpdate
) -> CodeRepositoryResponse:
    """Updates an existing code repository.

    Args:
        code_repository_id: The ID of the code repository to update.
        update: The update to be applied to the code repository.

    Returns:
        The updated code repository.

    Raises:
        KeyError: If no code repository with the given name exists.
    """
update_event_source(self, event_source_id, event_source_update)

Update an existing event_source.

Parameters:

Name Type Description Default
event_source_id UUID

The ID of the event_source to update.

required
event_source_update EventSourceUpdate

The update to be applied to the event_source.

required

Returns:

Type Description
EventSourceResponse

The updated event_source.

Exceptions:

Type Description
KeyError

if the event_source doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_event_source(
    self,
    event_source_id: UUID,
    event_source_update: EventSourceUpdate,
) -> EventSourceResponse:
    """Update an existing event_source.

    Args:
        event_source_id: The ID of the event_source to update.
        event_source_update: The update to be applied to the event_source.

    Returns:
        The updated event_source.

    Raises:
        KeyError: if the event_source doesn't exist.
    """
update_flavor(self, flavor_id, flavor_update)

Updates an existing user.

Parameters:

Name Type Description Default
flavor_id UUID

The id of the flavor to update.

required
flavor_update FlavorUpdate

The update to be applied to the flavor.

required

Returns:

Type Description
FlavorResponse

The updated flavor.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_flavor(
    self, flavor_id: UUID, flavor_update: FlavorUpdate
) -> FlavorResponse:
    """Updates an existing user.

    Args:
        flavor_id: The id of the flavor to update.
        flavor_update: The update to be applied to the flavor.

    Returns:
        The updated flavor.
    """
update_model(self, model_id, model_update)

Updates an existing model.

Parameters:

Name Type Description Default
model_id UUID

UUID of the model to be updated.

required
model_update ModelUpdate

the Model to be updated.

required

Returns:

Type Description
ModelResponse

The updated model.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_model(
    self,
    model_id: UUID,
    model_update: ModelUpdate,
) -> ModelResponse:
    """Updates an existing model.

    Args:
        model_id: UUID of the model to be updated.
        model_update: the Model to be updated.

    Returns:
        The updated model.
    """
update_model_version(self, model_version_id, model_version_update_model)

Get all model versions by filter.

Parameters:

Name Type Description Default
model_version_id UUID

The ID of model version to be updated.

required
model_version_update_model ModelVersionUpdate

The model version to be updated.

required

Returns:

Type Description
ModelVersionResponse

An updated model version.

Exceptions:

Type Description
KeyError

If the model version not found

RuntimeError

If there is a model version with target stage, but force flag is off

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_model_version(
    self,
    model_version_id: UUID,
    model_version_update_model: ModelVersionUpdate,
) -> ModelVersionResponse:
    """Get all model versions by filter.

    Args:
        model_version_id: The ID of model version to be updated.
        model_version_update_model: The model version to be updated.

    Returns:
        An updated model version.

    Raises:
        KeyError: If the model version not found
        RuntimeError: If there is a model version with target stage,
            but `force` flag is off
    """
update_pipeline(self, pipeline_id, pipeline_update)

Updates a pipeline.

Parameters:

Name Type Description Default
pipeline_id UUID

The ID of the pipeline to be updated.

required
pipeline_update PipelineUpdate

The update to be applied.

required

Returns:

Type Description
PipelineResponse

The updated pipeline.

Exceptions:

Type Description
KeyError

if the pipeline doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_pipeline(
    self,
    pipeline_id: UUID,
    pipeline_update: PipelineUpdate,
) -> PipelineResponse:
    """Updates a pipeline.

    Args:
        pipeline_id: The ID of the pipeline to be updated.
        pipeline_update: The update to be applied.

    Returns:
        The updated pipeline.

    Raises:
        KeyError: if the pipeline doesn't exist.
    """
update_run(self, run_id, run_update)

Updates a pipeline run.

Parameters:

Name Type Description Default
run_id UUID

The ID of the pipeline run to update.

required
run_update PipelineRunUpdate

The update to be applied to the pipeline run.

required

Returns:

Type Description
PipelineRunResponse

The updated pipeline run.

Exceptions:

Type Description
KeyError

if the pipeline run doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_run(
    self, run_id: UUID, run_update: PipelineRunUpdate
) -> PipelineRunResponse:
    """Updates a pipeline run.

    Args:
        run_id: The ID of the pipeline run to update.
        run_update: The update to be applied to the pipeline run.

    Returns:
        The updated pipeline run.

    Raises:
        KeyError: if the pipeline run doesn't exist.
    """
update_run_step(self, step_run_id, step_run_update)

Updates a step run.

Parameters:

Name Type Description Default
step_run_id UUID

The ID of the step to update.

required
step_run_update StepRunUpdate

The update to be applied to the step.

required

Returns:

Type Description
StepRunResponse

The updated step run.

Exceptions:

Type Description
KeyError

if the step run doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_run_step(
    self,
    step_run_id: UUID,
    step_run_update: StepRunUpdate,
) -> StepRunResponse:
    """Updates a step run.

    Args:
        step_run_id: The ID of the step to update.
        step_run_update: The update to be applied to the step.

    Returns:
        The updated step run.

    Raises:
        KeyError: if the step run doesn't exist.
    """
update_schedule(self, schedule_id, schedule_update)

Updates a schedule.

Parameters:

Name Type Description Default
schedule_id UUID

The ID of the schedule to be updated.

required
schedule_update ScheduleUpdate

The update to be applied.

required

Returns:

Type Description
ScheduleResponse

The updated schedule.

Exceptions:

Type Description
KeyError

if the schedule doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_schedule(
    self,
    schedule_id: UUID,
    schedule_update: ScheduleUpdate,
) -> ScheduleResponse:
    """Updates a schedule.

    Args:
        schedule_id: The ID of the schedule to be updated.
        schedule_update: The update to be applied.

    Returns:
        The updated schedule.

    Raises:
        KeyError: if the schedule doesn't exist.
    """
update_secret(self, secret_id, secret_update)

Updates a secret.

Secret values that are specified as None in the update that are present in the existing secret are removed from the existing secret. Values that are present in both secrets are overwritten. All other values in both the existing secret and the update are kept (merged).

If the update includes a change of name or scope, the scoping rules enforced in the secrets store are used to validate the update:

  • only one workspace-scoped secret with the given name can exist in the target workspace.
  • only one user-scoped secret with the given name can exist in the target workspace for the target user.

Parameters:

Name Type Description Default
secret_id UUID

The ID of the secret to be updated.

required
secret_update SecretUpdate

The update to be applied.

required

Returns:

Type Description
SecretResponse

The updated secret.

Exceptions:

Type Description
KeyError

if the secret doesn't exist.

EntityExistsError

If a secret with the same name already exists in the same scope.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_secret(
    self,
    secret_id: UUID,
    secret_update: SecretUpdate,
) -> SecretResponse:
    """Updates a secret.

    Secret values that are specified as `None` in the update that are
    present in the existing secret are removed from the existing secret.
    Values that are present in both secrets are overwritten. All other
    values in both the existing secret and the update are kept (merged).

    If the update includes a change of name or scope, the scoping rules
    enforced in the secrets store are used to validate the update:

      - only one workspace-scoped secret with the given name can exist
        in the target workspace.
      - only one user-scoped secret with the given name can exist in the
        target workspace for the target user.

    Args:
        secret_id: The ID of the secret to be updated.
        secret_update: The update to be applied.

    Returns:
        The updated secret.

    Raises:
        KeyError: if the secret doesn't exist.
        EntityExistsError: If a secret with the same name already exists in
            the same scope.
    """
update_service_account(self, service_account_name_or_id, service_account_update)

Updates an existing service account.

Parameters:

Name Type Description Default
service_account_name_or_id Union[str, uuid.UUID]

The name or the ID of the service account to update.

required
service_account_update ServiceAccountUpdate

The update to be applied to the service account.

required

Returns:

Type Description
ServiceAccountResponse

The updated service account.

Exceptions:

Type Description
KeyError

If no service account with the given name exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_service_account(
    self,
    service_account_name_or_id: Union[str, UUID],
    service_account_update: ServiceAccountUpdate,
) -> ServiceAccountResponse:
    """Updates an existing service account.

    Args:
        service_account_name_or_id: The name or the ID of the service
            account to update.
        service_account_update: The update to be applied to the service
            account.

    Returns:
        The updated service account.

    Raises:
        KeyError: If no service account with the given name exists.
    """
update_service_connector(self, service_connector_id, update)

Updates an existing service connector.

The update model contains the fields to be updated. If a field value is set to None in the model, the field is not updated, but there are special rules concerning some fields:

  • the configuration and secrets fields together represent a full valid configuration update, not just a partial update. If either is set (i.e. not None) in the update, their values are merged together and will replace the existing configuration and secrets values.
  • the resource_id field value is also a full replacement value: if set to None, the resource ID is removed from the service connector.
  • the expiration_seconds field value is also a full replacement value: if set to None, the expiration is removed from the service connector.
  • the secret_id field value in the update is ignored, given that secrets are managed internally by the ZenML store.
  • the labels field is also a full labels update: if set (i.e. not None), all existing labels are removed and replaced by the new labels in the update.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to update.

required
update ServiceConnectorUpdate

The update to be applied to the service connector.

required

Returns:

Type Description
ServiceConnectorResponse

The updated service connector.

Exceptions:

Type Description
KeyError

If no service connector with the given name exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_service_connector(
    self, service_connector_id: UUID, update: ServiceConnectorUpdate
) -> ServiceConnectorResponse:
    """Updates an existing service connector.

    The update model contains the fields to be updated. If a field value is
    set to None in the model, the field is not updated, but there are
    special rules concerning some fields:

    * the `configuration` and `secrets` fields together represent a full
    valid configuration update, not just a partial update. If either is
    set (i.e. not None) in the update, their values are merged together and
    will replace the existing configuration and secrets values.
    * the `resource_id` field value is also a full replacement value: if set
    to `None`, the resource ID is removed from the service connector.
    * the `expiration_seconds` field value is also a full replacement value:
    if set to `None`, the expiration is removed from the service connector.
    * the `secret_id` field value in the update is ignored, given that
    secrets are managed internally by the ZenML store.
    * the `labels` field is also a full labels update: if set (i.e. not
    `None`), all existing labels are removed and replaced by the new labels
    in the update.

    Args:
        service_connector_id: The ID of the service connector to update.
        update: The update to be applied to the service connector.

    Returns:
        The updated service connector.

    Raises:
        KeyError: If no service connector with the given name exists.
    """
update_stack(self, stack_id, stack_update)

Update a stack.

Parameters:

Name Type Description Default
stack_id UUID

The ID of the stack update.

required
stack_update StackUpdate

The update request on the stack.

required

Returns:

Type Description
StackResponse

The updated stack.

Exceptions:

Type Description
KeyError

if the stack doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_stack(
    self, stack_id: UUID, stack_update: StackUpdate
) -> StackResponse:
    """Update a stack.

    Args:
        stack_id: The ID of the stack update.
        stack_update: The update request on the stack.

    Returns:
        The updated stack.

    Raises:
        KeyError: if the stack doesn't exist.
    """
update_stack_component(self, component_id, component_update)

Update an existing stack component.

Parameters:

Name Type Description Default
component_id UUID

The ID of the stack component to update.

required
component_update ComponentUpdate

The update to be applied to the stack component.

required

Returns:

Type Description
ComponentResponse

The updated stack component.

Exceptions:

Type Description
KeyError

if the stack component doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_stack_component(
    self,
    component_id: UUID,
    component_update: ComponentUpdate,
) -> ComponentResponse:
    """Update an existing stack component.

    Args:
        component_id: The ID of the stack component to update.
        component_update: The update to be applied to the stack component.

    Returns:
        The updated stack component.

    Raises:
        KeyError: if the stack component doesn't exist.
    """
update_tag(self, tag_name_or_id, tag_update_model)

Update tag.

Parameters:

Name Type Description Default
tag_name_or_id Union[str, uuid.UUID]

name or id of the tag to be updated.

required
tag_update_model TagUpdate

Tag to use for the update.

required

Returns:

Type Description
TagResponse

An updated tag.

Exceptions:

Type Description
KeyError

If the tag is not found

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_tag(
    self,
    tag_name_or_id: Union[str, UUID],
    tag_update_model: TagUpdate,
) -> TagResponse:
    """Update tag.

    Args:
        tag_name_or_id: name or id of the tag to be updated.
        tag_update_model: Tag to use for the update.

    Returns:
        An updated tag.

    Raises:
        KeyError: If the tag is not found
    """
update_trigger(self, trigger_id, trigger_update)

Update an existing trigger.

Parameters:

Name Type Description Default
trigger_id UUID

The ID of the trigger to update.

required
trigger_update TriggerUpdate

The update to be applied to the trigger.

required

Returns:

Type Description
TriggerResponse

The updated trigger.

Exceptions:

Type Description
KeyError

if the trigger doesn't exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_trigger(
    self,
    trigger_id: UUID,
    trigger_update: TriggerUpdate,
) -> TriggerResponse:
    """Update an existing trigger.

    Args:
        trigger_id: The ID of the trigger to update.
        trigger_update: The update to be applied to the trigger.

    Returns:
        The updated trigger.

    Raises:
        KeyError: if the trigger doesn't exist.
    """
update_user(self, user_id, user_update)

Updates an existing user.

Parameters:

Name Type Description Default
user_id UUID

The id of the user to update.

required
user_update UserUpdate

The update to be applied to the user.

required

Returns:

Type Description
UserResponse

The updated user.

Exceptions:

Type Description
KeyError

If no user with the given name exists.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_user(
    self, user_id: UUID, user_update: UserUpdate
) -> UserResponse:
    """Updates an existing user.

    Args:
        user_id: The id of the user to update.
        user_update: The update to be applied to the user.

    Returns:
        The updated user.

    Raises:
        KeyError: If no user with the given name exists.
    """
update_workspace(self, workspace_id, workspace_update)

Update an existing workspace.

Parameters:

Name Type Description Default
workspace_id UUID

The ID of the workspace to be updated.

required
workspace_update WorkspaceUpdate

The update to be applied to the workspace.

required

Returns:

Type Description
WorkspaceResponse

The updated workspace.

Exceptions:

Type Description
KeyError

if the workspace does not exist.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_workspace(
    self, workspace_id: UUID, workspace_update: WorkspaceUpdate
) -> WorkspaceResponse:
    """Update an existing workspace.

    Args:
        workspace_id: The ID of the workspace to be updated.
        workspace_update: The update to be applied to the workspace.

    Returns:
        The updated workspace.

    Raises:
        KeyError: if the workspace does not exist.
    """
verify_service_connector(self, service_connector_id, resource_type=None, resource_id=None, list_resources=True)

Verifies if a service connector instance has access to one or more resources.

Parameters:

Name Type Description Default
service_connector_id UUID

The ID of the service connector to verify.

required
resource_type Optional[str]

The type of resource to verify access to.

None
resource_id Optional[str]

The ID of the resource to verify access to.

None
list_resources bool

If True, the list of all resources accessible through the service connector and matching the supplied resource type and ID are returned.

True

Returns:

Type Description
ServiceConnectorResourcesModel

The list of resources that the service connector has access to, scoped to the supplied resource type and ID, if provided.

Exceptions:

Type Description
KeyError

If no service connector with the given name exists.

NotImplementError

If the service connector cannot be verified e.g. due to missing package dependencies.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def verify_service_connector(
    self,
    service_connector_id: UUID,
    resource_type: Optional[str] = None,
    resource_id: Optional[str] = None,
    list_resources: bool = True,
) -> ServiceConnectorResourcesModel:
    """Verifies if a service connector instance has access to one or more resources.

    Args:
        service_connector_id: The ID of the service connector to verify.
        resource_type: The type of resource to verify access to.
        resource_id: The ID of the resource to verify access to.
        list_resources: If True, the list of all resources accessible
            through the service connector and matching the supplied resource
            type and ID are returned.

    Returns:
        The list of resources that the service connector has access to,
        scoped to the supplied resource type and ID, if provided.

    Raises:
        KeyError: If no service connector with the given name exists.
        NotImplementError: If the service connector cannot be verified
            e.g. due to missing package dependencies.
    """
verify_service_connector_config(self, service_connector, list_resources=True)

Verifies if a service connector configuration has access to resources.

Parameters:

Name Type Description Default
service_connector ServiceConnectorRequest

The service connector configuration to verify.

required
list_resources bool

If True, the list of all resources accessible through the service connector is returned.

True

Returns:

Type Description
ServiceConnectorResourcesModel

The list of resources that the service connector configuration has access to.

Exceptions:

Type Description
NotImplementError

If the service connector cannot be verified on the store e.g. due to missing package dependencies.

Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def verify_service_connector_config(
    self,
    service_connector: ServiceConnectorRequest,
    list_resources: bool = True,
) -> ServiceConnectorResourcesModel:
    """Verifies if a service connector configuration has access to resources.

    Args:
        service_connector: The service connector configuration to verify.
        list_resources: If True, the list of all resources accessible
            through the service connector is returned.

    Returns:
        The list of resources that the service connector configuration has
        access to.

    Raises:
        NotImplementError: If the service connector cannot be verified
            on the store e.g. due to missing package dependencies.
    """