Zen Stores
zenml.zen_stores
special
ZenStores define ways to store ZenML relevant data locally or remotely.
base_zen_store
Base Zen Store implementation.
BaseZenStore (BaseModel, ZenStoreInterface, AnalyticsTrackerMixin, ABC)
pydantic-model
Base class for accessing and persisting ZenML core objects.
Attributes:
| Name | Type | Description |
|---|---|---|
config |
StoreConfiguration |
The configuration of the store. |
track_analytics |
bool |
Only send analytics if set to |
Source code in zenml/zen_stores/base_zen_store.py
class BaseZenStore(BaseModel, ZenStoreInterface, AnalyticsTrackerMixin, ABC):
"""Base class for accessing and persisting ZenML core objects.
Attributes:
config: The configuration of the store.
track_analytics: Only send analytics if set to `True`.
"""
config: StoreConfiguration
track_analytics: bool = True
TYPE: ClassVar[StoreType]
CONFIG_TYPE: ClassVar[Type[StoreConfiguration]]
# ---------------------------------
# Initialization and configuration
# ---------------------------------
def __init__(
self,
skip_default_registrations: bool = False,
**kwargs: Any,
) -> None:
"""Create and initialize a store.
Args:
skip_default_registrations: If `True`, the creation of the default
stack and user in the store will be skipped.
**kwargs: Additional keyword arguments to pass to the Pydantic
constructor.
Raises:
RuntimeError: If the store cannot be initialized.
"""
super().__init__(**kwargs)
try:
self._initialize()
except Exception as e:
raise RuntimeError(
f"Error initializing {self.type.value} store with URL "
f"'{self.url}': {str(e)}"
) from e
if not skip_default_registrations:
logger.debug("Initializing database")
self._initialize_database()
else:
logger.debug("Skipping database initialization")
@staticmethod
def get_store_class(store_type: StoreType) -> Type["BaseZenStore"]:
"""Returns the class of the given store type.
Args:
store_type: The type of the store to get the class for.
Returns:
The class of the given store type or None if the type is unknown.
Raises:
TypeError: If the store type is unsupported.
"""
if store_type == StoreType.SQL:
from zenml.zen_stores.sql_zen_store import SqlZenStore
return SqlZenStore
elif store_type == StoreType.REST:
from zenml.zen_stores.rest_zen_store import RestZenStore
return RestZenStore
else:
raise TypeError(
f"No store implementation found for store type "
f"`{store_type.value}`."
)
@staticmethod
def get_store_config_class(
store_type: StoreType,
) -> Type["StoreConfiguration"]:
"""Returns the store config class of the given store type.
Args:
store_type: The type of the store to get the class for.
Returns:
The config class of the given store type.
"""
store_class = BaseZenStore.get_store_class(store_type)
return store_class.CONFIG_TYPE
@staticmethod
def get_store_type(url: str) -> StoreType:
"""Returns the store type associated with a URL schema.
Args:
url: The store URL.
Returns:
The store type associated with the supplied URL schema.
Raises:
TypeError: If no store type was found to support the supplied URL.
"""
from zenml.zen_stores.rest_zen_store import RestZenStoreConfiguration
from zenml.zen_stores.sql_zen_store import SqlZenStoreConfiguration
if SqlZenStoreConfiguration.supports_url_scheme(url):
return StoreType.SQL
elif RestZenStoreConfiguration.supports_url_scheme(url):
return StoreType.REST
else:
raise TypeError(f"No store implementation found for URL: {url}.")
@staticmethod
def create_store(
config: StoreConfiguration,
skip_default_registrations: bool = False,
**kwargs: Any,
) -> "BaseZenStore":
"""Create and initialize a store from a store configuration.
Args:
config: The store configuration to use.
skip_default_registrations: If `True`, the creation of the default
stack and user in the store will be skipped.
**kwargs: Additional keyword arguments to pass to the store class
Returns:
The initialized store.
"""
logger.debug(f"Creating store with config '{config}'...")
store_class = BaseZenStore.get_store_class(config.type)
store = store_class(
config=config,
skip_default_registrations=skip_default_registrations,
**kwargs,
)
return store
@staticmethod
def get_default_store_config(path: str) -> StoreConfiguration:
"""Get the default store configuration.
The default store is a SQLite store that saves the DB contents on the
local filesystem.
Args:
path: The local path where the store DB will be stored.
Returns:
The default store configuration.
"""
from zenml.zen_stores.sql_zen_store import SqlZenStoreConfiguration
config = SqlZenStoreConfiguration(
type=StoreType.SQL,
url=SqlZenStoreConfiguration.get_local_url(path),
)
return config
def _initialize_database(self) -> None:
"""Initialize the database on first use."""
try:
default_workspace = self._default_workspace
except KeyError:
default_workspace = self._create_default_workspace()
try:
assert self._admin_role
except KeyError:
self._create_admin_role()
try:
assert self._guest_role
except KeyError:
self._create_guest_role()
try:
default_user = self._default_user
except KeyError:
default_user = self._create_default_user()
try:
self._get_default_stack(
workspace_name_or_id=default_workspace.id,
user_name_or_id=default_user.id,
)
except KeyError:
self._create_default_stack(
workspace_name_or_id=default_workspace.id,
user_name_or_id=default_user.id,
)
@property
def url(self) -> str:
"""The URL of the store.
Returns:
The URL of the store.
"""
return self.config.url
@property
def type(self) -> StoreType:
"""The type of the store.
Returns:
The type of the store.
"""
return self.TYPE
def validate_active_config(
self,
active_workspace_name_or_id: Optional[Union[str, UUID]] = None,
active_stack_id: Optional[UUID] = None,
config_name: str = "",
) -> Tuple[WorkspaceResponseModel, StackResponseModel]:
"""Validate the active configuration.
Call this method to validate the supplied active workspace and active
stack values.
This method is guaranteed to return valid workspace ID and stack ID
values. If the supplied workspace and stack are not set or are not valid
(e.g. they do not exist or are not accessible), the default workspace and
default workspace stack will be returned in their stead.
Args:
active_workspace_name_or_id: The name or ID of the active workspace.
active_stack_id: The ID of the active stack.
config_name: The name of the configuration to validate (used in the
displayed logs/messages).
Returns:
A tuple containing the active workspace and active stack.
"""
active_workspace: WorkspaceResponseModel
if active_workspace_name_or_id:
try:
active_workspace = self.get_workspace(
active_workspace_name_or_id
)
except KeyError:
active_workspace = self._get_or_create_default_workspace()
logger.warning(
f"The current {config_name} active workspace is no longer "
f"available. Resetting the active workspace to "
f"'{active_workspace.name}'."
)
else:
active_workspace = self._get_or_create_default_workspace()
logger.info(
f"Setting the {config_name} active workspace "
f"to '{active_workspace.name}'."
)
active_stack: StackResponseModel
# Sanitize the active stack
if active_stack_id:
# Ensure that the active stack is still valid
try:
active_stack = self.get_stack(stack_id=active_stack_id)
except KeyError:
logger.warning(
"The current %s active stack is no longer available. "
"Resetting the active stack to default.",
config_name,
)
active_stack = self._get_or_create_default_stack(
active_workspace
)
else:
if active_stack.workspace.id != active_workspace.id:
logger.warning(
"The current %s active stack is not part of the active "
"workspace. Resetting the active stack to default.",
config_name,
)
active_stack = self._get_or_create_default_stack(
active_workspace
)
elif not active_stack.is_shared and (
not active_stack.user
or (active_stack.user.id != self.get_user().id)
):
logger.warning(
"The current %s active stack is not shared and not "
"owned by the active user. "
"Resetting the active stack to default.",
config_name,
)
active_stack = self._get_or_create_default_stack(
active_workspace
)
else:
logger.warning(
"Setting the %s active stack to default.",
config_name,
)
active_stack = self._get_or_create_default_stack(active_workspace)
return active_workspace, active_stack
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
return ServerModel(
id=GlobalConfiguration().user_id,
version=zenml.__version__,
deployment_type=os.environ.get(
ENV_ZENML_SERVER_DEPLOYMENT_TYPE, ServerDeploymentType.OTHER
),
database_type=ServerDatabaseType.OTHER,
)
def is_local_store(self) -> bool:
"""Check if the store is local or connected to a local ZenML server.
Returns:
True if the store is local, False otherwise.
"""
return self.get_store_info().is_local()
def _get_or_create_default_stack(
self, workspace: "WorkspaceResponseModel"
) -> "StackResponseModel":
try:
return self._get_default_stack(
workspace_name_or_id=workspace.id,
user_name_or_id=self.get_user().id,
)
except KeyError:
return self._create_default_stack( # type: ignore[no-any-return]
workspace_name_or_id=workspace.id,
user_name_or_id=self.get_user().id,
)
def _get_or_create_default_workspace(self) -> "WorkspaceResponseModel":
try:
return self._default_workspace
except KeyError:
return self._create_default_workspace() # type: ignore[no-any-return]
# ------
# Stacks
# ------
@track(AnalyticsEvent.REGISTERED_DEFAULT_STACK)
def _create_default_stack(
self,
workspace_name_or_id: Union[str, UUID],
user_name_or_id: Union[str, UUID],
) -> StackResponseModel:
"""Create the default stack components and stack.
The default stack contains a local orchestrator and a local artifact
store.
Args:
workspace_name_or_id: Name or ID of the workspace to which the stack
belongs.
user_name_or_id: The name or ID of the user that owns the stack.
Returns:
The model of the created default stack.
"""
workspace = self.get_workspace(
workspace_name_or_id=workspace_name_or_id
)
user = self.get_user(user_name_or_id=user_name_or_id)
logger.info(
f"Creating default stack for user '{user.name}' in workspace "
f"{workspace.name}..."
)
# Register the default orchestrator
orchestrator = self.create_stack_component(
component=ComponentRequestModel(
user=user.id,
workspace=workspace.id,
name=DEFAULT_STACK_COMPONENT_NAME,
type=StackComponentType.ORCHESTRATOR,
flavor="local",
configuration={},
),
)
# Register the default artifact store
artifact_store = self.create_stack_component(
component=ComponentRequestModel(
user=user.id,
workspace=workspace.id,
name=DEFAULT_STACK_COMPONENT_NAME,
type=StackComponentType.ARTIFACT_STORE,
flavor="local",
configuration={},
),
)
components = {c.type: [c.id] for c in [orchestrator, artifact_store]}
# Register the default stack
stack = StackRequestModel(
name=DEFAULT_STACK_NAME,
components=components,
is_shared=False,
workspace=workspace.id,
user=user.id,
)
return self.create_stack(stack=stack)
def _get_default_stack(
self,
workspace_name_or_id: Union[str, UUID],
user_name_or_id: Union[str, UUID],
) -> StackResponseModel:
"""Get the default stack for a user in a workspace.
Args:
workspace_name_or_id: Name or ID of the workspace.
user_name_or_id: Name or ID of the user.
Returns:
The default stack in the workspace owned by the supplied user.
Raises:
KeyError: if the workspace or default stack doesn't exist.
"""
default_stacks = self.list_stacks(
StackFilterModel(
workspace_id=workspace_name_or_id,
user_id=user_name_or_id,
name=DEFAULT_STACK_NAME,
)
)
if default_stacks.total == 0:
raise KeyError(
f"No default stack found for user {str(user_name_or_id)} in "
f"workspace {str(workspace_name_or_id)}"
)
return default_stacks.items[0]
# -----
# Roles
# -----
@property
def _admin_role(self) -> RoleResponseModel:
"""Get the admin role.
Returns:
The default admin role.
"""
return self.get_role(DEFAULT_ADMIN_ROLE)
@track(AnalyticsEvent.CREATED_DEFAULT_ROLES)
def _create_admin_role(self) -> RoleResponseModel:
"""Creates the admin role.
Returns:
The admin role
"""
logger.info(f"Creating '{DEFAULT_ADMIN_ROLE}' role ...")
return self.create_role(
RoleRequestModel(
name=DEFAULT_ADMIN_ROLE,
permissions={
PermissionType.READ.value,
PermissionType.WRITE.value,
PermissionType.ME.value,
},
)
)
@property
def _guest_role(self) -> RoleResponseModel:
"""Get the guest role.
Returns:
The guest role.
"""
return self.get_role(DEFAULT_GUEST_ROLE)
@track(AnalyticsEvent.CREATED_DEFAULT_ROLES)
def _create_guest_role(self) -> RoleResponseModel:
"""Creates the guest role.
Returns:
The guest role
"""
logger.info(f"Creating '{DEFAULT_GUEST_ROLE}' role ...")
return self.create_role(
RoleRequestModel(
name=DEFAULT_GUEST_ROLE,
permissions={
PermissionType.READ.value,
PermissionType.ME.value,
},
)
)
# -----
# Users
# -----
@property
def _default_user_name(self) -> str:
"""Get the default user name.
Returns:
The default user name.
"""
return os.getenv(ENV_ZENML_DEFAULT_USER_NAME, DEFAULT_USERNAME)
@property
def _default_user(self) -> UserResponseModel:
"""Get the default user.
Returns:
The default user.
Raises:
KeyError: If the default user doesn't exist.
"""
user_name = self._default_user_name
try:
return self.get_user(user_name)
except KeyError:
raise KeyError(f"The default user '{user_name}' is not configured")
@track(AnalyticsEvent.CREATED_DEFAULT_USER)
def _create_default_user(self) -> UserResponseModel:
"""Creates a default user with the admin role.
Returns:
The default user.
"""
user_name = os.getenv(ENV_ZENML_DEFAULT_USER_NAME, DEFAULT_USERNAME)
user_password = os.getenv(
ENV_ZENML_DEFAULT_USER_PASSWORD, DEFAULT_PASSWORD
)
logger.info(f"Creating default user '{user_name}' ...")
new_user = self.create_user(
UserRequestModel(
name=user_name,
active=True,
password=user_password,
)
)
self.create_user_role_assignment(
UserRoleAssignmentRequestModel(
role=self._admin_role.id,
user=new_user.id,
workspace=None,
)
)
return new_user
# -----
# Roles
# -----
@property
def roles(self) -> Page[RoleResponseModel]:
"""All existing roles.
Returns:
A list of all existing roles.
"""
return self.list_roles(RoleFilterModel())
# --------
# Workspaces
# --------
@property
def _default_workspace_name(self) -> str:
"""Get the default workspace name.
Returns:
The default workspace name.
"""
return os.getenv(
ENV_ZENML_DEFAULT_WORKSPACE_NAME, DEFAULT_WORKSPACE_NAME
)
@property
def _default_workspace(self) -> WorkspaceResponseModel:
"""Get the default workspace.
Returns:
The default workspace.
Raises:
KeyError: if the default workspace doesn't exist.
"""
workspace_name = self._default_workspace_name
try:
return self.get_workspace(workspace_name)
except KeyError:
raise KeyError(
f"The default workspace '{workspace_name}' is not configured"
)
@track(AnalyticsEvent.CREATED_DEFAULT_WORKSPACE)
def _create_default_workspace(self) -> WorkspaceResponseModel:
"""Creates a default workspace.
Returns:
The default workspace.
"""
workspace_name = self._default_workspace_name
logger.info(f"Creating default workspace '{workspace_name}' ...")
return self.create_workspace(
WorkspaceRequestModel(name=workspace_name)
)
# ---------
# Analytics
# ---------
def track_event(
self,
event: Union[str, AnalyticsEvent],
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Track an analytics event.
Args:
event: The event to track.
metadata: Additional metadata to track with the event.
"""
if self.track_analytics:
# Server information is always tracked, if available.
track_event(event, metadata)
class Config:
"""Pydantic configuration class."""
# Validate attributes when assigning them. We need to set this in order
# to have a mix of mutable and immutable attributes
validate_assignment = True
# Ignore extra attributes from configs of previous ZenML versions
extra = "ignore"
# all attributes with leading underscore are private and therefore
# are mutable and not included in serialization
underscore_attrs_are_private = True
roles: Page[RoleResponseModel]
property
readonly
All existing roles.
Returns:
| Type | Description |
|---|---|
Page[RoleResponseModel] |
A list of all existing roles. |
type: StoreType
property
readonly
The type of the store.
Returns:
| Type | Description |
|---|---|
StoreType |
The type of the store. |
url: str
property
readonly
The URL of the store.
Returns:
| Type | Description |
|---|---|
str |
The URL of the store. |
Config
Pydantic configuration class.
Source code in zenml/zen_stores/base_zen_store.py
class Config:
"""Pydantic configuration class."""
# Validate attributes when assigning them. We need to set this in order
# to have a mix of mutable and immutable attributes
validate_assignment = True
# Ignore extra attributes from configs of previous ZenML versions
extra = "ignore"
# all attributes with leading underscore are private and therefore
# are mutable and not included in serialization
underscore_attrs_are_private = True
__init__(self, skip_default_registrations=False, **kwargs)
special
Create and initialize a store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
skip_default_registrations |
bool |
If |
False |
**kwargs |
Any |
Additional keyword arguments to pass to the Pydantic constructor. |
{} |
Exceptions:
| Type | Description |
|---|---|
RuntimeError |
If the store cannot be initialized. |
Source code in zenml/zen_stores/base_zen_store.py
def __init__(
self,
skip_default_registrations: bool = False,
**kwargs: Any,
) -> None:
"""Create and initialize a store.
Args:
skip_default_registrations: If `True`, the creation of the default
stack and user in the store will be skipped.
**kwargs: Additional keyword arguments to pass to the Pydantic
constructor.
Raises:
RuntimeError: If the store cannot be initialized.
"""
super().__init__(**kwargs)
try:
self._initialize()
except Exception as e:
raise RuntimeError(
f"Error initializing {self.type.value} store with URL "
f"'{self.url}': {str(e)}"
) from e
if not skip_default_registrations:
logger.debug("Initializing database")
self._initialize_database()
else:
logger.debug("Skipping database initialization")
create_store(config, skip_default_registrations=False, **kwargs)
staticmethod
Create and initialize a store from a store configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The store configuration to use. |
required |
skip_default_registrations |
bool |
If |
False |
**kwargs |
Any |
Additional keyword arguments to pass to the store class |
{} |
Returns:
| Type | Description |
|---|---|
BaseZenStore |
The initialized store. |
Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def create_store(
config: StoreConfiguration,
skip_default_registrations: bool = False,
**kwargs: Any,
) -> "BaseZenStore":
"""Create and initialize a store from a store configuration.
Args:
config: The store configuration to use.
skip_default_registrations: If `True`, the creation of the default
stack and user in the store will be skipped.
**kwargs: Additional keyword arguments to pass to the store class
Returns:
The initialized store.
"""
logger.debug(f"Creating store with config '{config}'...")
store_class = BaseZenStore.get_store_class(config.type)
store = store_class(
config=config,
skip_default_registrations=skip_default_registrations,
**kwargs,
)
return store
get_default_store_config(path)
staticmethod
Get the default store configuration.
The default store is a SQLite store that saves the DB contents on the local filesystem.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The local path where the store DB will be stored. |
required |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
The default store configuration. |
Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_default_store_config(path: str) -> StoreConfiguration:
"""Get the default store configuration.
The default store is a SQLite store that saves the DB contents on the
local filesystem.
Args:
path: The local path where the store DB will be stored.
Returns:
The default store configuration.
"""
from zenml.zen_stores.sql_zen_store import SqlZenStoreConfiguration
config = SqlZenStoreConfiguration(
type=StoreType.SQL,
url=SqlZenStoreConfiguration.get_local_url(path),
)
return config
get_store_class(store_type)
staticmethod
Returns the class of the given store type.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
store_type |
StoreType |
The type of the store to get the class for. |
required |
Returns:
| Type | Description |
|---|---|
Type[BaseZenStore] |
The class of the given store type or None if the type is unknown. |
Exceptions:
| Type | Description |
|---|---|
TypeError |
If the store type is unsupported. |
Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_store_class(store_type: StoreType) -> Type["BaseZenStore"]:
"""Returns the class of the given store type.
Args:
store_type: The type of the store to get the class for.
Returns:
The class of the given store type or None if the type is unknown.
Raises:
TypeError: If the store type is unsupported.
"""
if store_type == StoreType.SQL:
from zenml.zen_stores.sql_zen_store import SqlZenStore
return SqlZenStore
elif store_type == StoreType.REST:
from zenml.zen_stores.rest_zen_store import RestZenStore
return RestZenStore
else:
raise TypeError(
f"No store implementation found for store type "
f"`{store_type.value}`."
)
get_store_config_class(store_type)
staticmethod
Returns the store config class of the given store type.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
store_type |
StoreType |
The type of the store to get the class for. |
required |
Returns:
| Type | Description |
|---|---|
Type[StoreConfiguration] |
The config class of the given store type. |
Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_store_config_class(
store_type: StoreType,
) -> Type["StoreConfiguration"]:
"""Returns the store config class of the given store type.
Args:
store_type: The type of the store to get the class for.
Returns:
The config class of the given store type.
"""
store_class = BaseZenStore.get_store_class(store_type)
return store_class.CONFIG_TYPE
get_store_info(self)
Get information about the store.
Returns:
| Type | Description |
|---|---|
ServerModel |
Information about the store. |
Source code in zenml/zen_stores/base_zen_store.py
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
return ServerModel(
id=GlobalConfiguration().user_id,
version=zenml.__version__,
deployment_type=os.environ.get(
ENV_ZENML_SERVER_DEPLOYMENT_TYPE, ServerDeploymentType.OTHER
),
database_type=ServerDatabaseType.OTHER,
)
get_store_type(url)
staticmethod
Returns the store type associated with a URL schema.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The store URL. |
required |
Returns:
| Type | Description |
|---|---|
StoreType |
The store type associated with the supplied URL schema. |
Exceptions:
| Type | Description |
|---|---|
TypeError |
If no store type was found to support the supplied URL. |
Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_store_type(url: str) -> StoreType:
"""Returns the store type associated with a URL schema.
Args:
url: The store URL.
Returns:
The store type associated with the supplied URL schema.
Raises:
TypeError: If no store type was found to support the supplied URL.
"""
from zenml.zen_stores.rest_zen_store import RestZenStoreConfiguration
from zenml.zen_stores.sql_zen_store import SqlZenStoreConfiguration
if SqlZenStoreConfiguration.supports_url_scheme(url):
return StoreType.SQL
elif RestZenStoreConfiguration.supports_url_scheme(url):
return StoreType.REST
else:
raise TypeError(f"No store implementation found for URL: {url}.")
is_local_store(self)
Check if the store is local or connected to a local ZenML server.
Returns:
| Type | Description |
|---|---|
bool |
True if the store is local, False otherwise. |
Source code in zenml/zen_stores/base_zen_store.py
def is_local_store(self) -> bool:
"""Check if the store is local or connected to a local ZenML server.
Returns:
True if the store is local, False otherwise.
"""
return self.get_store_info().is_local()
track_event(self, event, metadata=None)
Track an analytics event.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
event |
Union[str, zenml.utils.analytics_utils.AnalyticsEvent] |
The event to track. |
required |
metadata |
Optional[Dict[str, Any]] |
Additional metadata to track with the event. |
None |
Source code in zenml/zen_stores/base_zen_store.py
def track_event(
self,
event: Union[str, AnalyticsEvent],
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Track an analytics event.
Args:
event: The event to track.
metadata: Additional metadata to track with the event.
"""
if self.track_analytics:
# Server information is always tracked, if available.
track_event(event, metadata)
validate_active_config(self, active_workspace_name_or_id=None, active_stack_id=None, config_name='')
Validate the active configuration.
Call this method to validate the supplied active workspace and active stack values.
This method is guaranteed to return valid workspace ID and stack ID values. If the supplied workspace and stack are not set or are not valid (e.g. they do not exist or are not accessible), the default workspace and default workspace stack will be returned in their stead.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
active_workspace_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the active workspace. |
None |
active_stack_id |
Optional[uuid.UUID] |
The ID of the active stack. |
None |
config_name |
str |
The name of the configuration to validate (used in the displayed logs/messages). |
'' |
Returns:
| Type | Description |
|---|---|
Tuple[zenml.models.workspace_models.WorkspaceResponseModel, zenml.models.stack_models.StackResponseModel] |
A tuple containing the active workspace and active stack. |
Source code in zenml/zen_stores/base_zen_store.py
def validate_active_config(
self,
active_workspace_name_or_id: Optional[Union[str, UUID]] = None,
active_stack_id: Optional[UUID] = None,
config_name: str = "",
) -> Tuple[WorkspaceResponseModel, StackResponseModel]:
"""Validate the active configuration.
Call this method to validate the supplied active workspace and active
stack values.
This method is guaranteed to return valid workspace ID and stack ID
values. If the supplied workspace and stack are not set or are not valid
(e.g. they do not exist or are not accessible), the default workspace and
default workspace stack will be returned in their stead.
Args:
active_workspace_name_or_id: The name or ID of the active workspace.
active_stack_id: The ID of the active stack.
config_name: The name of the configuration to validate (used in the
displayed logs/messages).
Returns:
A tuple containing the active workspace and active stack.
"""
active_workspace: WorkspaceResponseModel
if active_workspace_name_or_id:
try:
active_workspace = self.get_workspace(
active_workspace_name_or_id
)
except KeyError:
active_workspace = self._get_or_create_default_workspace()
logger.warning(
f"The current {config_name} active workspace is no longer "
f"available. Resetting the active workspace to "
f"'{active_workspace.name}'."
)
else:
active_workspace = self._get_or_create_default_workspace()
logger.info(
f"Setting the {config_name} active workspace "
f"to '{active_workspace.name}'."
)
active_stack: StackResponseModel
# Sanitize the active stack
if active_stack_id:
# Ensure that the active stack is still valid
try:
active_stack = self.get_stack(stack_id=active_stack_id)
except KeyError:
logger.warning(
"The current %s active stack is no longer available. "
"Resetting the active stack to default.",
config_name,
)
active_stack = self._get_or_create_default_stack(
active_workspace
)
else:
if active_stack.workspace.id != active_workspace.id:
logger.warning(
"The current %s active stack is not part of the active "
"workspace. Resetting the active stack to default.",
config_name,
)
active_stack = self._get_or_create_default_stack(
active_workspace
)
elif not active_stack.is_shared and (
not active_stack.user
or (active_stack.user.id != self.get_user().id)
):
logger.warning(
"The current %s active stack is not shared and not "
"owned by the active user. "
"Resetting the active stack to default.",
config_name,
)
active_stack = self._get_or_create_default_stack(
active_workspace
)
else:
logger.warning(
"Setting the %s active stack to default.",
config_name,
)
active_stack = self._get_or_create_default_stack(active_workspace)
return active_workspace, active_stack
migrations
special
Alembic database migration utilities.
alembic
Alembic utilities wrapper.
The Alembic class defined here acts as a wrapper around the Alembic library that automatically configures Alembic to use the ZenML SQL store database connection.
Alembic
Alembic environment and migration API.
This class provides a wrapper around the Alembic library that automatically configures Alembic to use the ZenML SQL store database connection.
Source code in zenml/zen_stores/migrations/alembic.py
class Alembic:
"""Alembic environment and migration API.
This class provides a wrapper around the Alembic library that automatically
configures Alembic to use the ZenML SQL store database connection.
"""
def __init__(
self,
engine: Engine,
metadata: MetaData = SQLModel.metadata,
context: Optional[EnvironmentContext] = None,
**kwargs: Any,
) -> None:
"""Initialize the Alembic wrapper.
Args:
engine: The SQLAlchemy engine to use.
metadata: The SQLAlchemy metadata to use.
context: The Alembic environment context to use. If not set, a new
context is created pointing to the ZenML migrations directory.
**kwargs: Additional keyword arguments to pass to the Alembic
environment context.
"""
self.engine = engine
self.metadata = metadata
self.context_kwargs = kwargs
self.config = Config()
self.config.set_main_option(
"script_location", str(Path(__file__).parent)
)
self.config.set_main_option(
"version_locations", str(Path(__file__).parent / "versions")
)
self.script_directory = ScriptDirectory.from_config(self.config)
if context is None:
self.environment_context = EnvironmentContext(
self.config, self.script_directory
)
else:
self.environment_context = context
def db_is_empty(self) -> bool:
"""Check if the database is empty.
Returns:
True if the database is empty, False otherwise.
"""
# Check the existence of any of the SQLModel tables
return not self.engine.dialect.has_table(
self.engine.connect(), schemas.StackSchema.__tablename__
)
def run_migrations(
self,
fn: Optional[Callable[[_RevIdType, MigrationContext], List[Any]]],
) -> None:
"""Run an online migration function in the current migration context.
Args:
fn: Migration function to run. If not set, the function configured
externally by the Alembic CLI command is used.
"""
fn_context_args: Dict[Any, Any] = {}
if fn is not None:
fn_context_args["fn"] = fn
with self.engine.connect() as connection:
self.environment_context.configure(
connection=connection,
target_metadata=self.metadata,
include_object=include_object,
compare_type=True,
render_as_batch=True,
**fn_context_args,
**self.context_kwargs,
)
with self.environment_context.begin_transaction():
self.environment_context.run_migrations()
def current_revisions(self) -> List[str]:
"""Get the current database revisions.
Returns:
List of head revisions.
"""
current_revisions: List[str] = []
def do_get_current_rev(rev: _RevIdType, context: Any) -> List[Any]:
nonlocal current_revisions
for r in self.script_directory.get_all_current(
rev # type:ignore [arg-type]
):
if r is None:
continue
current_revisions.append(r.revision)
return []
self.run_migrations(do_get_current_rev)
return current_revisions
def stamp(self, revision: str) -> None:
"""Stamp the revision table with the given revision without running any migrations.
Args:
revision: String revision target.
"""
def do_stamp(rev: _RevIdType, context: Any) -> List[Any]:
return self.script_directory._stamp_revs(revision, rev)
self.run_migrations(do_stamp)
def upgrade(self, revision: str = "heads") -> None:
"""Upgrade the database to a later version.
Args:
revision: String revision target.
"""
def do_upgrade(rev: _RevIdType, context: Any) -> List[Any]:
return self.script_directory._upgrade_revs(
revision, rev # type:ignore [arg-type]
)
self.run_migrations(do_upgrade)
def downgrade(self, revision: str) -> None:
"""Revert the database to a previous version.
Args:
revision: String revision target.
"""
def do_downgrade(rev: _RevIdType, context: Any) -> List[Any]:
return self.script_directory._downgrade_revs(
revision, rev # type:ignore [arg-type]
)
self.run_migrations(do_downgrade)
__init__(self, engine, metadata=MetaData(), context=None, **kwargs)
special
Initialize the Alembic wrapper.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
engine |
Engine |
The SQLAlchemy engine to use. |
required |
metadata |
MetaData |
The SQLAlchemy metadata to use. |
MetaData() |
context |
Optional[alembic.runtime.environment.EnvironmentContext] |
The Alembic environment context to use. If not set, a new context is created pointing to the ZenML migrations directory. |
None |
**kwargs |
Any |
Additional keyword arguments to pass to the Alembic environment context. |
{} |
Source code in zenml/zen_stores/migrations/alembic.py
def __init__(
self,
engine: Engine,
metadata: MetaData = SQLModel.metadata,
context: Optional[EnvironmentContext] = None,
**kwargs: Any,
) -> None:
"""Initialize the Alembic wrapper.
Args:
engine: The SQLAlchemy engine to use.
metadata: The SQLAlchemy metadata to use.
context: The Alembic environment context to use. If not set, a new
context is created pointing to the ZenML migrations directory.
**kwargs: Additional keyword arguments to pass to the Alembic
environment context.
"""
self.engine = engine
self.metadata = metadata
self.context_kwargs = kwargs
self.config = Config()
self.config.set_main_option(
"script_location", str(Path(__file__).parent)
)
self.config.set_main_option(
"version_locations", str(Path(__file__).parent / "versions")
)
self.script_directory = ScriptDirectory.from_config(self.config)
if context is None:
self.environment_context = EnvironmentContext(
self.config, self.script_directory
)
else:
self.environment_context = context
current_revisions(self)
Get the current database revisions.
Returns:
| Type | Description |
|---|---|
List[str] |
List of head revisions. |
Source code in zenml/zen_stores/migrations/alembic.py
def current_revisions(self) -> List[str]:
"""Get the current database revisions.
Returns:
List of head revisions.
"""
current_revisions: List[str] = []
def do_get_current_rev(rev: _RevIdType, context: Any) -> List[Any]:
nonlocal current_revisions
for r in self.script_directory.get_all_current(
rev # type:ignore [arg-type]
):
if r is None:
continue
current_revisions.append(r.revision)
return []
self.run_migrations(do_get_current_rev)
return current_revisions
db_is_empty(self)
Check if the database is empty.
Returns:
| Type | Description |
|---|---|
bool |
True if the database is empty, False otherwise. |
Source code in zenml/zen_stores/migrations/alembic.py
def db_is_empty(self) -> bool:
"""Check if the database is empty.
Returns:
True if the database is empty, False otherwise.
"""
# Check the existence of any of the SQLModel tables
return not self.engine.dialect.has_table(
self.engine.connect(), schemas.StackSchema.__tablename__
)
downgrade(self, revision)
Revert the database to a previous version.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
revision |
str |
String revision target. |
required |
Source code in zenml/zen_stores/migrations/alembic.py
def downgrade(self, revision: str) -> None:
"""Revert the database to a previous version.
Args:
revision: String revision target.
"""
def do_downgrade(rev: _RevIdType, context: Any) -> List[Any]:
return self.script_directory._downgrade_revs(
revision, rev # type:ignore [arg-type]
)
self.run_migrations(do_downgrade)
run_migrations(self, fn)
Run an online migration function in the current migration context.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
fn |
Optional[Callable[[Union[str, Sequence[str]], alembic.runtime.migration.MigrationContext], List[Any]]] |
Migration function to run. If not set, the function configured externally by the Alembic CLI command is used. |
required |
Source code in zenml/zen_stores/migrations/alembic.py
def run_migrations(
self,
fn: Optional[Callable[[_RevIdType, MigrationContext], List[Any]]],
) -> None:
"""Run an online migration function in the current migration context.
Args:
fn: Migration function to run. If not set, the function configured
externally by the Alembic CLI command is used.
"""
fn_context_args: Dict[Any, Any] = {}
if fn is not None:
fn_context_args["fn"] = fn
with self.engine.connect() as connection:
self.environment_context.configure(
connection=connection,
target_metadata=self.metadata,
include_object=include_object,
compare_type=True,
render_as_batch=True,
**fn_context_args,
**self.context_kwargs,
)
with self.environment_context.begin_transaction():
self.environment_context.run_migrations()
stamp(self, revision)
Stamp the revision table with the given revision without running any migrations.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
revision |
str |
String revision target. |
required |
Source code in zenml/zen_stores/migrations/alembic.py
def stamp(self, revision: str) -> None:
"""Stamp the revision table with the given revision without running any migrations.
Args:
revision: String revision target.
"""
def do_stamp(rev: _RevIdType, context: Any) -> List[Any]:
return self.script_directory._stamp_revs(revision, rev)
self.run_migrations(do_stamp)
upgrade(self, revision='heads')
Upgrade the database to a later version.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
revision |
str |
String revision target. |
'heads' |
Source code in zenml/zen_stores/migrations/alembic.py
def upgrade(self, revision: str = "heads") -> None:
"""Upgrade the database to a later version.
Args:
revision: String revision target.
"""
def do_upgrade(rev: _RevIdType, context: Any) -> List[Any]:
return self.script_directory._upgrade_revs(
revision, rev # type:ignore [arg-type]
)
self.run_migrations(do_upgrade)
AlembicVersion (Base)
Alembic version table.
Source code in zenml/zen_stores/migrations/alembic.py
class AlembicVersion(Base): # type: ignore[valid-type,misc]
"""Alembic version table."""
__tablename__ = "alembic_version"
version_num = Column(String, nullable=False, primary_key=True)
include_object(object, name, type_, *args, **kwargs)
Function used to exclude tables from the migration scripts.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
object |
Any |
The schema item object to check. |
required |
name |
str |
The name of the object to check. |
required |
type_ |
str |
The type of the object to check. |
required |
*args |
Any |
Additional arguments. |
() |
**kwargs |
Any |
Additional keyword arguments. |
{} |
Returns:
| Type | Description |
|---|---|
bool |
True if the object should be included, False otherwise. |
Source code in zenml/zen_stores/migrations/alembic.py
def include_object(
object: Any, name: str, type_: str, *args: Any, **kwargs: Any
) -> bool:
"""Function used to exclude tables from the migration scripts.
Args:
object: The schema item object to check.
name: The name of the object to check.
type_: The type of the object to check.
*args: Additional arguments.
**kwargs: Additional keyword arguments.
Returns:
True if the object should be included, False otherwise.
"""
return not (type_ == "table" and name in exclude_tables)
rest_zen_store
REST Zen Store implementation.
RestZenStore (BaseZenStore)
pydantic-model
Store implementation for accessing data from a REST API.
Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStore(BaseZenStore):
"""Store implementation for accessing data from a REST API."""
config: RestZenStoreConfiguration
TYPE: ClassVar[StoreType] = StoreType.REST
CONFIG_TYPE: ClassVar[Type[StoreConfiguration]] = RestZenStoreConfiguration
_api_token: Optional[str] = None
_session: Optional[requests.Session] = None
def _initialize_database(self) -> None:
"""Initialize the database."""
# don't do anything for a REST store
# ====================================
# ZenML Store interface implementation
# ====================================
# --------------------------------
# Initialization and configuration
# --------------------------------
def _initialize(self) -> None:
"""Initialize the REST store."""
client_version = zenml.__version__
server_version = self.get_store_info().version
if not DISABLE_CLIENT_SERVER_MISMATCH_WARNING and (
server_version != client_version
):
logger.warning(
"Your ZenML client version (%s) does not match the server "
"version (%s). This version mismatch might lead to errors or "
"unexpected behavior. \nTo disable this warning message, set "
"the environment variable `%s=True`",
client_version,
server_version,
ENV_ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING,
)
def get_store_info(self) -> ServerModel:
"""Get information about the server.
Returns:
Information about the server.
"""
body = self.get(INFO)
return ServerModel.parse_obj(body)
# ------
# Stacks
# ------
@track(AnalyticsEvent.REGISTERED_STACK)
def create_stack(self, stack: StackRequestModel) -> StackResponseModel:
"""Register a new stack.
Args:
stack: The stack to register.
Returns:
The registered stack.
"""
return self._create_workspace_scoped_resource(
resource=stack,
route=STACKS,
response_model=StackResponseModel,
)
def get_stack(self, stack_id: UUID) -> StackResponseModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
"""
return self._get_resource(
resource_id=stack_id,
route=STACKS,
response_model=StackResponseModel,
)
def list_stacks(
self, stack_filter_model: StackFilterModel
) -> Page[StackResponseModel]:
"""List all stacks matching the given filter criteria.
Args:
stack_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stacks matching the filter criteria.
"""
return self._list_paginated_resources(
route=STACKS,
response_model=StackResponseModel,
filter_model=stack_filter_model,
)
@track(AnalyticsEvent.UPDATED_STACK)
def update_stack(
self, stack_id: UUID, stack_update: StackUpdateModel
) -> StackResponseModel:
"""Update a stack.
Args:
stack_id: The ID of the stack update.
stack_update: The update request on the stack.
Returns:
The updated stack.
"""
return self._update_resource(
resource_id=stack_id,
resource_update=stack_update,
route=STACKS,
response_model=StackResponseModel,
)
@track(AnalyticsEvent.DELETED_STACK)
def delete_stack(self, stack_id: UUID) -> None:
"""Delete a stack.
Args:
stack_id: The ID of the stack to delete.
"""
self._delete_resource(
resource_id=stack_id,
route=STACKS,
)
# ----------------
# Stack components
# ----------------
@track(AnalyticsEvent.REGISTERED_STACK_COMPONENT)
def create_stack_component(
self,
component: ComponentRequestModel,
) -> ComponentResponseModel:
"""Create a stack component.
Args:
component: The stack component to create.
Returns:
The created stack component.
"""
return self._create_workspace_scoped_resource(
resource=component,
route=STACK_COMPONENTS,
response_model=ComponentResponseModel,
)
def get_stack_component(
self, component_id: UUID
) -> ComponentResponseModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
"""
return self._get_resource(
resource_id=component_id,
route=STACK_COMPONENTS,
response_model=ComponentResponseModel,
)
def list_stack_components(
self, component_filter_model: ComponentFilterModel
) -> Page[ComponentResponseModel]:
"""List all stack components matching the given filter criteria.
Args:
component_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stack components matching the filter criteria.
"""
return self._list_paginated_resources(
route=STACK_COMPONENTS,
response_model=ComponentResponseModel,
filter_model=component_filter_model,
)
@track(AnalyticsEvent.UPDATED_STACK_COMPONENT)
def update_stack_component(
self,
component_id: UUID,
component_update: ComponentUpdateModel,
) -> ComponentResponseModel:
"""Update an existing stack component.
Args:
component_id: The ID of the stack component to update.
component_update: The update to be applied to the stack component.
Returns:
The updated stack component.
"""
return self._update_resource(
resource_id=component_id,
resource_update=component_update,
route=STACK_COMPONENTS,
response_model=ComponentResponseModel,
)
@track(AnalyticsEvent.DELETED_STACK_COMPONENT)
def delete_stack_component(self, component_id: UUID) -> None:
"""Delete a stack component.
Args:
component_id: The ID of the stack component to delete.
"""
self._delete_resource(
resource_id=component_id,
route=STACK_COMPONENTS,
)
# -----------------------
# Stack component flavors
# -----------------------
@track(AnalyticsEvent.CREATED_FLAVOR)
def create_flavor(self, flavor: FlavorRequestModel) -> FlavorResponseModel:
"""Creates a new stack component flavor.
Args:
flavor: The stack component flavor to create.
Returns:
The newly created flavor.
"""
return self._create_workspace_scoped_resource(
resource=flavor,
route=FLAVORS,
response_model=FlavorResponseModel,
)
def get_flavor(self, flavor_id: UUID) -> FlavorResponseModel:
"""Get a stack component flavor by ID.
Args:
flavor_id: The ID of the stack component flavor to get.
Returns:
The stack component flavor.
"""
return self._get_resource(
resource_id=flavor_id,
route=FLAVORS,
response_model=FlavorResponseModel,
)
def list_flavors(
self, flavor_filter_model: FlavorFilterModel
) -> Page[FlavorResponseModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
flavor_filter_model: All filter parameters including pagination
params
Returns:
List of all the stack component flavors matching the given criteria.
"""
return self._list_paginated_resources(
route=FLAVORS,
response_model=FlavorResponseModel,
filter_model=flavor_filter_model,
)
@track(AnalyticsEvent.DELETED_FLAVOR)
def delete_flavor(self, flavor_id: UUID) -> None:
"""Delete a stack component flavor.
Args:
flavor_id: The ID of the stack component flavor to delete.
"""
self._delete_resource(
resource_id=flavor_id,
route=FLAVORS,
)
# -----
# Users
# -----
@track(AnalyticsEvent.CREATED_USER)
def create_user(self, user: UserRequestModel) -> UserResponseModel:
"""Creates a new user.
Args:
user: User to be created.
Returns:
The newly created user.
"""
return self._create_resource(
resource=user,
route=USERS + "?assign_default_role=False",
response_model=UserResponseModel,
)
def get_user(
self,
user_name_or_id: Optional[Union[str, UUID]] = None,
include_private: bool = False,
) -> UserResponseModel:
"""Gets a specific user, when no id is specified the active user is returned.
The `include_private` parameter is ignored here as it is handled
implicitly by the /current-user endpoint that is queried when no
user_name_or_id is set. Raises a KeyError in case a user with that id
does not exist.
Args:
user_name_or_id: The name or ID of the user to get.
include_private: Whether to include private user information
Returns:
The requested user, if it was found.
"""
if user_name_or_id:
return self._get_resource(
resource_id=user_name_or_id,
route=USERS,
response_model=UserResponseModel,
)
else:
body = self.get(CURRENT_USER)
return UserResponseModel.parse_obj(body)
def get_auth_user(
self, user_name_or_id: Union[str, UUID]
) -> "UserAuthModel":
"""Gets the auth model to a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Raises:
NotImplementedError: This method is only available for the
SQLZenStore.
"""
raise NotImplementedError(
"This method is only designed for use"
" by the server endpoints. It is not designed"
" to be called from the client side."
)
def list_users(
self, user_filter_model: UserFilterModel
) -> Page[UserResponseModel]:
"""List all users.
Args:
user_filter_model: All filter parameters including pagination
params.
Returns:
A list of all users.
"""
return self._list_paginated_resources(
route=USERS,
response_model=UserResponseModel,
filter_model=user_filter_model,
)
@track(AnalyticsEvent.UPDATED_USER)
def update_user(
self, user_id: UUID, user_update: UserUpdateModel
) -> UserResponseModel:
"""Updates an existing user.
Args:
user_id: The id of the user to update.
user_update: The update to be applied to the user.
Returns:
The updated user.
"""
return self._update_resource(
resource_id=user_id,
resource_update=user_update,
route=USERS,
response_model=UserResponseModel,
)
@track(AnalyticsEvent.DELETED_USER)
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
"""Deletes a user.
Args:
user_name_or_id: The name or ID of the user to delete.
"""
self._delete_resource(
resource_id=user_name_or_id,
route=USERS,
)
# -----
# Teams
# -----
@track(AnalyticsEvent.CREATED_TEAM)
def create_team(self, team: TeamRequestModel) -> TeamResponseModel:
"""Creates a new team.
Args:
team: The team model to create.
Returns:
The newly created team.
"""
return self._create_resource(
resource=team,
route=TEAMS,
response_model=TeamResponseModel,
)
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamResponseModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
"""
return self._get_resource(
resource_id=team_name_or_id,
route=TEAMS,
response_model=TeamResponseModel,
)
def list_teams(
self, team_filter_model: TeamFilterModel
) -> Page[TeamResponseModel]:
"""List all teams matching the given filter criteria.
Args:
team_filter_model: All filter parameters including pagination
params.
Returns:
A list of all teams matching the filter criteria.
"""
return self._list_paginated_resources(
route=TEAMS,
response_model=TeamResponseModel,
filter_model=team_filter_model,
)
@track(AnalyticsEvent.UPDATED_TEAM)
def update_team(
self, team_id: UUID, team_update: TeamUpdateModel
) -> TeamResponseModel:
"""Update an existing team.
Args:
team_id: The ID of the team to be updated.
team_update: The update to be applied to the team.
Returns:
The updated team.
"""
return self._update_resource(
resource_id=team_id,
resource_update=team_update,
route=TEAMS,
response_model=TeamResponseModel,
)
@track(AnalyticsEvent.DELETED_TEAM)
def delete_team(self, team_name_or_id: Union[str, UUID]) -> None:
"""Deletes a team.
Args:
team_name_or_id: Name or ID of the team to delete.
"""
self._delete_resource(
resource_id=team_name_or_id,
route=TEAMS,
)
# -----
# Roles
# -----
@track(AnalyticsEvent.CREATED_ROLE)
def create_role(self, role: RoleRequestModel) -> RoleResponseModel:
"""Creates a new role.
Args:
role: The role model to create.
Returns:
The newly created role.
"""
return self._create_resource(
resource=role,
route=ROLES,
response_model=RoleResponseModel,
)
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleResponseModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
"""
return self._get_resource(
resource_id=role_name_or_id,
route=ROLES,
response_model=RoleResponseModel,
)
def list_roles(
self, role_filter_model: RoleFilterModel
) -> Page[RoleResponseModel]:
"""List all roles matching the given filter criteria.
Args:
role_filter_model: All filter parameters including pagination
params.
Returns:
A list of all roles matching the filter criteria.
"""
return self._list_paginated_resources(
route=ROLES,
response_model=RoleResponseModel,
filter_model=role_filter_model,
)
@track(AnalyticsEvent.UPDATED_ROLE)
def update_role(
self, role_id: UUID, role_update: RoleUpdateModel
) -> RoleResponseModel:
"""Update an existing role.
Args:
role_id: The ID of the role to be updated.
role_update: The update to be applied to the role.
Returns:
The updated role.
"""
return self._update_resource(
resource_id=role_id,
resource_update=role_update,
route=ROLES,
response_model=RoleResponseModel,
)
@track(AnalyticsEvent.DELETED_ROLE)
def delete_role(self, role_name_or_id: Union[str, UUID]) -> None:
"""Deletes a role.
Args:
role_name_or_id: Name or ID of the role to delete.
"""
self._delete_resource(
resource_id=role_name_or_id,
route=ROLES,
)
# ----------------
# Role assignments
# ----------------
def list_user_role_assignments(
self, user_role_assignment_filter_model: UserRoleAssignmentFilterModel
) -> Page[UserRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
user_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
return self._list_paginated_resources(
route=USER_ROLE_ASSIGNMENTS,
response_model=UserRoleAssignmentResponseModel,
filter_model=user_role_assignment_filter_model,
)
def get_user_role_assignment(
self, user_role_assignment_id: UUID
) -> UserRoleAssignmentResponseModel:
"""Get an existing role assignment by name or ID.
Args:
user_role_assignment_id: Name or ID of the role assignment to get.
Returns:
The requested workspace.
"""
return self._get_resource(
resource_id=user_role_assignment_id,
route=USER_ROLE_ASSIGNMENTS,
response_model=UserRoleAssignmentResponseModel,
)
def delete_user_role_assignment(
self, user_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
user_role_assignment_id: The ID of the specific role assignment
"""
self._delete_resource(
resource_id=user_role_assignment_id,
route=USER_ROLE_ASSIGNMENTS,
)
def create_user_role_assignment(
self, user_role_assignment: UserRoleAssignmentRequestModel
) -> UserRoleAssignmentResponseModel:
"""Creates a new role assignment.
Args:
user_role_assignment: The role assignment to create.
Returns:
The newly created workspace.
"""
return self._create_resource(
resource=user_role_assignment,
route=USER_ROLE_ASSIGNMENTS,
response_model=UserRoleAssignmentResponseModel,
)
# ---------------------
# Team Role assignments
# ---------------------
def create_team_role_assignment(
self, team_role_assignment: TeamRoleAssignmentRequestModel
) -> TeamRoleAssignmentResponseModel:
"""Creates a new team role assignment.
Args:
team_role_assignment: The role assignment model to create.
Returns:
The newly created role assignment.
"""
return self._create_resource(
resource=team_role_assignment,
route=TEAM_ROLE_ASSIGNMENTS,
response_model=TeamRoleAssignmentResponseModel,
)
def get_team_role_assignment(
self, team_role_assignment_id: UUID
) -> TeamRoleAssignmentResponseModel:
"""Gets a specific role assignment.
Args:
team_role_assignment_id: ID of the role assignment to get.
Returns:
The requested role assignment.
"""
return self._get_resource(
resource_id=team_role_assignment_id,
route=TEAM_ROLE_ASSIGNMENTS,
response_model=TeamRoleAssignmentResponseModel,
)
def delete_team_role_assignment(
self, team_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
team_role_assignment_id: The ID of the specific role assignment
"""
self._delete_resource(
resource_id=team_role_assignment_id,
route=TEAM_ROLE_ASSIGNMENTS,
)
def list_team_role_assignments(
self, team_role_assignment_filter_model: TeamRoleAssignmentFilterModel
) -> Page[TeamRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
team_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
return self._list_paginated_resources(
route=TEAM_ROLE_ASSIGNMENTS,
response_model=TeamRoleAssignmentResponseModel,
filter_model=team_role_assignment_filter_model,
)
# --------
# Workspaces
# --------
@track(AnalyticsEvent.CREATED_WORKSPACE)
def create_workspace(
self, workspace: WorkspaceRequestModel
) -> WorkspaceResponseModel:
"""Creates a new workspace.
Args:
workspace: The workspace to create.
Returns:
The newly created workspace.
"""
return self._create_resource(
resource=workspace,
route=WORKSPACES,
response_model=WorkspaceResponseModel,
)
def get_workspace(
self, workspace_name_or_id: Union[UUID, str]
) -> WorkspaceResponseModel:
"""Get an existing workspace by name or ID.
Args:
workspace_name_or_id: Name or ID of the workspace to get.
Returns:
The requested workspace.
"""
return self._get_resource(
resource_id=workspace_name_or_id,
route=WORKSPACES,
response_model=WorkspaceResponseModel,
)
def list_workspaces(
self, workspace_filter_model: WorkspaceFilterModel
) -> Page[WorkspaceResponseModel]:
"""List all workspace matching the given filter criteria.
Args:
workspace_filter_model: All filter parameters including pagination
params.
Returns:
A list of all workspace matching the filter criteria.
"""
return self._list_paginated_resources(
route=WORKSPACES,
response_model=WorkspaceResponseModel,
filter_model=workspace_filter_model,
)
@track(AnalyticsEvent.UPDATED_WORKSPACE)
def update_workspace(
self, workspace_id: UUID, workspace_update: WorkspaceUpdateModel
) -> WorkspaceResponseModel:
"""Update an existing workspace.
Args:
workspace_id: The ID of the workspace to be updated.
workspace_update: The update to be applied to the workspace.
Returns:
The updated workspace.
"""
return self._update_resource(
resource_id=workspace_id,
resource_update=workspace_update,
route=WORKSPACES,
response_model=WorkspaceResponseModel,
)
@track(AnalyticsEvent.DELETED_WORKSPACE)
def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
"""Deletes a workspace.
Args:
workspace_name_or_id: Name or ID of the workspace to delete.
"""
self._delete_resource(
resource_id=workspace_name_or_id,
route=WORKSPACES,
)
# ---------
# Pipelines
# ---------
@track(AnalyticsEvent.CREATE_PIPELINE)
def create_pipeline(
self, pipeline: PipelineRequestModel
) -> PipelineResponseModel:
"""Creates a new pipeline in a workspace.
Args:
pipeline: The pipeline to create.
Returns:
The newly created pipeline.
"""
return self._create_workspace_scoped_resource(
resource=pipeline,
route=PIPELINES,
response_model=PipelineResponseModel,
)
def get_pipeline(self, pipeline_id: UUID) -> PipelineResponseModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
"""
return self._get_resource(
resource_id=pipeline_id,
route=PIPELINES,
response_model=PipelineResponseModel,
)
def list_pipelines(
self, pipeline_filter_model: PipelineFilterModel
) -> Page[PipelineResponseModel]:
"""List all pipelines matching the given filter criteria.
Args:
pipeline_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipelines matching the filter criteria.
"""
return self._list_paginated_resources(
route=PIPELINES,
response_model=PipelineResponseModel,
filter_model=pipeline_filter_model,
)
@track(AnalyticsEvent.UPDATE_PIPELINE)
def update_pipeline(
self, pipeline_id: UUID, pipeline_update: PipelineUpdateModel
) -> PipelineResponseModel:
"""Updates a pipeline.
Args:
pipeline_id: The ID of the pipeline to be updated.
pipeline_update: The update to be applied.
Returns:
The updated pipeline.
"""
return self._update_resource(
resource_id=pipeline_id,
resource_update=pipeline_update,
route=PIPELINES,
response_model=PipelineResponseModel,
)
@track(AnalyticsEvent.DELETE_PIPELINE)
def delete_pipeline(self, pipeline_id: UUID) -> None:
"""Deletes a pipeline.
Args:
pipeline_id: The ID of the pipeline to delete.
"""
self._delete_resource(
resource_id=pipeline_id,
route=PIPELINES,
)
# ---------
# Schedules
# ---------
def create_schedule(
self, schedule: ScheduleRequestModel
) -> ScheduleResponseModel:
"""Creates a new schedule.
Args:
schedule: The schedule to create.
Returns:
The newly created schedule.
"""
return self._create_workspace_scoped_resource(
resource=schedule,
route=SCHEDULES,
response_model=ScheduleResponseModel,
)
def get_schedule(self, schedule_id: UUID) -> ScheduleResponseModel:
"""Get a schedule with a given ID.
Args:
schedule_id: ID of the schedule.
Returns:
The schedule.
"""
return self._get_resource(
resource_id=schedule_id,
route=SCHEDULES,
response_model=ScheduleResponseModel,
)
def list_schedules(
self, schedule_filter_model: ScheduleFilterModel
) -> Page[ScheduleResponseModel]:
"""List all schedules in the workspace.
Args:
schedule_filter_model: All filter parameters including pagination
params
Returns:
A list of schedules.
"""
return self._list_paginated_resources(
route=SCHEDULES,
response_model=ScheduleResponseModel,
filter_model=schedule_filter_model,
)
def update_schedule(
self,
schedule_id: UUID,
schedule_update: ScheduleUpdateModel,
) -> ScheduleResponseModel:
"""Updates a schedule.
Args:
schedule_id: The ID of the schedule to be updated.
schedule_update: The update to be applied.
Returns:
The updated schedule.
"""
return self._update_resource(
resource_id=schedule_id,
resource_update=schedule_update,
route=SCHEDULES,
response_model=ScheduleResponseModel,
)
def delete_schedule(self, schedule_id: UUID) -> None:
"""Deletes a schedule.
Args:
schedule_id: The ID of the schedule to delete.
"""
self._delete_resource(
resource_id=schedule_id,
route=SCHEDULES,
)
# --------------
# Pipeline runs
# --------------
def create_run(
self, pipeline_run: PipelineRunRequestModel
) -> PipelineRunResponseModel:
"""Creates a pipeline run.
Args:
pipeline_run: The pipeline run to create.
Returns:
The created pipeline run.
"""
return self._create_workspace_scoped_resource(
resource=pipeline_run,
response_model=PipelineRunResponseModel,
route=RUNS,
)
def get_run(
self, run_name_or_id: Union[UUID, str]
) -> PipelineRunResponseModel:
"""Gets a pipeline run.
Args:
run_name_or_id: The name or ID of the pipeline run to get.
Returns:
The pipeline run.
"""
return self._get_resource(
resource_id=run_name_or_id,
route=RUNS,
response_model=PipelineRunResponseModel,
)
def get_or_create_run(
self, pipeline_run: PipelineRunRequestModel
) -> Tuple[PipelineRunResponseModel, bool]:
"""Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned.
Otherwise, a new run is created.
Args:
pipeline_run: The pipeline run to get or create.
Returns:
The pipeline run, and a boolean indicating whether the run was
created or not.
"""
return self._get_or_create_workspace_scoped_resource(
resource=pipeline_run,
route=RUNS,
response_model=PipelineRunResponseModel,
)
def list_runs(
self, runs_filter_model: PipelineRunFilterModel
) -> Page[PipelineRunResponseModel]:
"""List all pipeline runs matching the given filter criteria.
Args:
runs_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipeline runs matching the filter criteria.
"""
return self._list_paginated_resources(
route=RUNS,
response_model=PipelineRunResponseModel,
filter_model=runs_filter_model,
)
def update_run(
self, run_id: UUID, run_update: PipelineRunUpdateModel
) -> PipelineRunResponseModel:
"""Updates a pipeline run.
Args:
run_id: The ID of the pipeline run to update.
run_update: The update to be applied to the pipeline run.
Returns:
The updated pipeline run.
"""
return self._update_resource(
resource_id=run_id,
resource_update=run_update,
response_model=PipelineRunResponseModel,
route=RUNS,
)
def delete_run(self, run_id: UUID) -> None:
"""Deletes a pipeline run.
Args:
run_id: The ID of the pipeline run to delete.
"""
self._delete_resource(
resource_id=run_id,
route=RUNS,
)
# ------------------
# Pipeline run steps
# ------------------
def create_run_step(
self, step_run: StepRunRequestModel
) -> StepRunResponseModel:
"""Creates a step run.
Args:
step_run: The step run to create.
Returns:
The created step run.
"""
return self._create_resource(
resource=step_run,
response_model=StepRunResponseModel,
route=STEPS,
)
def get_run_step(self, step_run_id: UUID) -> StepRunResponseModel:
"""Get a step run by ID.
Args:
step_run_id: The ID of the step run to get.
Returns:
The step run.
"""
return self._get_resource(
resource_id=step_run_id,
route=STEPS,
response_model=StepRunResponseModel,
)
def list_run_steps(
self, step_run_filter_model: StepRunFilterModel
) -> Page[StepRunResponseModel]:
"""List all step runs matching the given filter criteria.
Args:
step_run_filter_model: All filter parameters including pagination
params.
Returns:
A list of all step runs matching the filter criteria.
"""
return self._list_paginated_resources(
route=STEPS,
response_model=StepRunResponseModel,
filter_model=step_run_filter_model,
)
def update_run_step(
self,
step_run_id: UUID,
step_run_update: StepRunUpdateModel,
) -> StepRunResponseModel:
"""Updates a step run.
Args:
step_run_id: The ID of the step to update.
step_run_update: The update to be applied to the step.
Returns:
The updated step run.
"""
return self._update_resource(
resource_id=step_run_id,
resource_update=step_run_update,
response_model=StepRunResponseModel,
route=STEPS,
)
# ---------
# Artifacts
# ---------
def create_artifact(
self, artifact: ArtifactRequestModel
) -> ArtifactResponseModel:
"""Creates an artifact.
Args:
artifact: The artifact to create.
Returns:
The created artifact.
"""
return self._create_resource(
resource=artifact,
response_model=ArtifactResponseModel,
route=ARTIFACTS,
)
def get_artifact(self, artifact_id: UUID) -> ArtifactResponseModel:
"""Gets an artifact.
Args:
artifact_id: The ID of the artifact to get.
Returns:
The artifact.
"""
return self._get_resource(
resource_id=artifact_id,
route=ARTIFACTS,
response_model=ArtifactResponseModel,
)
def list_artifacts(
self, artifact_filter_model: ArtifactFilterModel
) -> Page[ArtifactResponseModel]:
"""List all artifacts matching the given filter criteria.
Args:
artifact_filter_model: All filter parameters including pagination
params.
Returns:
A list of all artifacts matching the filter criteria.
"""
return self._list_paginated_resources(
route=ARTIFACTS,
response_model=ArtifactResponseModel,
filter_model=artifact_filter_model,
)
def delete_artifact(self, artifact_id: UUID) -> None:
"""Deletes an artifact.
Args:
artifact_id: The ID of the artifact to delete.
"""
self._delete_resource(resource_id=artifact_id, route=ARTIFACTS)
# ------------
# Run Metadata
# ------------
def create_run_metadata(
self, run_metadata: RunMetadataRequestModel
) -> RunMetadataResponseModel:
"""Creates run metadata.
Args:
run_metadata: The run metadata to create.
Returns:
The created run metadata.
"""
return self._create_workspace_scoped_resource(
resource=run_metadata,
response_model=RunMetadataResponseModel,
route=RUN_METADATA,
)
def list_run_metadata(
self,
run_metadata_filter_model: RunMetadataFilterModel,
) -> Page[RunMetadataResponseModel]:
"""List run metadata.
Args:
run_metadata_filter_model: All filter parameters including
pagination params.
Returns:
The run metadata.
"""
return self._list_paginated_resources(
route=RUN_METADATA,
response_model=RunMetadataResponseModel,
filter_model=run_metadata_filter_model,
)
# =======================
# Internal helper methods
# =======================
def _get_auth_token(self) -> str:
"""Get the authentication token for the REST store.
Returns:
The authentication token.
Raises:
ValueError: if the response from the server isn't in the right
format.
"""
if self._api_token is None:
# Check if the API token is already stored in the config
if self.config.api_token:
self._api_token = self.config.api_token
# Check if the username and password are provided in the config
elif (
self.config.username is not None
and self.config.password is not None
):
response = self._handle_response(
requests.post(
self.url + API + VERSION_1 + LOGIN,
data={
"username": self.config.username,
"password": self.config.password,
},
verify=self.config.verify_ssl,
timeout=self.config.http_timeout,
)
)
if (
not isinstance(response, dict)
or "access_token" not in response
):
raise ValueError(
f"Bad API Response. Expected access token dict, got "
f"{type(response)}"
)
self._api_token = response["access_token"]
self.config.api_token = self._api_token
else:
raise ValueError(
"No API token or username/password provided. Please "
"provide either a token or a username and password in "
"the ZenStore config."
)
return self._api_token
@property
def session(self) -> requests.Session:
"""Authenticate to the ZenML server.
Returns:
A requests session with the authentication token.
"""
if self._session is None:
if self.config.verify_ssl is False:
urllib3.disable_warnings(
urllib3.exceptions.InsecureRequestWarning
)
self._session = requests.Session()
self._session.verify = self.config.verify_ssl
token = self._get_auth_token()
self._session.headers.update({"Authorization": "Bearer " + token})
logger.debug("Authenticated to ZenML server.")
return self._session
@staticmethod
def _handle_response(response: requests.Response) -> Json:
"""Handle API response, translating http status codes to Exception.
Args:
response: The response to handle.
Returns:
The parsed response.
Raises:
DoesNotExistException: If the response indicates that the
requested entity does not exist.
EntityExistsError: If the response indicates that the requested
entity already exists.
AuthorizationException: If the response indicates that the request
is not authorized.
IllegalOperationError: If the response indicates that the requested
operation is forbidden.
KeyError: If the response indicates that the requested entity
does not exist.
RuntimeError: If the response indicates that the requested entity
does not exist.
StackComponentExistsError: If the response indicates that the
requested entity already exists.
StackExistsError: If the response indicates that the requested
entity already exists.
ValueError: If the response indicates that the requested entity
does not exist.
"""
if 200 <= response.status_code < 300:
try:
payload: Json = response.json()
return payload
except requests.exceptions.JSONDecodeError:
raise ValueError(
"Bad response from API. Expected json, got\n"
f"{response.text}"
)
elif response.status_code == 401:
raise AuthorizationException(
f"{response.status_code} Client Error: Unauthorized request to "
f"URL {response.url}: {response.json().get('detail')}"
)
elif response.status_code == 403:
msg = response.json().get("detail", response.text)
if isinstance(msg, list):
msg = msg[-1]
raise IllegalOperationError(msg)
elif response.status_code == 404:
if "KeyError" in response.text:
raise KeyError(
response.json().get("detail", (response.text,))[1]
)
elif "DoesNotExistException" in response.text:
message = ": ".join(
response.json().get("detail", (response.text,))
)
raise DoesNotExistException(message)
raise DoesNotExistException("Endpoint does not exist.")
elif response.status_code == 409:
if "StackComponentExistsError" in response.text:
raise StackComponentExistsError(
message=": ".join(
response.json().get("detail", (response.text,))
)
)
elif "StackExistsError" in response.text:
raise StackExistsError(
message=": ".join(
response.json().get("detail", (response.text,))
)
)
elif "EntityExistsError" in response.text:
raise EntityExistsError(
message=": ".join(
response.json().get("detail", (response.text,))
)
)
else:
raise ValueError(
": ".join(response.json().get("detail", (response.text,)))
)
elif response.status_code == 422:
response_details = response.json().get("detail", (response.text,))
if isinstance(response_details[0], str):
response_msg = ": ".join(response_details)
else:
# This is an "Unprocessable Entity" error, which has a special
# structure in the response.
response_msg = response.text
raise RuntimeError(response_msg)
elif response.status_code == 500:
raise RuntimeError(response.text)
else:
raise RuntimeError(
"Error retrieving from API. Got response "
f"{response.status_code} with body:\n{response.text}"
)
def _request(
self,
method: str,
url: str,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a request to the REST API.
Args:
method: The HTTP method to use.
url: The URL to request.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The parsed response.
"""
params = {k: str(v) for k, v in params.items()} if params else {}
try:
return self._handle_response(
self.session.request(
method,
url,
params=params,
verify=self.config.verify_ssl,
timeout=self.config.http_timeout,
**kwargs,
)
)
except AuthorizationException:
# The authentication token could have expired; refresh it and try
# again
self._session = None
return self._handle_response(
self.session.request(
method,
url,
params=params,
verify=self.config.verify_ssl,
timeout=self.config.http_timeout,
**kwargs,
)
)
def get(
self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
"""Make a GET request to the given endpoint path.
Args:
path: The path to the endpoint.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending GET request to {path}...")
return self._request(
"GET", self.url + API + VERSION_1 + path, params=params, **kwargs
)
def delete(
self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
"""Make a DELETE request to the given endpoint path.
Args:
path: The path to the endpoint.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending DELETE request to {path}...")
return self._request(
"DELETE",
self.url + API + VERSION_1 + path,
params=params,
**kwargs,
)
def post(
self,
path: str,
body: BaseModel,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a POST request to the given endpoint path.
Args:
path: The path to the endpoint.
body: The body to send.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending POST request to {path}...")
return self._request(
"POST",
self.url + API + VERSION_1 + path,
data=body.json(),
params=params,
**kwargs,
)
def put(
self,
path: str,
body: BaseModel,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a PUT request to the given endpoint path.
Args:
path: The path to the endpoint.
body: The body to send.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending PUT request to {path}...")
return self._request(
"PUT",
self.url + API + VERSION_1 + path,
data=body.json(exclude_unset=True),
params=params,
**kwargs,
)
def _create_resource(
self,
resource: BaseRequestModel,
response_model: Type[AnyResponseModel],
route: str,
params: Optional[Dict[str, Any]] = None,
) -> AnyResponseModel:
"""Create a new resource.
Args:
resource: The resource to create.
route: The resource REST API route to use.
response_model: Optional model to use to deserialize the response
body. If not provided, the resource class itself will be used.
params: Optional query parameters to pass to the endpoint.
Returns:
The created resource.
"""
response_body = self.post(f"{route}", body=resource, params=params)
return response_model.parse_obj(response_body)
def _create_workspace_scoped_resource(
self,
resource: WorkspaceScopedRequestModel,
response_model: Type[AnyProjestResponseModel],
route: str,
params: Optional[Dict[str, Any]] = None,
) -> AnyProjestResponseModel:
"""Create a new workspace scoped resource.
Args:
resource: The resource to create.
route: The resource REST API route to use.
response_model: Optional model to use to deserialize the response
body. If not provided, the resource class itself will be used.
params: Optional query parameters to pass to the endpoint.
Returns:
The created resource.
"""
return self._create_resource(
resource=resource,
response_model=response_model,
route=f"{WORKSPACES}/{str(resource.workspace)}{route}",
params=params,
)
def _get_or_create_resource(
self,
resource: BaseRequestModel,
response_model: Type[AnyResponseModel],
route: str,
params: Optional[Dict[str, Any]] = None,
) -> Tuple[AnyResponseModel, bool]:
"""Get or create a resource.
Args:
resource: The resource to get or create.
route: The resource REST API route to use.
response_model: Optional model to use to deserialize the response
body. If not provided, the resource class itself will be used.
params: Optional query parameters to pass to the endpoint.
Returns:
The created resource, and a boolean indicating whether the resource
was created or not.
Raises:
ValueError: If the response body is not a list with 2 elements
where the first element is the resource and the second element
a boolean indicating whether the resource was created or not.
"""
response_body = self.post(
f"{route}{GET_OR_CREATE}",
body=resource,
params=params,
)
if not isinstance(response_body, list):
raise ValueError(
f"Expected a list response from the {route}{GET_OR_CREATE} "
f"endpoint but got {type(response_body)} instead."
)
if len(response_body) != 2:
raise ValueError(
f"Expected a list response with 2 elements from the "
f"{route}{GET_OR_CREATE} endpoint but got {len(response_body)} "
f"elements instead."
)
model_json, was_created = response_body
if not isinstance(was_created, bool):
raise ValueError(
f"Expected a boolean as the second element of the list "
f"response from the {route}{GET_OR_CREATE} endpoint but got "
f"{type(was_created)} instead."
)
return response_model.parse_obj(model_json), was_created
def _get_or_create_workspace_scoped_resource(
self,
resource: WorkspaceScopedRequestModel,
response_model: Type[AnyProjestResponseModel],
route: str,
params: Optional[Dict[str, Any]] = None,
) -> Tuple[AnyProjestResponseModel, bool]:
"""Get or create a workspace scoped resource.
Args:
resource: The resource to get or create.
route: The resource REST API route to use.
response_model: Optional model to use to deserialize the response
body. If not provided, the resource class itself will be used.
params: Optional query parameters to pass to the endpoint.
Returns:
The created resource, and a boolean indicating whether the resource
was created or not.
"""
return self._get_or_create_resource(
resource=resource,
response_model=response_model,
route=f"{WORKSPACES}/{str(resource.workspace)}{route}",
params=params,
)
def _get_resource(
self,
resource_id: Union[str, UUID],
route: str,
response_model: Type[AnyResponseModel],
) -> AnyResponseModel:
"""Retrieve a single resource.
Args:
resource_id: The ID of the resource to retrieve.
route: The resource REST API route to use.
response_model: Model to use to serialize the response body.
Returns:
The retrieved resource.
"""
body = self.get(f"{route}/{str(resource_id)}")
return response_model.parse_obj(body)
def _list_paginated_resources(
self,
route: str,
response_model: Type[AnyResponseModel],
filter_model: BaseFilterModel,
) -> Page[AnyResponseModel]:
"""Retrieve a list of resources filtered by some criteria.
Args:
route: The resource REST API route to use.
response_model: Model to use to serialize the response body.
filter_model: The filter model to use for the list query.
Returns:
List of retrieved resources matching the filter criteria.
Raises:
ValueError: If the value returned by the server is not a list.
"""
# leave out filter params that are not supplied
body = self.get(
f"{route}", params=filter_model.dict(exclude_none=True)
)
if not isinstance(body, dict):
raise ValueError(
f"Bad API Response. Expected list, got {type(body)}"
)
# The initial page of items will be of type BaseResponseModel
page_of_items: Page[AnyResponseModel] = Page.parse_obj(body)
# So these items will be parsed into their correct types like here
page_of_items.items = [
response_model.parse_obj(generic_item)
for generic_item in page_of_items.items
]
return page_of_items
def _list_resources(
self,
route: str,
response_model: Type[AnyResponseModel],
**filters: Any,
) -> List[AnyResponseModel]:
"""Retrieve a list of resources filtered by some criteria.
Args:
route: The resource REST API route to use.
response_model: Model to use to serialize the response body.
filters: Filter parameters to use in the query.
Returns:
List of retrieved resources matching the filter criteria.
Raises:
ValueError: If the value returned by the server is not a list.
"""
# leave out filter params that are not supplied
params = dict(filter(lambda x: x[1] is not None, filters.items()))
body = self.get(f"{route}", params=params)
if not isinstance(body, list):
raise ValueError(
f"Bad API Response. Expected list, got {type(body)}"
)
return [response_model.parse_obj(entry) for entry in body]
def _update_resource(
self,
resource_id: UUID,
resource_update: BaseModel,
response_model: Type[AnyResponseModel],
route: str,
) -> AnyResponseModel:
"""Update an existing resource.
Args:
resource_id: The id of the resource to update.
resource_update: The resource update.
route: The resource REST API route to use.
response_model: Optional model to use to deserialize the response
body. If not provided, the resource class itself will be used.
Returns:
The updated resource.
"""
response_body = self.put(
f"{route}/{str(resource_id)}", body=resource_update
)
return response_model.parse_obj(response_body)
def _delete_resource(
self, resource_id: Union[str, UUID], route: str
) -> None:
"""Delete a resource.
Args:
resource_id: The ID of the resource to delete.
route: The resource REST API route to use.
"""
self.delete(f"{route}/{str(resource_id)}")
session: Session
property
readonly
Authenticate to the ZenML server.
Returns:
| Type | Description |
|---|---|
Session |
A requests session with the authentication token. |
CONFIG_TYPE (StoreConfiguration)
pydantic-model
REST ZenML store configuration.
Attributes:
| Name | Type | Description |
|---|---|---|
username |
Optional[str] |
The username to use to connect to the Zen server. |
password |
Optional[str] |
The password to use to connect to the Zen server. |
verify_ssl |
Union[bool, str] |
Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use or the CA bundle value itself. |
http_timeout |
int |
The timeout to use for all requests. |
Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStoreConfiguration(StoreConfiguration):
"""REST ZenML store configuration.
Attributes:
username: The username to use to connect to the Zen server.
password: The password to use to connect to the Zen server.
verify_ssl: Either a boolean, in which case it controls whether we
verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use or the CA bundle value itself.
http_timeout: The timeout to use for all requests.
"""
type: StoreType = StoreType.REST
username: Optional[str] = None
password: Optional[str] = None
api_token: Optional[str] = None
verify_ssl: Union[bool, str] = True
http_timeout: int = DEFAULT_HTTP_TIMEOUT
@root_validator
def validate_credentials(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validates the credentials provided in the values dictionary.
Args:
values: A dictionary containing the values to be validated.
Raises:
ValueError: If neither api_token nor username is set.
Returns:
The values dictionary.
"""
# Check if the values dictionary contains either an api_token or a
# username as non-empty strings.
if values.get("api_token") or values.get("username"):
return values
else:
raise ValueError(
"Neither api_token nor username is set in the store config."
)
@validator("url")
def validate_url(cls, url: str) -> str:
"""Validates that the URL is a well-formed REST store URL.
Args:
url: The URL to be validated.
Returns:
The validated URL without trailing slashes.
Raises:
ValueError: If the URL is not a well-formed REST store URL.
"""
url = url.rstrip("/")
scheme = re.search("^([a-z0-9]+://)", url)
if scheme is None or scheme.group() not in ("https://", "http://"):
raise ValueError(
"Invalid URL for REST store: {url}. Should be in the form "
"https://hostname[:port] or http://hostname[:port]."
)
# When running inside a container, if the URL uses localhost, the
# target service will not be available. We try to replace localhost
# with one of the special Docker or K3D internal hostnames.
url = replace_localhost_with_internal_hostname(url)
return url
@validator("verify_ssl")
def validate_verify_ssl(
cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
"""Validates that the verify_ssl either points to a file or is a bool.
Args:
verify_ssl: The verify_ssl value to be validated.
Returns:
The validated verify_ssl value.
"""
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
if isinstance(verify_ssl, bool) or verify_ssl.startswith(
str(secret_folder)
):
return verify_ssl
if os.path.isfile(verify_ssl):
with open(verify_ssl, "r") as f:
verify_ssl = f.read()
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, "ca_bundle.pem")
with open(file_path, "w") as f:
f.write(verify_ssl)
file_path.chmod(0o600)
verify_ssl = str(file_path)
return verify_ssl
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
"""Check if a URL scheme is supported by this store.
Args:
url: The URL to check.
Returns:
True if the URL scheme is supported, False otherwise.
"""
return urlparse(url).scheme in ("http", "https")
def expand_certificates(self) -> None:
"""Expands the certificates in the verify_ssl field."""
# Load the certificate values back into the configuration
if isinstance(self.verify_ssl, str) and os.path.isfile(
self.verify_ssl
):
with open(self.verify_ssl, "r") as f:
self.verify_ssl = f.read()
@classmethod
def copy_configuration(
cls,
config: "StoreConfiguration",
config_path: str,
load_config_path: Optional[PurePath] = None,
) -> "StoreConfiguration":
"""Create a copy of the store config using a different path.
This method is used to create a copy of the store configuration that can
be loaded using a different configuration path or in the context of a
new environment, such as a container image.
The configuration files accompanying the store configuration are also
copied to the new configuration path (e.g. certificates etc.).
Args:
config: The store configuration to copy.
config_path: new path where the configuration copy will be loaded
from.
load_config_path: absolute path that will be used to load the copied
configuration. This can be set to a value different from
`config_path` if the configuration copy will be loaded from
a different environment, e.g. when the configuration is copied
to a container image and loaded using a different absolute path.
This will be reflected in the paths and URLs encoded in the
copied configuration.
Returns:
A new store configuration object that reflects the new configuration
path.
"""
assert isinstance(config, RestZenStoreConfiguration)
assert config.api_token is not None
config = config.copy(exclude={"username", "password"}, deep=True)
# Load the certificate values back into the configuration
config.expand_certificates()
return config
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the `verify_ssl` attribute can be expanded to the contents
# of the certificate file.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
Config
Pydantic configuration class.
Source code in zenml/zen_stores/rest_zen_store.py
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the `verify_ssl` attribute can be expanded to the contents
# of the certificate file.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
copy_configuration(config, config_path, load_config_path=None)
classmethod
Create a copy of the store config using a different path.
This method is used to create a copy of the store configuration that can be loaded using a different configuration path or in the context of a new environment, such as a container image.
The configuration files accompanying the store configuration are also copied to the new configuration path (e.g. certificates etc.).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The store configuration to copy. |
required |
config_path |
str |
new path where the configuration copy will be loaded from. |
required |
load_config_path |
Optional[pathlib.PurePath] |
absolute path that will be used to load the copied
configuration. This can be set to a value different from
|
None |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
A new store configuration object that reflects the new configuration path. |
Source code in zenml/zen_stores/rest_zen_store.py
@classmethod
def copy_configuration(
cls,
config: "StoreConfiguration",
config_path: str,
load_config_path: Optional[PurePath] = None,
) -> "StoreConfiguration":
"""Create a copy of the store config using a different path.
This method is used to create a copy of the store configuration that can
be loaded using a different configuration path or in the context of a
new environment, such as a container image.
The configuration files accompanying the store configuration are also
copied to the new configuration path (e.g. certificates etc.).
Args:
config: The store configuration to copy.
config_path: new path where the configuration copy will be loaded
from.
load_config_path: absolute path that will be used to load the copied
configuration. This can be set to a value different from
`config_path` if the configuration copy will be loaded from
a different environment, e.g. when the configuration is copied
to a container image and loaded using a different absolute path.
This will be reflected in the paths and URLs encoded in the
copied configuration.
Returns:
A new store configuration object that reflects the new configuration
path.
"""
assert isinstance(config, RestZenStoreConfiguration)
assert config.api_token is not None
config = config.copy(exclude={"username", "password"}, deep=True)
# Load the certificate values back into the configuration
config.expand_certificates()
return config
expand_certificates(self)
Expands the certificates in the verify_ssl field.
Source code in zenml/zen_stores/rest_zen_store.py
def expand_certificates(self) -> None:
"""Expands the certificates in the verify_ssl field."""
# Load the certificate values back into the configuration
if isinstance(self.verify_ssl, str) and os.path.isfile(
self.verify_ssl
):
with open(self.verify_ssl, "r") as f:
self.verify_ssl = f.read()
supports_url_scheme(url)
classmethod
Check if a URL scheme is supported by this store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The URL to check. |
required |
Returns:
| Type | Description |
|---|---|
bool |
True if the URL scheme is supported, False otherwise. |
Source code in zenml/zen_stores/rest_zen_store.py
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
"""Check if a URL scheme is supported by this store.
Args:
url: The URL to check.
Returns:
True if the URL scheme is supported, False otherwise.
"""
return urlparse(url).scheme in ("http", "https")
validate_credentials(values)
classmethod
Validates the credentials provided in the values dictionary.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Dict[str, Any] |
A dictionary containing the values to be validated. |
required |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If neither api_token nor username is set. |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The values dictionary. |
Source code in zenml/zen_stores/rest_zen_store.py
@root_validator
def validate_credentials(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validates the credentials provided in the values dictionary.
Args:
values: A dictionary containing the values to be validated.
Raises:
ValueError: If neither api_token nor username is set.
Returns:
The values dictionary.
"""
# Check if the values dictionary contains either an api_token or a
# username as non-empty strings.
if values.get("api_token") or values.get("username"):
return values
else:
raise ValueError(
"Neither api_token nor username is set in the store config."
)
validate_url(url)
classmethod
Validates that the URL is a well-formed REST store URL.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The URL to be validated. |
required |
Returns:
| Type | Description |
|---|---|
str |
The validated URL without trailing slashes. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the URL is not a well-formed REST store URL. |
Source code in zenml/zen_stores/rest_zen_store.py
@validator("url")
def validate_url(cls, url: str) -> str:
"""Validates that the URL is a well-formed REST store URL.
Args:
url: The URL to be validated.
Returns:
The validated URL without trailing slashes.
Raises:
ValueError: If the URL is not a well-formed REST store URL.
"""
url = url.rstrip("/")
scheme = re.search("^([a-z0-9]+://)", url)
if scheme is None or scheme.group() not in ("https://", "http://"):
raise ValueError(
"Invalid URL for REST store: {url}. Should be in the form "
"https://hostname[:port] or http://hostname[:port]."
)
# When running inside a container, if the URL uses localhost, the
# target service will not be available. We try to replace localhost
# with one of the special Docker or K3D internal hostnames.
url = replace_localhost_with_internal_hostname(url)
return url
validate_verify_ssl(verify_ssl)
classmethod
Validates that the verify_ssl either points to a file or is a bool.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
verify_ssl |
Union[bool, str] |
The verify_ssl value to be validated. |
required |
Returns:
| Type | Description |
|---|---|
Union[bool, str] |
The validated verify_ssl value. |
Source code in zenml/zen_stores/rest_zen_store.py
@validator("verify_ssl")
def validate_verify_ssl(
cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
"""Validates that the verify_ssl either points to a file or is a bool.
Args:
verify_ssl: The verify_ssl value to be validated.
Returns:
The validated verify_ssl value.
"""
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
if isinstance(verify_ssl, bool) or verify_ssl.startswith(
str(secret_folder)
):
return verify_ssl
if os.path.isfile(verify_ssl):
with open(verify_ssl, "r") as f:
verify_ssl = f.read()
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, "ca_bundle.pem")
with open(file_path, "w") as f:
f.write(verify_ssl)
file_path.chmod(0o600)
verify_ssl = str(file_path)
return verify_ssl
create_artifact(self, artifact)
Creates an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact |
ArtifactRequestModel |
The artifact to create. |
required |
Returns:
| Type | Description |
|---|---|
ArtifactResponseModel |
The created artifact. |
Source code in zenml/zen_stores/rest_zen_store.py
def create_artifact(
self, artifact: ArtifactRequestModel
) -> ArtifactResponseModel:
"""Creates an artifact.
Args:
artifact: The artifact to create.
Returns:
The created artifact.
"""
return self._create_resource(
resource=artifact,
response_model=ArtifactResponseModel,
route=ARTIFACTS,
)
create_flavor(*args, **kwargs)
Creates a new stack component flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor |
The stack component flavor to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created flavor. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_pipeline(*args, **kwargs)
Creates a new pipeline in a workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline |
The pipeline to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created pipeline. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_role(*args, **kwargs)
Creates a new role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role |
The role model to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created role. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_run(self, pipeline_run)
Creates a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_run |
PipelineRunRequestModel |
The pipeline run to create. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The created pipeline run. |
Source code in zenml/zen_stores/rest_zen_store.py
def create_run(
self, pipeline_run: PipelineRunRequestModel
) -> PipelineRunResponseModel:
"""Creates a pipeline run.
Args:
pipeline_run: The pipeline run to create.
Returns:
The created pipeline run.
"""
return self._create_workspace_scoped_resource(
resource=pipeline_run,
response_model=PipelineRunResponseModel,
route=RUNS,
)
create_run_metadata(self, run_metadata)
Creates run metadata.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_metadata |
RunMetadataRequestModel |
The run metadata to create. |
required |
Returns:
| Type | Description |
|---|---|
RunMetadataResponseModel |
The created run metadata. |
Source code in zenml/zen_stores/rest_zen_store.py
def create_run_metadata(
self, run_metadata: RunMetadataRequestModel
) -> RunMetadataResponseModel:
"""Creates run metadata.
Args:
run_metadata: The run metadata to create.
Returns:
The created run metadata.
"""
return self._create_workspace_scoped_resource(
resource=run_metadata,
response_model=RunMetadataResponseModel,
route=RUN_METADATA,
)
create_run_step(self, step_run)
Creates a step run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run |
StepRunRequestModel |
The step run to create. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The created step run. |
Source code in zenml/zen_stores/rest_zen_store.py
def create_run_step(
self, step_run: StepRunRequestModel
) -> StepRunResponseModel:
"""Creates a step run.
Args:
step_run: The step run to create.
Returns:
The created step run.
"""
return self._create_resource(
resource=step_run,
response_model=StepRunResponseModel,
route=STEPS,
)
create_schedule(self, schedule)
Creates a new schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule |
ScheduleRequestModel |
The schedule to create. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The newly created schedule. |
Source code in zenml/zen_stores/rest_zen_store.py
def create_schedule(
self, schedule: ScheduleRequestModel
) -> ScheduleResponseModel:
"""Creates a new schedule.
Args:
schedule: The schedule to create.
Returns:
The newly created schedule.
"""
return self._create_workspace_scoped_resource(
resource=schedule,
route=SCHEDULES,
response_model=ScheduleResponseModel,
)
create_stack(*args, **kwargs)
Register a new stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack |
The stack to register. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The registered stack. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_stack_component(*args, **kwargs)
Create a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
The stack component to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The created stack component. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_team(*args, **kwargs)
Creates a new team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team |
The team model to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created team. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_team_role_assignment(self, team_role_assignment)
Creates a new team role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment |
TeamRoleAssignmentRequestModel |
The role assignment model to create. |
required |
Returns:
| Type | Description |
|---|---|
TeamRoleAssignmentResponseModel |
The newly created role assignment. |
Source code in zenml/zen_stores/rest_zen_store.py
def create_team_role_assignment(
self, team_role_assignment: TeamRoleAssignmentRequestModel
) -> TeamRoleAssignmentResponseModel:
"""Creates a new team role assignment.
Args:
team_role_assignment: The role assignment model to create.
Returns:
The newly created role assignment.
"""
return self._create_resource(
resource=team_role_assignment,
route=TEAM_ROLE_ASSIGNMENTS,
response_model=TeamRoleAssignmentResponseModel,
)
create_user(*args, **kwargs)
Creates a new user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user |
User to be created. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created user. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_user_role_assignment(self, user_role_assignment)
Creates a new role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment |
UserRoleAssignmentRequestModel |
The role assignment to create. |
required |
Returns:
| Type | Description |
|---|---|
UserRoleAssignmentResponseModel |
The newly created workspace. |
Source code in zenml/zen_stores/rest_zen_store.py
def create_user_role_assignment(
self, user_role_assignment: UserRoleAssignmentRequestModel
) -> UserRoleAssignmentResponseModel:
"""Creates a new role assignment.
Args:
user_role_assignment: The role assignment to create.
Returns:
The newly created workspace.
"""
return self._create_resource(
resource=user_role_assignment,
route=USER_ROLE_ASSIGNMENTS,
response_model=UserRoleAssignmentResponseModel,
)
create_workspace(*args, **kwargs)
Creates a new workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace |
The workspace to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created workspace. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete(self, path, params=None, **kwargs)
Make a DELETE request to the given endpoint path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the endpoint. |
required |
params |
Optional[Dict[str, Any]] |
The query parameters to pass to the endpoint. |
None |
kwargs |
Any |
Additional keyword arguments to pass to the request. |
{} |
Returns:
| Type | Description |
|---|---|
Union[Dict[str, Any], List[Any], str, int, float, bool] |
The response body. |
Source code in zenml/zen_stores/rest_zen_store.py
def delete(
self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
"""Make a DELETE request to the given endpoint path.
Args:
path: The path to the endpoint.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending DELETE request to {path}...")
return self._request(
"DELETE",
self.url + API + VERSION_1 + path,
params=params,
**kwargs,
)
delete_artifact(self, artifact_id)
Deletes an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_id |
UUID |
The ID of the artifact to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def delete_artifact(self, artifact_id: UUID) -> None:
"""Deletes an artifact.
Args:
artifact_id: The ID of the artifact to delete.
"""
self._delete_resource(resource_id=artifact_id, route=ARTIFACTS)
delete_flavor(*args, **kwargs)
Delete a stack component flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
The ID of the stack component flavor to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_pipeline(*args, **kwargs)
Deletes a pipeline.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
The ID of the pipeline to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_role(*args, **kwargs)
Deletes a role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Name or ID of the role to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_run(self, run_id)
Deletes a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def delete_run(self, run_id: UUID) -> None:
"""Deletes a pipeline run.
Args:
run_id: The ID of the pipeline run to delete.
"""
self._delete_resource(
resource_id=run_id,
route=RUNS,
)
delete_schedule(self, schedule_id)
Deletes a schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
The ID of the schedule to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def delete_schedule(self, schedule_id: UUID) -> None:
"""Deletes a schedule.
Args:
schedule_id: The ID of the schedule to delete.
"""
self._delete_resource(
resource_id=schedule_id,
route=SCHEDULES,
)
delete_stack(*args, **kwargs)
Delete a stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
The ID of the stack to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_stack_component(*args, **kwargs)
Delete a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
The ID of the stack component to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_team(*args, **kwargs)
Deletes a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Name or ID of the team to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_team_role_assignment(self, team_role_assignment_id)
Delete a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_id |
UUID |
The ID of the specific role assignment |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def delete_team_role_assignment(
self, team_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
team_role_assignment_id: The ID of the specific role assignment
"""
self._delete_resource(
resource_id=team_role_assignment_id,
route=TEAM_ROLE_ASSIGNMENTS,
)
delete_user(*args, **kwargs)
Deletes a user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
The name or ID of the user to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_user_role_assignment(self, user_role_assignment_id)
Delete a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_id |
UUID |
The ID of the specific role assignment |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def delete_user_role_assignment(
self, user_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
user_role_assignment_id: The ID of the specific role assignment
"""
self._delete_resource(
resource_id=user_role_assignment_id,
route=USER_ROLE_ASSIGNMENTS,
)
delete_workspace(*args, **kwargs)
Deletes a workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_name_or_id |
Name or ID of the workspace to delete. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
get(self, path, params=None, **kwargs)
Make a GET request to the given endpoint path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the endpoint. |
required |
params |
Optional[Dict[str, Any]] |
The query parameters to pass to the endpoint. |
None |
kwargs |
Any |
Additional keyword arguments to pass to the request. |
{} |
Returns:
| Type | Description |
|---|---|
Union[Dict[str, Any], List[Any], str, int, float, bool] |
The response body. |
Source code in zenml/zen_stores/rest_zen_store.py
def get(
self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
"""Make a GET request to the given endpoint path.
Args:
path: The path to the endpoint.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending GET request to {path}...")
return self._request(
"GET", self.url + API + VERSION_1 + path, params=params, **kwargs
)
get_artifact(self, artifact_id)
Gets an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_id |
UUID |
The ID of the artifact to get. |
required |
Returns:
| Type | Description |
|---|---|
ArtifactResponseModel |
The artifact. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_artifact(self, artifact_id: UUID) -> ArtifactResponseModel:
"""Gets an artifact.
Args:
artifact_id: The ID of the artifact to get.
Returns:
The artifact.
"""
return self._get_resource(
resource_id=artifact_id,
route=ARTIFACTS,
response_model=ArtifactResponseModel,
)
get_auth_user(self, user_name_or_id)
Gets the auth model to a specific user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
required |
Exceptions:
| Type | Description |
|---|---|
NotImplementedError |
This method is only available for the SQLZenStore. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_auth_user(
self, user_name_or_id: Union[str, UUID]
) -> "UserAuthModel":
"""Gets the auth model to a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Raises:
NotImplementedError: This method is only available for the
SQLZenStore.
"""
raise NotImplementedError(
"This method is only designed for use"
" by the server endpoints. It is not designed"
" to be called from the client side."
)
get_flavor(self, flavor_id)
Get a stack component flavor by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
UUID |
The ID of the stack component flavor to get. |
required |
Returns:
| Type | Description |
|---|---|
FlavorResponseModel |
The stack component flavor. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_flavor(self, flavor_id: UUID) -> FlavorResponseModel:
"""Get a stack component flavor by ID.
Args:
flavor_id: The ID of the stack component flavor to get.
Returns:
The stack component flavor.
"""
return self._get_resource(
resource_id=flavor_id,
route=FLAVORS,
response_model=FlavorResponseModel,
)
get_or_create_run(self, pipeline_run)
Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned. Otherwise, a new run is created.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_run |
PipelineRunRequestModel |
The pipeline run to get or create. |
required |
Returns:
| Type | Description |
|---|---|
Tuple[zenml.models.pipeline_run_models.PipelineRunResponseModel, bool] |
The pipeline run, and a boolean indicating whether the run was created or not. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_or_create_run(
self, pipeline_run: PipelineRunRequestModel
) -> Tuple[PipelineRunResponseModel, bool]:
"""Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned.
Otherwise, a new run is created.
Args:
pipeline_run: The pipeline run to get or create.
Returns:
The pipeline run, and a boolean indicating whether the run was
created or not.
"""
return self._get_or_create_workspace_scoped_resource(
resource=pipeline_run,
route=RUNS,
response_model=PipelineRunResponseModel,
)
get_pipeline(self, pipeline_id)
Get a pipeline with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
ID of the pipeline. |
required |
Returns:
| Type | Description |
|---|---|
PipelineResponseModel |
The pipeline. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_pipeline(self, pipeline_id: UUID) -> PipelineResponseModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
"""
return self._get_resource(
resource_id=pipeline_id,
route=PIPELINES,
response_model=PipelineResponseModel,
)
get_role(self, role_name_or_id)
Gets a specific role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to get. |
required |
Returns:
| Type | Description |
|---|---|
RoleResponseModel |
The requested role. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleResponseModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
"""
return self._get_resource(
resource_id=role_name_or_id,
route=ROLES,
response_model=RoleResponseModel,
)
get_run(self, run_name_or_id)
Gets a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_name_or_id |
Union[uuid.UUID, str] |
The name or ID of the pipeline run to get. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The pipeline run. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run(
self, run_name_or_id: Union[UUID, str]
) -> PipelineRunResponseModel:
"""Gets a pipeline run.
Args:
run_name_or_id: The name or ID of the pipeline run to get.
Returns:
The pipeline run.
"""
return self._get_resource(
resource_id=run_name_or_id,
route=RUNS,
response_model=PipelineRunResponseModel,
)
get_run_step(self, step_run_id)
Get a step run by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_id |
UUID |
The ID of the step run to get. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The step run. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run_step(self, step_run_id: UUID) -> StepRunResponseModel:
"""Get a step run by ID.
Args:
step_run_id: The ID of the step run to get.
Returns:
The step run.
"""
return self._get_resource(
resource_id=step_run_id,
route=STEPS,
response_model=StepRunResponseModel,
)
get_schedule(self, schedule_id)
Get a schedule with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
ID of the schedule. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The schedule. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_schedule(self, schedule_id: UUID) -> ScheduleResponseModel:
"""Get a schedule with a given ID.
Args:
schedule_id: ID of the schedule.
Returns:
The schedule.
"""
return self._get_resource(
resource_id=schedule_id,
route=SCHEDULES,
response_model=ScheduleResponseModel,
)
get_stack(self, stack_id)
Get a stack by its unique ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack to get. |
required |
Returns:
| Type | Description |
|---|---|
StackResponseModel |
The stack with the given ID. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_stack(self, stack_id: UUID) -> StackResponseModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
"""
return self._get_resource(
resource_id=stack_id,
route=STACKS,
response_model=StackResponseModel,
)
get_stack_component(self, component_id)
Get a stack component by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to get. |
required |
Returns:
| Type | Description |
|---|---|
ComponentResponseModel |
The stack component. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_stack_component(
self, component_id: UUID
) -> ComponentResponseModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
"""
return self._get_resource(
resource_id=component_id,
route=STACK_COMPONENTS,
response_model=ComponentResponseModel,
)
get_store_info(self)
Get information about the server.
Returns:
| Type | Description |
|---|---|
ServerModel |
Information about the server. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_store_info(self) -> ServerModel:
"""Get information about the server.
Returns:
Information about the server.
"""
body = self.get(INFO)
return ServerModel.parse_obj(body)
get_team(self, team_name_or_id)
Gets a specific team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamResponseModel |
The requested team. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamResponseModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
"""
return self._get_resource(
resource_id=team_name_or_id,
route=TEAMS,
response_model=TeamResponseModel,
)
get_team_role_assignment(self, team_role_assignment_id)
Gets a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_id |
UUID |
ID of the role assignment to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamRoleAssignmentResponseModel |
The requested role assignment. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_team_role_assignment(
self, team_role_assignment_id: UUID
) -> TeamRoleAssignmentResponseModel:
"""Gets a specific role assignment.
Args:
team_role_assignment_id: ID of the role assignment to get.
Returns:
The requested role assignment.
"""
return self._get_resource(
resource_id=team_role_assignment_id,
route=TEAM_ROLE_ASSIGNMENTS,
response_model=TeamRoleAssignmentResponseModel,
)
get_user(self, user_name_or_id=None, include_private=False)
Gets a specific user, when no id is specified the active user is returned.
The include_private parameter is ignored here as it is handled
implicitly by the /current-user endpoint that is queried when no
user_name_or_id is set. Raises a KeyError in case a user with that id
does not exist.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
None |
include_private |
bool |
Whether to include private user information |
False |
Returns:
| Type | Description |
|---|---|
UserResponseModel |
The requested user, if it was found. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_user(
self,
user_name_or_id: Optional[Union[str, UUID]] = None,
include_private: bool = False,
) -> UserResponseModel:
"""Gets a specific user, when no id is specified the active user is returned.
The `include_private` parameter is ignored here as it is handled
implicitly by the /current-user endpoint that is queried when no
user_name_or_id is set. Raises a KeyError in case a user with that id
does not exist.
Args:
user_name_or_id: The name or ID of the user to get.
include_private: Whether to include private user information
Returns:
The requested user, if it was found.
"""
if user_name_or_id:
return self._get_resource(
resource_id=user_name_or_id,
route=USERS,
response_model=UserResponseModel,
)
else:
body = self.get(CURRENT_USER)
return UserResponseModel.parse_obj(body)
get_user_role_assignment(self, user_role_assignment_id)
Get an existing role assignment by name or ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_id |
UUID |
Name or ID of the role assignment to get. |
required |
Returns:
| Type | Description |
|---|---|
UserRoleAssignmentResponseModel |
The requested workspace. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_user_role_assignment(
self, user_role_assignment_id: UUID
) -> UserRoleAssignmentResponseModel:
"""Get an existing role assignment by name or ID.
Args:
user_role_assignment_id: Name or ID of the role assignment to get.
Returns:
The requested workspace.
"""
return self._get_resource(
resource_id=user_role_assignment_id,
route=USER_ROLE_ASSIGNMENTS,
response_model=UserRoleAssignmentResponseModel,
)
get_workspace(self, workspace_name_or_id)
Get an existing workspace by name or ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_name_or_id |
Union[uuid.UUID, str] |
Name or ID of the workspace to get. |
required |
Returns:
| Type | Description |
|---|---|
WorkspaceResponseModel |
The requested workspace. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_workspace(
self, workspace_name_or_id: Union[UUID, str]
) -> WorkspaceResponseModel:
"""Get an existing workspace by name or ID.
Args:
workspace_name_or_id: Name or ID of the workspace to get.
Returns:
The requested workspace.
"""
return self._get_resource(
resource_id=workspace_name_or_id,
route=WORKSPACES,
response_model=WorkspaceResponseModel,
)
list_artifacts(self, artifact_filter_model)
List all artifacts matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_filter_model |
ArtifactFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[ArtifactResponseModel] |
A list of all artifacts matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_artifacts(
self, artifact_filter_model: ArtifactFilterModel
) -> Page[ArtifactResponseModel]:
"""List all artifacts matching the given filter criteria.
Args:
artifact_filter_model: All filter parameters including pagination
params.
Returns:
A list of all artifacts matching the filter criteria.
"""
return self._list_paginated_resources(
route=ARTIFACTS,
response_model=ArtifactResponseModel,
filter_model=artifact_filter_model,
)
list_flavors(self, flavor_filter_model)
List all stack component flavors matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_filter_model |
FlavorFilterModel |
All filter parameters including pagination params |
required |
Returns:
| Type | Description |
|---|---|
Page[FlavorResponseModel] |
List of all the stack component flavors matching the given criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_flavors(
self, flavor_filter_model: FlavorFilterModel
) -> Page[FlavorResponseModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
flavor_filter_model: All filter parameters including pagination
params
Returns:
List of all the stack component flavors matching the given criteria.
"""
return self._list_paginated_resources(
route=FLAVORS,
response_model=FlavorResponseModel,
filter_model=flavor_filter_model,
)
list_pipelines(self, pipeline_filter_model)
List all pipelines matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_filter_model |
PipelineFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[PipelineResponseModel] |
A list of all pipelines matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_pipelines(
self, pipeline_filter_model: PipelineFilterModel
) -> Page[PipelineResponseModel]:
"""List all pipelines matching the given filter criteria.
Args:
pipeline_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipelines matching the filter criteria.
"""
return self._list_paginated_resources(
route=PIPELINES,
response_model=PipelineResponseModel,
filter_model=pipeline_filter_model,
)
list_roles(self, role_filter_model)
List all roles matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_filter_model |
RoleFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[RoleResponseModel] |
A list of all roles matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_roles(
self, role_filter_model: RoleFilterModel
) -> Page[RoleResponseModel]:
"""List all roles matching the given filter criteria.
Args:
role_filter_model: All filter parameters including pagination
params.
Returns:
A list of all roles matching the filter criteria.
"""
return self._list_paginated_resources(
route=ROLES,
response_model=RoleResponseModel,
filter_model=role_filter_model,
)
list_run_metadata(self, run_metadata_filter_model)
List run metadata.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_metadata_filter_model |
RunMetadataFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[RunMetadataResponseModel] |
The run metadata. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_run_metadata(
self,
run_metadata_filter_model: RunMetadataFilterModel,
) -> Page[RunMetadataResponseModel]:
"""List run metadata.
Args:
run_metadata_filter_model: All filter parameters including
pagination params.
Returns:
The run metadata.
"""
return self._list_paginated_resources(
route=RUN_METADATA,
response_model=RunMetadataResponseModel,
filter_model=run_metadata_filter_model,
)
list_run_steps(self, step_run_filter_model)
List all step runs matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_filter_model |
StepRunFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[StepRunResponseModel] |
A list of all step runs matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_run_steps(
self, step_run_filter_model: StepRunFilterModel
) -> Page[StepRunResponseModel]:
"""List all step runs matching the given filter criteria.
Args:
step_run_filter_model: All filter parameters including pagination
params.
Returns:
A list of all step runs matching the filter criteria.
"""
return self._list_paginated_resources(
route=STEPS,
response_model=StepRunResponseModel,
filter_model=step_run_filter_model,
)
list_runs(self, runs_filter_model)
List all pipeline runs matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
runs_filter_model |
PipelineRunFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[PipelineRunResponseModel] |
A list of all pipeline runs matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_runs(
self, runs_filter_model: PipelineRunFilterModel
) -> Page[PipelineRunResponseModel]:
"""List all pipeline runs matching the given filter criteria.
Args:
runs_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipeline runs matching the filter criteria.
"""
return self._list_paginated_resources(
route=RUNS,
response_model=PipelineRunResponseModel,
filter_model=runs_filter_model,
)
list_schedules(self, schedule_filter_model)
List all schedules in the workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_filter_model |
ScheduleFilterModel |
All filter parameters including pagination params |
required |
Returns:
| Type | Description |
|---|---|
Page[ScheduleResponseModel] |
A list of schedules. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_schedules(
self, schedule_filter_model: ScheduleFilterModel
) -> Page[ScheduleResponseModel]:
"""List all schedules in the workspace.
Args:
schedule_filter_model: All filter parameters including pagination
params
Returns:
A list of schedules.
"""
return self._list_paginated_resources(
route=SCHEDULES,
response_model=ScheduleResponseModel,
filter_model=schedule_filter_model,
)
list_stack_components(self, component_filter_model)
List all stack components matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_filter_model |
ComponentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[ComponentResponseModel] |
A list of all stack components matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_stack_components(
self, component_filter_model: ComponentFilterModel
) -> Page[ComponentResponseModel]:
"""List all stack components matching the given filter criteria.
Args:
component_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stack components matching the filter criteria.
"""
return self._list_paginated_resources(
route=STACK_COMPONENTS,
response_model=ComponentResponseModel,
filter_model=component_filter_model,
)
list_stacks(self, stack_filter_model)
List all stacks matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_filter_model |
StackFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[StackResponseModel] |
A list of all stacks matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_stacks(
self, stack_filter_model: StackFilterModel
) -> Page[StackResponseModel]:
"""List all stacks matching the given filter criteria.
Args:
stack_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stacks matching the filter criteria.
"""
return self._list_paginated_resources(
route=STACKS,
response_model=StackResponseModel,
filter_model=stack_filter_model,
)
list_team_role_assignments(self, team_role_assignment_filter_model)
List all roles assignments matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_filter_model |
TeamRoleAssignmentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[TeamRoleAssignmentResponseModel] |
A list of all roles assignments matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_team_role_assignments(
self, team_role_assignment_filter_model: TeamRoleAssignmentFilterModel
) -> Page[TeamRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
team_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
return self._list_paginated_resources(
route=TEAM_ROLE_ASSIGNMENTS,
response_model=TeamRoleAssignmentResponseModel,
filter_model=team_role_assignment_filter_model,
)
list_teams(self, team_filter_model)
List all teams matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_filter_model |
TeamFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[TeamResponseModel] |
A list of all teams matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_teams(
self, team_filter_model: TeamFilterModel
) -> Page[TeamResponseModel]:
"""List all teams matching the given filter criteria.
Args:
team_filter_model: All filter parameters including pagination
params.
Returns:
A list of all teams matching the filter criteria.
"""
return self._list_paginated_resources(
route=TEAMS,
response_model=TeamResponseModel,
filter_model=team_filter_model,
)
list_user_role_assignments(self, user_role_assignment_filter_model)
List all roles assignments matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_filter_model |
UserRoleAssignmentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[UserRoleAssignmentResponseModel] |
A list of all roles assignments matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_user_role_assignments(
self, user_role_assignment_filter_model: UserRoleAssignmentFilterModel
) -> Page[UserRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
user_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
return self._list_paginated_resources(
route=USER_ROLE_ASSIGNMENTS,
response_model=UserRoleAssignmentResponseModel,
filter_model=user_role_assignment_filter_model,
)
list_users(self, user_filter_model)
List all users.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_filter_model |
UserFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[UserResponseModel] |
A list of all users. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_users(
self, user_filter_model: UserFilterModel
) -> Page[UserResponseModel]:
"""List all users.
Args:
user_filter_model: All filter parameters including pagination
params.
Returns:
A list of all users.
"""
return self._list_paginated_resources(
route=USERS,
response_model=UserResponseModel,
filter_model=user_filter_model,
)
list_workspaces(self, workspace_filter_model)
List all workspace matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_filter_model |
WorkspaceFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[WorkspaceResponseModel] |
A list of all workspace matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_workspaces(
self, workspace_filter_model: WorkspaceFilterModel
) -> Page[WorkspaceResponseModel]:
"""List all workspace matching the given filter criteria.
Args:
workspace_filter_model: All filter parameters including pagination
params.
Returns:
A list of all workspace matching the filter criteria.
"""
return self._list_paginated_resources(
route=WORKSPACES,
response_model=WorkspaceResponseModel,
filter_model=workspace_filter_model,
)
post(self, path, body, params=None, **kwargs)
Make a POST request to the given endpoint path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the endpoint. |
required |
body |
BaseModel |
The body to send. |
required |
params |
Optional[Dict[str, Any]] |
The query parameters to pass to the endpoint. |
None |
kwargs |
Any |
Additional keyword arguments to pass to the request. |
{} |
Returns:
| Type | Description |
|---|---|
Union[Dict[str, Any], List[Any], str, int, float, bool] |
The response body. |
Source code in zenml/zen_stores/rest_zen_store.py
def post(
self,
path: str,
body: BaseModel,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a POST request to the given endpoint path.
Args:
path: The path to the endpoint.
body: The body to send.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending POST request to {path}...")
return self._request(
"POST",
self.url + API + VERSION_1 + path,
data=body.json(),
params=params,
**kwargs,
)
put(self, path, body, params=None, **kwargs)
Make a PUT request to the given endpoint path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the endpoint. |
required |
body |
BaseModel |
The body to send. |
required |
params |
Optional[Dict[str, Any]] |
The query parameters to pass to the endpoint. |
None |
kwargs |
Any |
Additional keyword arguments to pass to the request. |
{} |
Returns:
| Type | Description |
|---|---|
Union[Dict[str, Any], List[Any], str, int, float, bool] |
The response body. |
Source code in zenml/zen_stores/rest_zen_store.py
def put(
self,
path: str,
body: BaseModel,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a PUT request to the given endpoint path.
Args:
path: The path to the endpoint.
body: The body to send.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending PUT request to {path}...")
return self._request(
"PUT",
self.url + API + VERSION_1 + path,
data=body.json(exclude_unset=True),
params=params,
**kwargs,
)
update_pipeline(*args, **kwargs)
Updates a pipeline.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
The ID of the pipeline to be updated. |
required | |
pipeline_update |
The update to be applied. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated pipeline. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_role(*args, **kwargs)
Update an existing role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_id |
The ID of the role to be updated. |
required | |
role_update |
The update to be applied to the role. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated role. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_run(self, run_id, run_update)
Updates a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to update. |
required |
run_update |
PipelineRunUpdateModel |
The update to be applied to the pipeline run. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The updated pipeline run. |
Source code in zenml/zen_stores/rest_zen_store.py
def update_run(
self, run_id: UUID, run_update: PipelineRunUpdateModel
) -> PipelineRunResponseModel:
"""Updates a pipeline run.
Args:
run_id: The ID of the pipeline run to update.
run_update: The update to be applied to the pipeline run.
Returns:
The updated pipeline run.
"""
return self._update_resource(
resource_id=run_id,
resource_update=run_update,
response_model=PipelineRunResponseModel,
route=RUNS,
)
update_run_step(self, step_run_id, step_run_update)
Updates a step run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_id |
UUID |
The ID of the step to update. |
required |
step_run_update |
StepRunUpdateModel |
The update to be applied to the step. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The updated step run. |
Source code in zenml/zen_stores/rest_zen_store.py
def update_run_step(
self,
step_run_id: UUID,
step_run_update: StepRunUpdateModel,
) -> StepRunResponseModel:
"""Updates a step run.
Args:
step_run_id: The ID of the step to update.
step_run_update: The update to be applied to the step.
Returns:
The updated step run.
"""
return self._update_resource(
resource_id=step_run_id,
resource_update=step_run_update,
response_model=StepRunResponseModel,
route=STEPS,
)
update_schedule(self, schedule_id, schedule_update)
Updates a schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
The ID of the schedule to be updated. |
required |
schedule_update |
ScheduleUpdateModel |
The update to be applied. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The updated schedule. |
Source code in zenml/zen_stores/rest_zen_store.py
def update_schedule(
self,
schedule_id: UUID,
schedule_update: ScheduleUpdateModel,
) -> ScheduleResponseModel:
"""Updates a schedule.
Args:
schedule_id: The ID of the schedule to be updated.
schedule_update: The update to be applied.
Returns:
The updated schedule.
"""
return self._update_resource(
resource_id=schedule_id,
resource_update=schedule_update,
route=SCHEDULES,
response_model=ScheduleResponseModel,
)
update_stack(*args, **kwargs)
Update a stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
The ID of the stack update. |
required | |
stack_update |
The update request on the stack. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated stack. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_stack_component(*args, **kwargs)
Update an existing stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
The ID of the stack component to update. |
required | |
component_update |
The update to be applied to the stack component. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated stack component. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_team(*args, **kwargs)
Update an existing team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_id |
The ID of the team to be updated. |
required | |
team_update |
The update to be applied to the team. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated team. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_user(*args, **kwargs)
Updates an existing user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_id |
The id of the user to update. |
required | |
user_update |
The update to be applied to the user. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated user. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_workspace(*args, **kwargs)
Update an existing workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_id |
The ID of the workspace to be updated. |
required | |
workspace_update |
The update to be applied to the workspace. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated workspace. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
RestZenStoreConfiguration (StoreConfiguration)
pydantic-model
REST ZenML store configuration.
Attributes:
| Name | Type | Description |
|---|---|---|
username |
Optional[str] |
The username to use to connect to the Zen server. |
password |
Optional[str] |
The password to use to connect to the Zen server. |
verify_ssl |
Union[bool, str] |
Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use or the CA bundle value itself. |
http_timeout |
int |
The timeout to use for all requests. |
Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStoreConfiguration(StoreConfiguration):
"""REST ZenML store configuration.
Attributes:
username: The username to use to connect to the Zen server.
password: The password to use to connect to the Zen server.
verify_ssl: Either a boolean, in which case it controls whether we
verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use or the CA bundle value itself.
http_timeout: The timeout to use for all requests.
"""
type: StoreType = StoreType.REST
username: Optional[str] = None
password: Optional[str] = None
api_token: Optional[str] = None
verify_ssl: Union[bool, str] = True
http_timeout: int = DEFAULT_HTTP_TIMEOUT
@root_validator
def validate_credentials(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validates the credentials provided in the values dictionary.
Args:
values: A dictionary containing the values to be validated.
Raises:
ValueError: If neither api_token nor username is set.
Returns:
The values dictionary.
"""
# Check if the values dictionary contains either an api_token or a
# username as non-empty strings.
if values.get("api_token") or values.get("username"):
return values
else:
raise ValueError(
"Neither api_token nor username is set in the store config."
)
@validator("url")
def validate_url(cls, url: str) -> str:
"""Validates that the URL is a well-formed REST store URL.
Args:
url: The URL to be validated.
Returns:
The validated URL without trailing slashes.
Raises:
ValueError: If the URL is not a well-formed REST store URL.
"""
url = url.rstrip("/")
scheme = re.search("^([a-z0-9]+://)", url)
if scheme is None or scheme.group() not in ("https://", "http://"):
raise ValueError(
"Invalid URL for REST store: {url}. Should be in the form "
"https://hostname[:port] or http://hostname[:port]."
)
# When running inside a container, if the URL uses localhost, the
# target service will not be available. We try to replace localhost
# with one of the special Docker or K3D internal hostnames.
url = replace_localhost_with_internal_hostname(url)
return url
@validator("verify_ssl")
def validate_verify_ssl(
cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
"""Validates that the verify_ssl either points to a file or is a bool.
Args:
verify_ssl: The verify_ssl value to be validated.
Returns:
The validated verify_ssl value.
"""
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
if isinstance(verify_ssl, bool) or verify_ssl.startswith(
str(secret_folder)
):
return verify_ssl
if os.path.isfile(verify_ssl):
with open(verify_ssl, "r") as f:
verify_ssl = f.read()
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, "ca_bundle.pem")
with open(file_path, "w") as f:
f.write(verify_ssl)
file_path.chmod(0o600)
verify_ssl = str(file_path)
return verify_ssl
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
"""Check if a URL scheme is supported by this store.
Args:
url: The URL to check.
Returns:
True if the URL scheme is supported, False otherwise.
"""
return urlparse(url).scheme in ("http", "https")
def expand_certificates(self) -> None:
"""Expands the certificates in the verify_ssl field."""
# Load the certificate values back into the configuration
if isinstance(self.verify_ssl, str) and os.path.isfile(
self.verify_ssl
):
with open(self.verify_ssl, "r") as f:
self.verify_ssl = f.read()
@classmethod
def copy_configuration(
cls,
config: "StoreConfiguration",
config_path: str,
load_config_path: Optional[PurePath] = None,
) -> "StoreConfiguration":
"""Create a copy of the store config using a different path.
This method is used to create a copy of the store configuration that can
be loaded using a different configuration path or in the context of a
new environment, such as a container image.
The configuration files accompanying the store configuration are also
copied to the new configuration path (e.g. certificates etc.).
Args:
config: The store configuration to copy.
config_path: new path where the configuration copy will be loaded
from.
load_config_path: absolute path that will be used to load the copied
configuration. This can be set to a value different from
`config_path` if the configuration copy will be loaded from
a different environment, e.g. when the configuration is copied
to a container image and loaded using a different absolute path.
This will be reflected in the paths and URLs encoded in the
copied configuration.
Returns:
A new store configuration object that reflects the new configuration
path.
"""
assert isinstance(config, RestZenStoreConfiguration)
assert config.api_token is not None
config = config.copy(exclude={"username", "password"}, deep=True)
# Load the certificate values back into the configuration
config.expand_certificates()
return config
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the `verify_ssl` attribute can be expanded to the contents
# of the certificate file.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
Config
Pydantic configuration class.
Source code in zenml/zen_stores/rest_zen_store.py
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the `verify_ssl` attribute can be expanded to the contents
# of the certificate file.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
copy_configuration(config, config_path, load_config_path=None)
classmethod
Create a copy of the store config using a different path.
This method is used to create a copy of the store configuration that can be loaded using a different configuration path or in the context of a new environment, such as a container image.
The configuration files accompanying the store configuration are also copied to the new configuration path (e.g. certificates etc.).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The store configuration to copy. |
required |
config_path |
str |
new path where the configuration copy will be loaded from. |
required |
load_config_path |
Optional[pathlib.PurePath] |
absolute path that will be used to load the copied
configuration. This can be set to a value different from
|
None |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
A new store configuration object that reflects the new configuration path. |
Source code in zenml/zen_stores/rest_zen_store.py
@classmethod
def copy_configuration(
cls,
config: "StoreConfiguration",
config_path: str,
load_config_path: Optional[PurePath] = None,
) -> "StoreConfiguration":
"""Create a copy of the store config using a different path.
This method is used to create a copy of the store configuration that can
be loaded using a different configuration path or in the context of a
new environment, such as a container image.
The configuration files accompanying the store configuration are also
copied to the new configuration path (e.g. certificates etc.).
Args:
config: The store configuration to copy.
config_path: new path where the configuration copy will be loaded
from.
load_config_path: absolute path that will be used to load the copied
configuration. This can be set to a value different from
`config_path` if the configuration copy will be loaded from
a different environment, e.g. when the configuration is copied
to a container image and loaded using a different absolute path.
This will be reflected in the paths and URLs encoded in the
copied configuration.
Returns:
A new store configuration object that reflects the new configuration
path.
"""
assert isinstance(config, RestZenStoreConfiguration)
assert config.api_token is not None
config = config.copy(exclude={"username", "password"}, deep=True)
# Load the certificate values back into the configuration
config.expand_certificates()
return config
expand_certificates(self)
Expands the certificates in the verify_ssl field.
Source code in zenml/zen_stores/rest_zen_store.py
def expand_certificates(self) -> None:
"""Expands the certificates in the verify_ssl field."""
# Load the certificate values back into the configuration
if isinstance(self.verify_ssl, str) and os.path.isfile(
self.verify_ssl
):
with open(self.verify_ssl, "r") as f:
self.verify_ssl = f.read()
supports_url_scheme(url)
classmethod
Check if a URL scheme is supported by this store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The URL to check. |
required |
Returns:
| Type | Description |
|---|---|
bool |
True if the URL scheme is supported, False otherwise. |
Source code in zenml/zen_stores/rest_zen_store.py
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
"""Check if a URL scheme is supported by this store.
Args:
url: The URL to check.
Returns:
True if the URL scheme is supported, False otherwise.
"""
return urlparse(url).scheme in ("http", "https")
validate_credentials(values)
classmethod
Validates the credentials provided in the values dictionary.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
values |
Dict[str, Any] |
A dictionary containing the values to be validated. |
required |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If neither api_token nor username is set. |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The values dictionary. |
Source code in zenml/zen_stores/rest_zen_store.py
@root_validator
def validate_credentials(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validates the credentials provided in the values dictionary.
Args:
values: A dictionary containing the values to be validated.
Raises:
ValueError: If neither api_token nor username is set.
Returns:
The values dictionary.
"""
# Check if the values dictionary contains either an api_token or a
# username as non-empty strings.
if values.get("api_token") or values.get("username"):
return values
else:
raise ValueError(
"Neither api_token nor username is set in the store config."
)
validate_url(url)
classmethod
Validates that the URL is a well-formed REST store URL.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The URL to be validated. |
required |
Returns:
| Type | Description |
|---|---|
str |
The validated URL without trailing slashes. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the URL is not a well-formed REST store URL. |
Source code in zenml/zen_stores/rest_zen_store.py
@validator("url")
def validate_url(cls, url: str) -> str:
"""Validates that the URL is a well-formed REST store URL.
Args:
url: The URL to be validated.
Returns:
The validated URL without trailing slashes.
Raises:
ValueError: If the URL is not a well-formed REST store URL.
"""
url = url.rstrip("/")
scheme = re.search("^([a-z0-9]+://)", url)
if scheme is None or scheme.group() not in ("https://", "http://"):
raise ValueError(
"Invalid URL for REST store: {url}. Should be in the form "
"https://hostname[:port] or http://hostname[:port]."
)
# When running inside a container, if the URL uses localhost, the
# target service will not be available. We try to replace localhost
# with one of the special Docker or K3D internal hostnames.
url = replace_localhost_with_internal_hostname(url)
return url
validate_verify_ssl(verify_ssl)
classmethod
Validates that the verify_ssl either points to a file or is a bool.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
verify_ssl |
Union[bool, str] |
The verify_ssl value to be validated. |
required |
Returns:
| Type | Description |
|---|---|
Union[bool, str] |
The validated verify_ssl value. |
Source code in zenml/zen_stores/rest_zen_store.py
@validator("verify_ssl")
def validate_verify_ssl(
cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
"""Validates that the verify_ssl either points to a file or is a bool.
Args:
verify_ssl: The verify_ssl value to be validated.
Returns:
The validated verify_ssl value.
"""
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
if isinstance(verify_ssl, bool) or verify_ssl.startswith(
str(secret_folder)
):
return verify_ssl
if os.path.isfile(verify_ssl):
with open(verify_ssl, "r") as f:
verify_ssl = f.read()
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, "ca_bundle.pem")
with open(file_path, "w") as f:
f.write(verify_ssl)
file_path.chmod(0o600)
verify_ssl = str(file_path)
return verify_ssl
schemas
special
SQL Model Implementations.
artifact_schemas
SQLModel implementation of artifact tables.
ArtifactSchema (NamedSchema)
pydantic-model
SQL Model for artifacts of steps.
Source code in zenml/zen_stores/schemas/artifact_schemas.py
class ArtifactSchema(NamedSchema, table=True):
"""SQL Model for artifacts of steps."""
__tablename__ = "artifact"
artifact_store_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=StackComponentSchema.__tablename__,
source_column="artifact_store_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: Optional["UserSchema"] = Relationship(back_populates="artifacts")
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="artifacts")
type: ArtifactType
uri: str
materializer: str
data_type: str
run_metadata: List["RunMetadataSchema"] = Relationship(
back_populates="artifact",
sa_relationship_kwargs={"cascade": "delete"},
)
input_to_step_runs: List["StepRunInputArtifactSchema"] = Relationship(
back_populates="artifact",
sa_relationship_kwargs={"cascade": "delete"},
)
output_of_step_runs: List["StepRunOutputArtifactSchema"] = Relationship(
back_populates="artifact",
sa_relationship_kwargs={"cascade": "delete"},
)
@classmethod
def from_request(
cls, artifact_request: ArtifactRequestModel
) -> "ArtifactSchema":
"""Convert an `ArtifactRequestModel` to an `ArtifactSchema`.
Args:
artifact_request: The request model to convert.
Returns:
The converted schema.
"""
return cls(
name=artifact_request.name,
artifact_store_id=artifact_request.artifact_store_id,
workspace_id=artifact_request.workspace,
user_id=artifact_request.user,
type=artifact_request.type,
uri=artifact_request.uri,
materializer=artifact_request.materializer,
data_type=artifact_request.data_type,
)
def to_model(
self, producer_step_run_id: Optional[UUID]
) -> ArtifactResponseModel:
"""Convert an `ArtifactSchema` to an `ArtifactModel`.
Args:
producer_step_run_id: The ID of the step run that produced this
artifact.
Returns:
The created `ArtifactModel`.
"""
metadata = {
metadata_schema.key: metadata_schema.to_model()
for metadata_schema in self.run_metadata
}
return ArtifactResponseModel(
id=self.id,
name=self.name,
artifact_store_id=self.artifact_store_id,
user=self.user.to_model() if self.user else None,
workspace=self.workspace.to_model(),
type=self.type,
uri=self.uri,
materializer=self.materializer,
data_type=self.data_type,
created=self.created,
updated=self.updated,
producer_step_run_id=producer_step_run_id,
metadata=metadata,
)
from_request(artifact_request)
classmethod
Convert an ArtifactRequestModel to an ArtifactSchema.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_request |
ArtifactRequestModel |
The request model to convert. |
required |
Returns:
| Type | Description |
|---|---|
ArtifactSchema |
The converted schema. |
Source code in zenml/zen_stores/schemas/artifact_schemas.py
@classmethod
def from_request(
cls, artifact_request: ArtifactRequestModel
) -> "ArtifactSchema":
"""Convert an `ArtifactRequestModel` to an `ArtifactSchema`.
Args:
artifact_request: The request model to convert.
Returns:
The converted schema.
"""
return cls(
name=artifact_request.name,
artifact_store_id=artifact_request.artifact_store_id,
workspace_id=artifact_request.workspace,
user_id=artifact_request.user,
type=artifact_request.type,
uri=artifact_request.uri,
materializer=artifact_request.materializer,
data_type=artifact_request.data_type,
)
to_model(self, producer_step_run_id)
Convert an ArtifactSchema to an ArtifactModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
producer_step_run_id |
Optional[uuid.UUID] |
The ID of the step run that produced this artifact. |
required |
Returns:
| Type | Description |
|---|---|
ArtifactResponseModel |
The created |
Source code in zenml/zen_stores/schemas/artifact_schemas.py
def to_model(
self, producer_step_run_id: Optional[UUID]
) -> ArtifactResponseModel:
"""Convert an `ArtifactSchema` to an `ArtifactModel`.
Args:
producer_step_run_id: The ID of the step run that produced this
artifact.
Returns:
The created `ArtifactModel`.
"""
metadata = {
metadata_schema.key: metadata_schema.to_model()
for metadata_schema in self.run_metadata
}
return ArtifactResponseModel(
id=self.id,
name=self.name,
artifact_store_id=self.artifact_store_id,
user=self.user.to_model() if self.user else None,
workspace=self.workspace.to_model(),
type=self.type,
uri=self.uri,
materializer=self.materializer,
data_type=self.data_type,
created=self.created,
updated=self.updated,
producer_step_run_id=producer_step_run_id,
metadata=metadata,
)
base_schemas
Base classes for SQLModel schemas.
BaseSchema (SQLModel)
pydantic-model
Base SQL Model for ZenML entities.
Source code in zenml/zen_stores/schemas/base_schemas.py
class BaseSchema(SQLModel):
"""Base SQL Model for ZenML entities."""
id: UUID = Field(default_factory=uuid4, primary_key=True)
created: datetime = Field(default_factory=datetime.utcnow)
updated: datetime = Field(default_factory=datetime.utcnow)
NamedSchema (BaseSchema)
pydantic-model
Base Named SQL Model.
Source code in zenml/zen_stores/schemas/base_schemas.py
class NamedSchema(BaseSchema):
"""Base Named SQL Model."""
name: str
ShareableSchema (NamedSchema)
pydantic-model
Base shareable SQL Model.
Source code in zenml/zen_stores/schemas/base_schemas.py
class ShareableSchema(NamedSchema):
"""Base shareable SQL Model."""
is_shared: bool
component_schemas
SQL Model Implementations for Stack Components.
StackComponentSchema (ShareableSchema)
pydantic-model
SQL Model for stack components.
Source code in zenml/zen_stores/schemas/component_schemas.py
class StackComponentSchema(ShareableSchema, table=True):
"""SQL Model for stack components."""
__tablename__ = "stack_component"
type: StackComponentType
flavor: str
configuration: bytes
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="components")
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: Optional["UserSchema"] = Relationship(back_populates="components")
stacks: List["StackSchema"] = Relationship(
back_populates="components", link_model=StackCompositionSchema
)
schedules: List["ScheduleSchema"] = Relationship(
back_populates="orchestrator",
)
run_metadata: List["RunMetadataSchema"] = Relationship(
back_populates="stack_component",
)
def update(
self, component_update: ComponentUpdateModel
) -> "StackComponentSchema":
"""Updates a `StackSchema` from a `ComponentUpdateModel`.
Args:
component_update: The `ComponentUpdateModel` to update from.
Returns:
The updated `StackComponentSchema`.
"""
for field, value in component_update.dict(
exclude_unset=True, exclude={"workspace", "user"}
).items():
if field == "configuration":
self.configuration = base64.b64encode(
json.dumps(component_update.configuration).encode("utf-8")
)
else:
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
def to_model(
self,
) -> "ComponentResponseModel":
"""Creates a `ComponentModel` from an instance of a `StackSchema`.
Returns:
A `ComponentModel`
"""
return ComponentResponseModel(
id=self.id,
name=self.name,
type=self.type,
flavor=self.flavor,
user=self.user.to_model(True) if self.user else None,
workspace=self.workspace.to_model(),
is_shared=self.is_shared,
configuration=json.loads(
base64.b64decode(self.configuration).decode()
),
created=self.created,
updated=self.updated,
)
to_model(self)
Creates a ComponentModel from an instance of a StackSchema.
Returns:
| Type | Description |
|---|---|
ComponentResponseModel |
A |
Source code in zenml/zen_stores/schemas/component_schemas.py
def to_model(
self,
) -> "ComponentResponseModel":
"""Creates a `ComponentModel` from an instance of a `StackSchema`.
Returns:
A `ComponentModel`
"""
return ComponentResponseModel(
id=self.id,
name=self.name,
type=self.type,
flavor=self.flavor,
user=self.user.to_model(True) if self.user else None,
workspace=self.workspace.to_model(),
is_shared=self.is_shared,
configuration=json.loads(
base64.b64decode(self.configuration).decode()
),
created=self.created,
updated=self.updated,
)
update(self, component_update)
Updates a StackSchema from a ComponentUpdateModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_update |
ComponentUpdateModel |
The |
required |
Returns:
| Type | Description |
|---|---|
StackComponentSchema |
The updated |
Source code in zenml/zen_stores/schemas/component_schemas.py
def update(
self, component_update: ComponentUpdateModel
) -> "StackComponentSchema":
"""Updates a `StackSchema` from a `ComponentUpdateModel`.
Args:
component_update: The `ComponentUpdateModel` to update from.
Returns:
The updated `StackComponentSchema`.
"""
for field, value in component_update.dict(
exclude_unset=True, exclude={"workspace", "user"}
).items():
if field == "configuration":
self.configuration = base64.b64encode(
json.dumps(component_update.configuration).encode("utf-8")
)
else:
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
flavor_schemas
SQL Model Implementations for Flavors.
FlavorSchema (NamedSchema)
pydantic-model
SQL Model for flavors.
Attributes:
| Name | Type | Description |
|---|---|---|
type |
StackComponentType |
The type of the flavor. |
source |
str |
The source of the flavor. |
config_schema |
str |
The config schema of the flavor. |
integration |
Optional[str] |
The integration associated with the flavor. |
Source code in zenml/zen_stores/schemas/flavor_schemas.py
class FlavorSchema(NamedSchema, table=True):
"""SQL Model for flavors.
Attributes:
type: The type of the flavor.
source: The source of the flavor.
config_schema: The config schema of the flavor.
integration: The integration associated with the flavor.
"""
__tablename__ = "flavor"
type: StackComponentType
source: str
config_schema: str = Field(sa_column=Column(TEXT, nullable=False))
integration: Optional[str] = Field(default="")
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="flavors")
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: Optional["UserSchema"] = Relationship(back_populates="flavors")
def to_model(self) -> FlavorResponseModel:
"""Converts a flavor schema to a flavor model.
Returns:
The flavor model.
"""
return FlavorResponseModel(
id=self.id,
name=self.name,
type=self.type,
source=self.source,
config_schema=self.config_schema,
integration=self.integration,
user=self.user.to_model() if self.user else None,
workspace=self.workspace.to_model(),
created=self.created,
updated=self.updated,
)
to_model(self)
Converts a flavor schema to a flavor model.
Returns:
| Type | Description |
|---|---|
FlavorResponseModel |
The flavor model. |
Source code in zenml/zen_stores/schemas/flavor_schemas.py
def to_model(self) -> FlavorResponseModel:
"""Converts a flavor schema to a flavor model.
Returns:
The flavor model.
"""
return FlavorResponseModel(
id=self.id,
name=self.name,
type=self.type,
source=self.source,
config_schema=self.config_schema,
integration=self.integration,
user=self.user.to_model() if self.user else None,
workspace=self.workspace.to_model(),
created=self.created,
updated=self.updated,
)
identity_schemas
SQLModel implementation for the server information table.
IdentitySchema (SQLModel)
pydantic-model
SQL Model for the client/server identity.
Source code in zenml/zen_stores/schemas/identity_schemas.py
class IdentitySchema(SQLModel, table=True):
"""SQL Model for the client/server identity."""
__tablename__ = "identity"
id: UUID = Field(primary_key=True)
pipeline_run_schemas
SQLModel implementation of pipeline run tables.
PipelineRunSchema (NamedSchema)
pydantic-model
SQL Model for pipeline runs.
Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
class PipelineRunSchema(NamedSchema, table=True):
"""SQL Model for pipeline runs."""
__tablename__ = "pipeline_run"
stack_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=StackSchema.__tablename__,
source_column="stack_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
stack: "StackSchema" = Relationship(back_populates="runs")
pipeline_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=PipelineSchema.__tablename__,
source_column="pipeline_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
pipeline: "PipelineSchema" = Relationship(back_populates="runs")
schedule_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=ScheduleSchema.__tablename__,
source_column="schedule_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
schedule: ScheduleSchema = Relationship(back_populates="runs")
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: Optional["UserSchema"] = Relationship(back_populates="runs")
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="runs")
orchestrator_run_id: Optional[str] = Field(nullable=True)
enable_cache: Optional[bool] = Field(nullable=True)
enable_artifact_metadata: Optional[bool] = Field(nullable=True)
start_time: Optional[datetime] = Field(nullable=True)
end_time: Optional[datetime] = Field(nullable=True)
status: ExecutionStatus
pipeline_configuration: str = Field(sa_column=Column(TEXT, nullable=False))
num_steps: Optional[int]
client_version: str
server_version: Optional[str] = Field(nullable=True)
client_environment: Optional[str] = Field(
sa_column=Column(TEXT, nullable=True)
)
orchestrator_environment: Optional[str] = Field(
sa_column=Column(TEXT, nullable=True)
)
git_sha: Optional[str] = Field(nullable=True)
run_metadata: List["RunMetadataSchema"] = Relationship(
back_populates="pipeline_run",
sa_relationship_kwargs={"cascade": "delete"},
)
step_runs: List["StepRunSchema"] = Relationship(
back_populates="pipeline_run",
sa_relationship_kwargs={"cascade": "delete"},
)
@classmethod
def from_request(
cls, request: PipelineRunRequestModel
) -> "PipelineRunSchema":
"""Convert a `PipelineRunRequestModel` to a `PipelineRunSchema`.
Args:
request: The request to convert.
Returns:
The created `PipelineRunSchema`.
"""
configuration = json.dumps(request.pipeline_configuration)
client_environment = json.dumps(request.client_environment)
orchestrator_environment = json.dumps(request.orchestrator_environment)
return cls(
id=request.id,
name=request.name,
orchestrator_run_id=request.orchestrator_run_id,
stack_id=request.stack,
workspace_id=request.workspace,
user_id=request.user,
pipeline_id=request.pipeline,
schedule_id=request.schedule_id,
enable_cache=request.enable_cache,
start_time=request.start_time,
status=request.status,
pipeline_configuration=configuration,
num_steps=request.num_steps,
git_sha=request.git_sha,
client_version=request.client_version,
server_version=request.server_version,
client_environment=client_environment,
orchestrator_environment=orchestrator_environment,
)
def to_model(
self, _block_recursion: bool = False
) -> PipelineRunResponseModel:
"""Convert a `PipelineRunSchema` to a `PipelineRunResponseModel`.
Args:
_block_recursion: If other models should be recursively filled
Returns:
The created `PipelineRunResponseModel`.
"""
client_environment = (
json.loads(self.client_environment)
if self.client_environment
else {}
)
orchestrator_environment = (
json.loads(self.orchestrator_environment)
if self.orchestrator_environment
else {}
)
metadata = {
metadata_schema.key: metadata_schema.to_model()
for metadata_schema in self.run_metadata
}
if _block_recursion:
return PipelineRunResponseModel(
id=self.id,
name=self.name,
workspace=self.workspace.to_model(),
user=self.user.to_model(True) if self.user else None,
schedule_id=self.schedule_id,
orchestrator_run_id=self.orchestrator_run_id,
enable_cache=self.enable_cache,
enable_artifact_metadata=self.enable_artifact_metadata,
start_time=self.start_time,
end_time=self.end_time,
status=self.status,
pipeline_configuration=json.loads(self.pipeline_configuration),
num_steps=self.num_steps,
git_sha=self.git_sha,
client_version=self.client_version,
server_version=self.server_version,
client_environment=client_environment,
orchestrator_environment=orchestrator_environment,
created=self.created,
updated=self.updated,
metadata=metadata,
)
else:
return PipelineRunResponseModel(
id=self.id,
name=self.name,
stack=self.stack.to_model() if self.stack else None,
workspace=self.workspace.to_model(),
user=self.user.to_model(True) if self.user else None,
orchestrator_run_id=self.orchestrator_run_id,
enable_cache=self.enable_cache,
enable_artifact_metadata=self.enable_artifact_metadata,
start_time=self.start_time,
end_time=self.end_time,
status=self.status,
pipeline=(
self.pipeline.to_model(False) if self.pipeline else None
),
schedule_id=self.schedule_id,
pipeline_configuration=json.loads(self.pipeline_configuration),
num_steps=self.num_steps,
git_sha=self.git_sha,
client_version=self.client_version,
server_version=self.server_version,
client_environment=client_environment,
orchestrator_environment=orchestrator_environment,
created=self.created,
updated=self.updated,
metadata=metadata,
)
def update(
self, run_update: "PipelineRunUpdateModel"
) -> "PipelineRunSchema":
"""Update a `PipelineRunSchema` with a `PipelineRunUpdateModel`.
Args:
run_update: The `PipelineRunUpdateModel` to update with.
Returns:
The updated `PipelineRunSchema`.
"""
if run_update.status:
self.status = run_update.status
self.end_time = run_update.end_time
self.updated = datetime.utcnow()
return self
from_request(request)
classmethod
Convert a PipelineRunRequestModel to a PipelineRunSchema.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
request |
PipelineRunRequestModel |
The request to convert. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunSchema |
The created |
Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
@classmethod
def from_request(
cls, request: PipelineRunRequestModel
) -> "PipelineRunSchema":
"""Convert a `PipelineRunRequestModel` to a `PipelineRunSchema`.
Args:
request: The request to convert.
Returns:
The created `PipelineRunSchema`.
"""
configuration = json.dumps(request.pipeline_configuration)
client_environment = json.dumps(request.client_environment)
orchestrator_environment = json.dumps(request.orchestrator_environment)
return cls(
id=request.id,
name=request.name,
orchestrator_run_id=request.orchestrator_run_id,
stack_id=request.stack,
workspace_id=request.workspace,
user_id=request.user,
pipeline_id=request.pipeline,
schedule_id=request.schedule_id,
enable_cache=request.enable_cache,
start_time=request.start_time,
status=request.status,
pipeline_configuration=configuration,
num_steps=request.num_steps,
git_sha=request.git_sha,
client_version=request.client_version,
server_version=request.server_version,
client_environment=client_environment,
orchestrator_environment=orchestrator_environment,
)
to_model(self, _block_recursion=False)
Convert a PipelineRunSchema to a PipelineRunResponseModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
_block_recursion |
bool |
If other models should be recursively filled |
False |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The created |
Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
def to_model(
self, _block_recursion: bool = False
) -> PipelineRunResponseModel:
"""Convert a `PipelineRunSchema` to a `PipelineRunResponseModel`.
Args:
_block_recursion: If other models should be recursively filled
Returns:
The created `PipelineRunResponseModel`.
"""
client_environment = (
json.loads(self.client_environment)
if self.client_environment
else {}
)
orchestrator_environment = (
json.loads(self.orchestrator_environment)
if self.orchestrator_environment
else {}
)
metadata = {
metadata_schema.key: metadata_schema.to_model()
for metadata_schema in self.run_metadata
}
if _block_recursion:
return PipelineRunResponseModel(
id=self.id,
name=self.name,
workspace=self.workspace.to_model(),
user=self.user.to_model(True) if self.user else None,
schedule_id=self.schedule_id,
orchestrator_run_id=self.orchestrator_run_id,
enable_cache=self.enable_cache,
enable_artifact_metadata=self.enable_artifact_metadata,
start_time=self.start_time,
end_time=self.end_time,
status=self.status,
pipeline_configuration=json.loads(self.pipeline_configuration),
num_steps=self.num_steps,
git_sha=self.git_sha,
client_version=self.client_version,
server_version=self.server_version,
client_environment=client_environment,
orchestrator_environment=orchestrator_environment,
created=self.created,
updated=self.updated,
metadata=metadata,
)
else:
return PipelineRunResponseModel(
id=self.id,
name=self.name,
stack=self.stack.to_model() if self.stack else None,
workspace=self.workspace.to_model(),
user=self.user.to_model(True) if self.user else None,
orchestrator_run_id=self.orchestrator_run_id,
enable_cache=self.enable_cache,
enable_artifact_metadata=self.enable_artifact_metadata,
start_time=self.start_time,
end_time=self.end_time,
status=self.status,
pipeline=(
self.pipeline.to_model(False) if self.pipeline else None
),
schedule_id=self.schedule_id,
pipeline_configuration=json.loads(self.pipeline_configuration),
num_steps=self.num_steps,
git_sha=self.git_sha,
client_version=self.client_version,
server_version=self.server_version,
client_environment=client_environment,
orchestrator_environment=orchestrator_environment,
created=self.created,
updated=self.updated,
metadata=metadata,
)
update(self, run_update)
Update a PipelineRunSchema with a PipelineRunUpdateModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_update |
PipelineRunUpdateModel |
The |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunSchema |
The updated |
Source code in zenml/zen_stores/schemas/pipeline_run_schemas.py
def update(
self, run_update: "PipelineRunUpdateModel"
) -> "PipelineRunSchema":
"""Update a `PipelineRunSchema` with a `PipelineRunUpdateModel`.
Args:
run_update: The `PipelineRunUpdateModel` to update with.
Returns:
The updated `PipelineRunSchema`.
"""
if run_update.status:
self.status = run_update.status
self.end_time = run_update.end_time
self.updated = datetime.utcnow()
return self
pipeline_schemas
SQL Model Implementations for Pipelines and Pipeline Runs.
PipelineSchema (NamedSchema)
pydantic-model
SQL Model for pipelines.
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
class PipelineSchema(NamedSchema, table=True):
"""SQL Model for pipelines."""
__tablename__ = "pipeline"
docstring: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
spec: str = Field(sa_column=Column(TEXT, nullable=False))
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="pipelines")
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: Optional["UserSchema"] = Relationship(back_populates="pipelines")
schedules: List["ScheduleSchema"] = Relationship(
back_populates="pipeline",
)
runs: List["PipelineRunSchema"] = Relationship(
back_populates="pipeline", sa_relationship_kwargs={"cascade": "delete"}
)
def to_model(
self,
_block_recursion: bool = False,
last_x_runs: int = 3,
) -> "PipelineResponseModel":
"""Convert a `PipelineSchema` to a `PipelineModel`.
Args:
_block_recursion: Don't recursively fill attributes
last_x_runs: How many runs to use for the execution status
Returns:
The created PipelineModel.
"""
x_runs = self.runs[:last_x_runs]
status_last_x_runs = []
for run in x_runs:
status_last_x_runs.append(run.status)
if _block_recursion:
return PipelineResponseModel(
id=self.id,
name=self.name,
workspace=self.workspace.to_model(),
user=self.user.to_model(True) if self.user else None,
docstring=self.docstring,
spec=PipelineSpec.parse_raw(self.spec),
created=self.created,
updated=self.updated,
)
else:
return PipelineResponseModel(
id=self.id,
name=self.name,
workspace=self.workspace.to_model(),
user=self.user.to_model(True) if self.user else None,
runs=[r.to_model(_block_recursion=True) for r in x_runs],
docstring=self.docstring,
spec=PipelineSpec.parse_raw(self.spec),
created=self.created,
updated=self.updated,
status=status_last_x_runs,
)
def update(
self, pipeline_update: "PipelineUpdateModel"
) -> "PipelineSchema":
"""Update a `PipelineSchema` with a `PipelineUpdateModel`.
Args:
pipeline_update: The update model.
Returns:
The updated `PipelineSchema`.
"""
if pipeline_update.name:
self.name = pipeline_update.name
if pipeline_update.docstring:
self.docstring = pipeline_update.docstring
if pipeline_update.spec:
self.spec = pipeline_update.spec.json(sort_keys=True)
self.updated = datetime.utcnow()
return self
to_model(self, _block_recursion=False, last_x_runs=3)
Convert a PipelineSchema to a PipelineModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
_block_recursion |
bool |
Don't recursively fill attributes |
False |
last_x_runs |
int |
How many runs to use for the execution status |
3 |
Returns:
| Type | Description |
|---|---|
PipelineResponseModel |
The created PipelineModel. |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def to_model(
self,
_block_recursion: bool = False,
last_x_runs: int = 3,
) -> "PipelineResponseModel":
"""Convert a `PipelineSchema` to a `PipelineModel`.
Args:
_block_recursion: Don't recursively fill attributes
last_x_runs: How many runs to use for the execution status
Returns:
The created PipelineModel.
"""
x_runs = self.runs[:last_x_runs]
status_last_x_runs = []
for run in x_runs:
status_last_x_runs.append(run.status)
if _block_recursion:
return PipelineResponseModel(
id=self.id,
name=self.name,
workspace=self.workspace.to_model(),
user=self.user.to_model(True) if self.user else None,
docstring=self.docstring,
spec=PipelineSpec.parse_raw(self.spec),
created=self.created,
updated=self.updated,
)
else:
return PipelineResponseModel(
id=self.id,
name=self.name,
workspace=self.workspace.to_model(),
user=self.user.to_model(True) if self.user else None,
runs=[r.to_model(_block_recursion=True) for r in x_runs],
docstring=self.docstring,
spec=PipelineSpec.parse_raw(self.spec),
created=self.created,
updated=self.updated,
status=status_last_x_runs,
)
update(self, pipeline_update)
Update a PipelineSchema with a PipelineUpdateModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_update |
PipelineUpdateModel |
The update model. |
required |
Returns:
| Type | Description |
|---|---|
PipelineSchema |
The updated |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def update(
self, pipeline_update: "PipelineUpdateModel"
) -> "PipelineSchema":
"""Update a `PipelineSchema` with a `PipelineUpdateModel`.
Args:
pipeline_update: The update model.
Returns:
The updated `PipelineSchema`.
"""
if pipeline_update.name:
self.name = pipeline_update.name
if pipeline_update.docstring:
self.docstring = pipeline_update.docstring
if pipeline_update.spec:
self.spec = pipeline_update.spec.json(sort_keys=True)
self.updated = datetime.utcnow()
return self
role_schemas
SQLModel implementation of roles that can be assigned to users or teams.
RolePermissionSchema (SQLModel)
pydantic-model
SQL Model for team assignments.
Source code in zenml/zen_stores/schemas/role_schemas.py
class RolePermissionSchema(SQLModel, table=True):
"""SQL Model for team assignments."""
__tablename__ = "role_permission"
name: PermissionType = Field(primary_key=True)
role_id: UUID = build_foreign_key_field(
source=__tablename__,
target=RoleSchema.__tablename__,
source_column="role_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
roles: List["RoleSchema"] = Relationship(back_populates="permissions")
RoleSchema (NamedSchema)
pydantic-model
SQL Model for roles.
Source code in zenml/zen_stores/schemas/role_schemas.py
class RoleSchema(NamedSchema, table=True):
"""SQL Model for roles."""
__tablename__ = "role"
permissions: List["RolePermissionSchema"] = Relationship(
back_populates="roles", sa_relationship_kwargs={"cascade": "delete"}
)
user_role_assignments: List["UserRoleAssignmentSchema"] = Relationship(
back_populates="role", sa_relationship_kwargs={"cascade": "delete"}
)
team_role_assignments: List["TeamRoleAssignmentSchema"] = Relationship(
back_populates="role", sa_relationship_kwargs={"cascade": "delete"}
)
@classmethod
def from_request(cls, model: RoleRequestModel) -> "RoleSchema":
"""Create a `RoleSchema` from a `RoleResponseModel`.
Args:
model: The `RoleResponseModel` from which to create the schema.
Returns:
The created `RoleSchema`.
"""
return cls(name=model.name)
def update(self, role_update: RoleUpdateModel) -> "RoleSchema":
"""Update a `RoleSchema` from a `RoleUpdateModel`.
Args:
role_update: The `RoleUpdateModel` from which to update the schema.
Returns:
The updated `RoleSchema`.
"""
for field, value in role_update.dict(
exclude_unset=True, exclude={"permissions"}
).items():
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
def to_model(self) -> RoleResponseModel:
"""Convert a `RoleSchema` to a `RoleResponseModel`.
Returns:
The converted `RoleResponseModel`.
"""
return RoleResponseModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
permissions={PermissionType(p.name) for p in self.permissions},
)
from_request(model)
classmethod
Create a RoleSchema from a RoleResponseModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
RoleRequestModel |
The |
required |
Returns:
| Type | Description |
|---|---|
RoleSchema |
The created |
Source code in zenml/zen_stores/schemas/role_schemas.py
@classmethod
def from_request(cls, model: RoleRequestModel) -> "RoleSchema":
"""Create a `RoleSchema` from a `RoleResponseModel`.
Args:
model: The `RoleResponseModel` from which to create the schema.
Returns:
The created `RoleSchema`.
"""
return cls(name=model.name)
to_model(self)
Convert a RoleSchema to a RoleResponseModel.
Returns:
| Type | Description |
|---|---|
RoleResponseModel |
The converted |
Source code in zenml/zen_stores/schemas/role_schemas.py
def to_model(self) -> RoleResponseModel:
"""Convert a `RoleSchema` to a `RoleResponseModel`.
Returns:
The converted `RoleResponseModel`.
"""
return RoleResponseModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
permissions={PermissionType(p.name) for p in self.permissions},
)
update(self, role_update)
Update a RoleSchema from a RoleUpdateModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_update |
RoleUpdateModel |
The |
required |
Returns:
| Type | Description |
|---|---|
RoleSchema |
The updated |
Source code in zenml/zen_stores/schemas/role_schemas.py
def update(self, role_update: RoleUpdateModel) -> "RoleSchema":
"""Update a `RoleSchema` from a `RoleUpdateModel`.
Args:
role_update: The `RoleUpdateModel` from which to update the schema.
Returns:
The updated `RoleSchema`.
"""
for field, value in role_update.dict(
exclude_unset=True, exclude={"permissions"}
).items():
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
TeamRoleAssignmentSchema (BaseSchema)
pydantic-model
SQL Model for assigning roles to teams for a given workspace.
Source code in zenml/zen_stores/schemas/role_schemas.py
class TeamRoleAssignmentSchema(BaseSchema, table=True):
"""SQL Model for assigning roles to teams for a given workspace."""
__tablename__ = "team_role_assignment"
id: UUID = Field(primary_key=True, default_factory=uuid4)
role_id: UUID = build_foreign_key_field(
source=__tablename__,
target=RoleSchema.__tablename__,
source_column="role_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
team_id: UUID = build_foreign_key_field(
source=__tablename__,
target=TeamSchema.__tablename__,
source_column="team_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=True,
)
role: RoleSchema = Relationship(back_populates="team_role_assignments")
team: "TeamSchema" = Relationship(back_populates="assigned_roles")
workspace: Optional["WorkspaceSchema"] = Relationship(
back_populates="team_role_assignments"
)
@classmethod
def from_request(
cls, role_assignment: TeamRoleAssignmentRequestModel
) -> "TeamRoleAssignmentSchema":
"""Create a `TeamRoleAssignmentSchema` from a `RoleAssignmentRequestModel`.
Args:
role_assignment: The `RoleAssignmentRequestModel` from which to
create the schema.
Returns:
The created `TeamRoleAssignmentSchema`.
"""
return cls(
role_id=role_assignment.role,
team_id=role_assignment.team,
workspace_id=role_assignment.workspace,
)
def to_model(self) -> TeamRoleAssignmentResponseModel:
"""Convert a `TeamRoleAssignmentSchema` to a `RoleAssignmentModel`.
Returns:
The converted `RoleAssignmentModel`.
"""
return TeamRoleAssignmentResponseModel(
id=self.id,
workspace=self.workspace.to_model() if self.workspace else None,
team=self.team.to_model(_block_recursion=True),
role=self.role.to_model(),
created=self.created,
updated=self.updated,
)
from_request(role_assignment)
classmethod
Create a TeamRoleAssignmentSchema from a RoleAssignmentRequestModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_assignment |
TeamRoleAssignmentRequestModel |
The |
required |
Returns:
| Type | Description |
|---|---|
TeamRoleAssignmentSchema |
The created |
Source code in zenml/zen_stores/schemas/role_schemas.py
@classmethod
def from_request(
cls, role_assignment: TeamRoleAssignmentRequestModel
) -> "TeamRoleAssignmentSchema":
"""Create a `TeamRoleAssignmentSchema` from a `RoleAssignmentRequestModel`.
Args:
role_assignment: The `RoleAssignmentRequestModel` from which to
create the schema.
Returns:
The created `TeamRoleAssignmentSchema`.
"""
return cls(
role_id=role_assignment.role,
team_id=role_assignment.team,
workspace_id=role_assignment.workspace,
)
to_model(self)
Convert a TeamRoleAssignmentSchema to a RoleAssignmentModel.
Returns:
| Type | Description |
|---|---|
TeamRoleAssignmentResponseModel |
The converted |
Source code in zenml/zen_stores/schemas/role_schemas.py
def to_model(self) -> TeamRoleAssignmentResponseModel:
"""Convert a `TeamRoleAssignmentSchema` to a `RoleAssignmentModel`.
Returns:
The converted `RoleAssignmentModel`.
"""
return TeamRoleAssignmentResponseModel(
id=self.id,
workspace=self.workspace.to_model() if self.workspace else None,
team=self.team.to_model(_block_recursion=True),
role=self.role.to_model(),
created=self.created,
updated=self.updated,
)
UserRoleAssignmentSchema (BaseSchema)
pydantic-model
SQL Model for assigning roles to users for a given workspace.
Source code in zenml/zen_stores/schemas/role_schemas.py
class UserRoleAssignmentSchema(BaseSchema, table=True):
"""SQL Model for assigning roles to users for a given workspace."""
__tablename__ = "user_role_assignment"
id: UUID = Field(primary_key=True, default_factory=uuid4)
role_id: UUID = build_foreign_key_field(
source=__tablename__,
target=RoleSchema.__tablename__,
source_column="role_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
user_id: UUID = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=True,
)
role: RoleSchema = Relationship(back_populates="user_role_assignments")
user: Optional["UserSchema"] = Relationship(
back_populates="assigned_roles"
)
workspace: Optional["WorkspaceSchema"] = Relationship(
back_populates="user_role_assignments"
)
@classmethod
def from_request(
cls, role_assignment: UserRoleAssignmentRequestModel
) -> "UserRoleAssignmentSchema":
"""Create a `UserRoleAssignmentSchema` from a `RoleAssignmentRequestModel`.
Args:
role_assignment: The `RoleAssignmentRequestModel` from which to
create the schema.
Returns:
The created `UserRoleAssignmentSchema`.
"""
return cls(
role_id=role_assignment.role,
user_id=role_assignment.user,
workspace_id=role_assignment.workspace,
)
def to_model(self) -> UserRoleAssignmentResponseModel:
"""Convert a `UserRoleAssignmentSchema` to a `RoleAssignmentModel`.
Returns:
The converted `RoleAssignmentModel`.
"""
return UserRoleAssignmentResponseModel(
id=self.id,
workspace=self.workspace.to_model() if self.workspace else None,
user=self.user.to_model(_block_recursion=True)
if self.user
else None,
role=self.role.to_model(),
created=self.created,
updated=self.updated,
)
from_request(role_assignment)
classmethod
Create a UserRoleAssignmentSchema from a RoleAssignmentRequestModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_assignment |
UserRoleAssignmentRequestModel |
The |
required |
Returns:
| Type | Description |
|---|---|
UserRoleAssignmentSchema |
The created |
Source code in zenml/zen_stores/schemas/role_schemas.py
@classmethod
def from_request(
cls, role_assignment: UserRoleAssignmentRequestModel
) -> "UserRoleAssignmentSchema":
"""Create a `UserRoleAssignmentSchema` from a `RoleAssignmentRequestModel`.
Args:
role_assignment: The `RoleAssignmentRequestModel` from which to
create the schema.
Returns:
The created `UserRoleAssignmentSchema`.
"""
return cls(
role_id=role_assignment.role,
user_id=role_assignment.user,
workspace_id=role_assignment.workspace,
)
to_model(self)
Convert a UserRoleAssignmentSchema to a RoleAssignmentModel.
Returns:
| Type | Description |
|---|---|
UserRoleAssignmentResponseModel |
The converted |
Source code in zenml/zen_stores/schemas/role_schemas.py
def to_model(self) -> UserRoleAssignmentResponseModel:
"""Convert a `UserRoleAssignmentSchema` to a `RoleAssignmentModel`.
Returns:
The converted `RoleAssignmentModel`.
"""
return UserRoleAssignmentResponseModel(
id=self.id,
workspace=self.workspace.to_model() if self.workspace else None,
user=self.user.to_model(_block_recursion=True)
if self.user
else None,
role=self.role.to_model(),
created=self.created,
updated=self.updated,
)
run_metadata_schemas
SQLModel implementation of pipeline run metadata tables.
RunMetadataSchema (BaseSchema)
pydantic-model
SQL Model for run metadata.
Source code in zenml/zen_stores/schemas/run_metadata_schemas.py
class RunMetadataSchema(BaseSchema, table=True):
"""SQL Model for run metadata."""
__tablename__ = "run_metadata"
pipeline_run_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=PipelineRunSchema.__tablename__,
source_column="pipeline_run_id",
target_column="id",
ondelete="CASCADE",
nullable=True,
)
pipeline_run: Optional["PipelineRunSchema"] = Relationship(
back_populates="run_metadata"
)
step_run_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=StepRunSchema.__tablename__,
source_column="step_run_id",
target_column="id",
ondelete="CASCADE",
nullable=True,
)
step_run: Optional["StepRunSchema"] = Relationship(
back_populates="run_metadata"
)
artifact_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=ArtifactSchema.__tablename__,
source_column="artifact_id",
target_column="id",
ondelete="CASCADE",
nullable=True,
)
artifact: Optional["ArtifactSchema"] = Relationship(
back_populates="run_metadata"
)
stack_component_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=StackComponentSchema.__tablename__,
source_column="stack_component_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
stack_component: Optional["StackComponentSchema"] = Relationship(
back_populates="run_metadata"
)
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: Optional["UserSchema"] = Relationship(back_populates="run_metadata")
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="run_metadata")
key: str
value: str = Field(sa_column=Column(TEXT, nullable=False))
type: MetadataTypeEnum
def to_model(self) -> "RunMetadataResponseModel":
"""Convert a `RunMetadataSchema` to a `RunMetadataResponseModel`.
Returns:
The created `RunMetadataResponseModel`.
"""
return RunMetadataResponseModel(
id=self.id,
pipeline_run_id=self.pipeline_run_id,
step_run_id=self.step_run_id,
artifact_id=self.artifact_id,
stack_component_id=self.stack_component_id,
key=self.key,
value=json.loads(self.value),
type=self.type,
workspace=self.workspace.to_model(),
user=self.user.to_model() if self.user else None,
created=self.created,
updated=self.updated,
)
@classmethod
def from_request(
cls, request: "RunMetadataRequestModel"
) -> "RunMetadataSchema":
"""Create a `RunMetadataSchema` from a `RunMetadataRequestModel`.
Args:
request: The request model to create the schema from.
Returns:
The created `RunMetadataSchema`.
"""
return cls(
workspace_id=request.workspace,
user_id=request.user,
pipeline_run_id=request.pipeline_run_id,
step_run_id=request.step_run_id,
artifact_id=request.artifact_id,
stack_component_id=request.stack_component_id,
key=request.key,
value=json.dumps(request.value),
type=request.type,
)
from_request(request)
classmethod
Create a RunMetadataSchema from a RunMetadataRequestModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
request |
RunMetadataRequestModel |
The request model to create the schema from. |
required |
Returns:
| Type | Description |
|---|---|
RunMetadataSchema |
The created |
Source code in zenml/zen_stores/schemas/run_metadata_schemas.py
@classmethod
def from_request(
cls, request: "RunMetadataRequestModel"
) -> "RunMetadataSchema":
"""Create a `RunMetadataSchema` from a `RunMetadataRequestModel`.
Args:
request: The request model to create the schema from.
Returns:
The created `RunMetadataSchema`.
"""
return cls(
workspace_id=request.workspace,
user_id=request.user,
pipeline_run_id=request.pipeline_run_id,
step_run_id=request.step_run_id,
artifact_id=request.artifact_id,
stack_component_id=request.stack_component_id,
key=request.key,
value=json.dumps(request.value),
type=request.type,
)
to_model(self)
Convert a RunMetadataSchema to a RunMetadataResponseModel.
Returns:
| Type | Description |
|---|---|
RunMetadataResponseModel |
The created |
Source code in zenml/zen_stores/schemas/run_metadata_schemas.py
def to_model(self) -> "RunMetadataResponseModel":
"""Convert a `RunMetadataSchema` to a `RunMetadataResponseModel`.
Returns:
The created `RunMetadataResponseModel`.
"""
return RunMetadataResponseModel(
id=self.id,
pipeline_run_id=self.pipeline_run_id,
step_run_id=self.step_run_id,
artifact_id=self.artifact_id,
stack_component_id=self.stack_component_id,
key=self.key,
value=json.loads(self.value),
type=self.type,
workspace=self.workspace.to_model(),
user=self.user.to_model() if self.user else None,
created=self.created,
updated=self.updated,
)
schedule_schema
SQL Model Implementations for Pipeline Schedules.
ScheduleSchema (NamedSchema)
pydantic-model
SQL Model for schedules.
Source code in zenml/zen_stores/schemas/schedule_schema.py
class ScheduleSchema(NamedSchema, table=True):
"""SQL Model for schedules."""
__tablename__ = "schedule"
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="schedules")
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: "UserSchema" = Relationship(back_populates="schedules")
pipeline_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=PipelineSchema.__tablename__,
source_column="pipeline_id",
target_column="id",
ondelete="CASCADE",
nullable=True,
)
pipeline: "PipelineSchema" = Relationship(back_populates="schedules")
orchestrator_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=StackComponentSchema.__tablename__,
source_column="orchestrator_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
orchestrator: "StackComponentSchema" = Relationship(
back_populates="schedules"
)
active: bool
cron_expression: Optional[str] = Field(nullable=True)
start_time: Optional[datetime] = Field(nullable=True)
end_time: Optional[datetime] = Field(nullable=True)
interval_second: Optional[float] = Field(nullable=True)
catchup: bool
runs: List["PipelineRunSchema"] = Relationship(
back_populates="schedule",
)
@classmethod
def from_create_model(
cls, model: ScheduleRequestModel
) -> "ScheduleSchema":
"""Create a `ScheduleSchema` from a `ScheduleRequestModel`.
Args:
model: The `ScheduleRequestModel` to create the schema from.
Returns:
The created `ScheduleSchema`.
"""
if model.interval_second is not None:
interval_second = model.interval_second.total_seconds()
else:
interval_second = None
return cls(
name=model.name,
workspace_id=model.workspace,
user_id=model.user,
pipeline_id=model.pipeline_id,
orchestrator_id=model.orchestrator_id,
active=model.active,
cron_expression=model.cron_expression,
start_time=model.start_time,
end_time=model.end_time,
interval_second=interval_second,
catchup=model.catchup,
)
def from_update_model(
self, model: ScheduleUpdateModel
) -> "ScheduleSchema":
"""Update a `ScheduleSchema` from a `ScheduleUpdateModel`.
Args:
model: The `ScheduleUpdateModel` to update the schema from.
Returns:
The updated `ScheduleSchema`.
"""
if model.name is not None:
self.name = model.name
if model.active is not None:
self.active = model.active
if model.cron_expression is not None:
self.cron_expression = model.cron_expression
if model.start_time is not None:
self.start_time = model.start_time
if model.end_time is not None:
self.end_time = model.end_time
if model.interval_second is not None:
self.interval_second = model.interval_second.total_seconds()
if model.catchup is not None:
self.catchup = model.catchup
self.updated = datetime.utcnow()
return self
def to_model(self) -> ScheduleResponseModel:
"""Convert a `ScheduleSchema` to a `ScheduleResponseModel`.
Returns:
The created `ScheduleResponseModel`.
"""
if self.interval_second is not None:
interval_second = timedelta(seconds=self.interval_second)
else:
interval_second = None
return ScheduleResponseModel(
id=self.id,
name=self.name,
workspace=self.workspace.to_model(),
user=self.user.to_model(),
pipeline_id=self.pipeline_id,
orchestrator_id=self.orchestrator_id,
active=self.active,
cron_expression=self.cron_expression,
start_time=self.start_time,
end_time=self.end_time,
interval_second=interval_second,
catchup=self.catchup,
created=self.created,
updated=self.updated,
)
from_create_model(model)
classmethod
Create a ScheduleSchema from a ScheduleRequestModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
ScheduleRequestModel |
The |
required |
Returns:
| Type | Description |
|---|---|
ScheduleSchema |
The created |
Source code in zenml/zen_stores/schemas/schedule_schema.py
@classmethod
def from_create_model(
cls, model: ScheduleRequestModel
) -> "ScheduleSchema":
"""Create a `ScheduleSchema` from a `ScheduleRequestModel`.
Args:
model: The `ScheduleRequestModel` to create the schema from.
Returns:
The created `ScheduleSchema`.
"""
if model.interval_second is not None:
interval_second = model.interval_second.total_seconds()
else:
interval_second = None
return cls(
name=model.name,
workspace_id=model.workspace,
user_id=model.user,
pipeline_id=model.pipeline_id,
orchestrator_id=model.orchestrator_id,
active=model.active,
cron_expression=model.cron_expression,
start_time=model.start_time,
end_time=model.end_time,
interval_second=interval_second,
catchup=model.catchup,
)
from_update_model(self, model)
Update a ScheduleSchema from a ScheduleUpdateModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
ScheduleUpdateModel |
The |
required |
Returns:
| Type | Description |
|---|---|
ScheduleSchema |
The updated |
Source code in zenml/zen_stores/schemas/schedule_schema.py
def from_update_model(
self, model: ScheduleUpdateModel
) -> "ScheduleSchema":
"""Update a `ScheduleSchema` from a `ScheduleUpdateModel`.
Args:
model: The `ScheduleUpdateModel` to update the schema from.
Returns:
The updated `ScheduleSchema`.
"""
if model.name is not None:
self.name = model.name
if model.active is not None:
self.active = model.active
if model.cron_expression is not None:
self.cron_expression = model.cron_expression
if model.start_time is not None:
self.start_time = model.start_time
if model.end_time is not None:
self.end_time = model.end_time
if model.interval_second is not None:
self.interval_second = model.interval_second.total_seconds()
if model.catchup is not None:
self.catchup = model.catchup
self.updated = datetime.utcnow()
return self
to_model(self)
Convert a ScheduleSchema to a ScheduleResponseModel.
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The created |
Source code in zenml/zen_stores/schemas/schedule_schema.py
def to_model(self) -> ScheduleResponseModel:
"""Convert a `ScheduleSchema` to a `ScheduleResponseModel`.
Returns:
The created `ScheduleResponseModel`.
"""
if self.interval_second is not None:
interval_second = timedelta(seconds=self.interval_second)
else:
interval_second = None
return ScheduleResponseModel(
id=self.id,
name=self.name,
workspace=self.workspace.to_model(),
user=self.user.to_model(),
pipeline_id=self.pipeline_id,
orchestrator_id=self.orchestrator_id,
active=self.active,
cron_expression=self.cron_expression,
start_time=self.start_time,
end_time=self.end_time,
interval_second=interval_second,
catchup=self.catchup,
created=self.created,
updated=self.updated,
)
schema_utils
Utility functions for SQLModel schemas.
build_foreign_key_field(source, target, source_column, target_column, ondelete, nullable, **sa_column_kwargs)
Build a SQLModel foreign key field.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
source |
str |
Source table name. |
required |
target |
str |
Target table name. |
required |
source_column |
str |
Source column name. |
required |
target_column |
str |
Target column name. |
required |
ondelete |
str |
On delete behavior. |
required |
nullable |
bool |
Whether the field is nullable. |
required |
**sa_column_kwargs |
Any |
Keyword arguments for the SQLAlchemy column. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
SQLModel foreign key field. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the ondelete and nullable arguments are not compatible. |
Source code in zenml/zen_stores/schemas/schema_utils.py
def build_foreign_key_field(
source: str,
target: str,
source_column: str,
target_column: str,
ondelete: str,
nullable: bool,
**sa_column_kwargs: Any,
) -> Any:
"""Build a SQLModel foreign key field.
Args:
source: Source table name.
target: Target table name.
source_column: Source column name.
target_column: Target column name.
ondelete: On delete behavior.
nullable: Whether the field is nullable.
**sa_column_kwargs: Keyword arguments for the SQLAlchemy column.
Returns:
SQLModel foreign key field.
Raises:
ValueError: If the ondelete and nullable arguments are not compatible.
"""
if not nullable and ondelete == "SET NULL":
raise ValueError(
"Cannot set ondelete to SET NULL if the field is not nullable."
)
constraint_name = foreign_key_constraint_name(
source=source,
target=target,
source_column=source_column,
)
return Field(
sa_column=Column(
ForeignKey(
f"{target}.{target_column}",
name=constraint_name,
ondelete=ondelete,
),
nullable=nullable,
**sa_column_kwargs,
),
)
foreign_key_constraint_name(source, target, source_column)
Defines the name of a foreign key constraint.
For simplicity, we use the naming convention used by alembic here: https://alembic.sqlalchemy.org/en/latest/batch.html#dropping-unnamed-or-named-foreign-key-constraints.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
source |
str |
Source table name. |
required |
target |
str |
Target table name. |
required |
source_column |
str |
Source column name. |
required |
Returns:
| Type | Description |
|---|---|
str |
Name of the foreign key constraint. |
Source code in zenml/zen_stores/schemas/schema_utils.py
def foreign_key_constraint_name(
source: str, target: str, source_column: str
) -> str:
"""Defines the name of a foreign key constraint.
For simplicity, we use the naming convention used by alembic here:
https://alembic.sqlalchemy.org/en/latest/batch.html#dropping-unnamed-or-named-foreign-key-constraints.
Args:
source: Source table name.
target: Target table name.
source_column: Source column name.
Returns:
Name of the foreign key constraint.
"""
return f"fk_{source}_{source_column}_{target}"
stack_schemas
SQL Model Implementations for Stacks.
StackCompositionSchema (SQLModel)
pydantic-model
SQL Model for stack definitions.
Join table between Stacks and StackComponents.
Source code in zenml/zen_stores/schemas/stack_schemas.py
class StackCompositionSchema(SQLModel, table=True):
"""SQL Model for stack definitions.
Join table between Stacks and StackComponents.
"""
__tablename__ = "stack_composition"
stack_id: UUID = build_foreign_key_field(
source=__tablename__,
target="stack", # TODO: how to reference `StackSchema.__tablename__`?
source_column="stack_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
component_id: UUID = build_foreign_key_field(
source=__tablename__,
target="stack_component", # TODO: how to reference `StackComponentSchema.__tablename__`?
source_column="component_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
StackSchema (ShareableSchema)
pydantic-model
SQL Model for stacks.
Source code in zenml/zen_stores/schemas/stack_schemas.py
class StackSchema(ShareableSchema, table=True):
"""SQL Model for stacks."""
__tablename__ = "stack"
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="stacks")
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: Optional["UserSchema"] = Relationship(back_populates="stacks")
components: List["StackComponentSchema"] = Relationship(
back_populates="stacks",
link_model=StackCompositionSchema,
)
runs: List["PipelineRunSchema"] = Relationship(back_populates="stack")
def update(
self,
stack_update: "StackUpdateModel",
components: List["StackComponentSchema"],
) -> "StackSchema":
"""Updates a stack schema with a stack update model.
Args:
stack_update: `StackUpdateModel` to update the stack with.
components: List of `StackComponentSchema` to update the stack with.
Returns:
The updated StackSchema.
"""
for field, value in stack_update.dict(exclude_unset=True).items():
if field == "components":
self.components = components
elif field == "user":
assert self.user_id == value
elif field == "workspace":
assert self.workspace_id == value
else:
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
def to_model(self) -> "StackResponseModel":
"""Converts the schema to a model.
Returns:
The converted model.
"""
return StackResponseModel(
id=self.id,
name=self.name,
user=self.user.to_model(True) if self.user else None,
workspace=self.workspace.to_model(),
is_shared=self.is_shared,
components={c.type: [c.to_model()] for c in self.components},
created=self.created,
updated=self.updated,
)
to_model(self)
Converts the schema to a model.
Returns:
| Type | Description |
|---|---|
StackResponseModel |
The converted model. |
Source code in zenml/zen_stores/schemas/stack_schemas.py
def to_model(self) -> "StackResponseModel":
"""Converts the schema to a model.
Returns:
The converted model.
"""
return StackResponseModel(
id=self.id,
name=self.name,
user=self.user.to_model(True) if self.user else None,
workspace=self.workspace.to_model(),
is_shared=self.is_shared,
components={c.type: [c.to_model()] for c in self.components},
created=self.created,
updated=self.updated,
)
update(self, stack_update, components)
Updates a stack schema with a stack update model.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_update |
StackUpdateModel |
|
required |
components |
List[StackComponentSchema] |
List of |
required |
Returns:
| Type | Description |
|---|---|
StackSchema |
The updated StackSchema. |
Source code in zenml/zen_stores/schemas/stack_schemas.py
def update(
self,
stack_update: "StackUpdateModel",
components: List["StackComponentSchema"],
) -> "StackSchema":
"""Updates a stack schema with a stack update model.
Args:
stack_update: `StackUpdateModel` to update the stack with.
components: List of `StackComponentSchema` to update the stack with.
Returns:
The updated StackSchema.
"""
for field, value in stack_update.dict(exclude_unset=True).items():
if field == "components":
self.components = components
elif field == "user":
assert self.user_id == value
elif field == "workspace":
assert self.workspace_id == value
else:
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
step_run_schemas
SQLModel implementation of step run tables.
StepRunInputArtifactSchema (SQLModel)
pydantic-model
SQL Model that defines which artifacts are inputs to which step.
Source code in zenml/zen_stores/schemas/step_run_schemas.py
class StepRunInputArtifactSchema(SQLModel, table=True):
"""SQL Model that defines which artifacts are inputs to which step."""
__tablename__ = "step_run_input_artifact"
step_id: UUID = build_foreign_key_field(
source=__tablename__,
target=StepRunSchema.__tablename__,
source_column="step_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
step_run: StepRunSchema = Relationship(back_populates="input_artifacts")
artifact_id: UUID = build_foreign_key_field(
source=__tablename__,
target=ArtifactSchema.__tablename__,
source_column="artifact_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
artifact: ArtifactSchema = Relationship(
back_populates="input_to_step_runs"
)
name: str
StepRunOutputArtifactSchema (SQLModel)
pydantic-model
SQL Model that defines which artifacts are outputs of which step.
Source code in zenml/zen_stores/schemas/step_run_schemas.py
class StepRunOutputArtifactSchema(SQLModel, table=True):
"""SQL Model that defines which artifacts are outputs of which step."""
__tablename__ = "step_run_output_artifact"
step_id: UUID = build_foreign_key_field(
source=__tablename__,
target=StepRunSchema.__tablename__,
source_column="step_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
step_run: StepRunSchema = Relationship(back_populates="output_artifacts")
artifact_id: UUID = build_foreign_key_field(
source=__tablename__,
target=ArtifactSchema.__tablename__,
source_column="artifact_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
artifact: ArtifactSchema = Relationship(
back_populates="output_of_step_runs"
)
name: str
StepRunParentsSchema (SQLModel)
pydantic-model
SQL Model that defines the order of steps.
Source code in zenml/zen_stores/schemas/step_run_schemas.py
class StepRunParentsSchema(SQLModel, table=True):
"""SQL Model that defines the order of steps."""
__tablename__ = "step_run_parents"
parent_id: UUID = build_foreign_key_field(
source=__tablename__,
target=StepRunSchema.__tablename__,
source_column="parent_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
parent: StepRunSchema = Relationship(
back_populates="children",
sa_relationship_kwargs={
"primaryjoin": "StepRunParentsSchema.parent_id == StepRunSchema.id"
},
)
child_id: UUID = build_foreign_key_field(
source=__tablename__,
target=StepRunSchema.__tablename__,
source_column="child_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
child: StepRunSchema = Relationship(
back_populates="parents",
sa_relationship_kwargs={
"primaryjoin": "StepRunParentsSchema.child_id == StepRunSchema.id"
},
)
StepRunSchema (NamedSchema)
pydantic-model
SQL Model for steps of pipeline runs.
Source code in zenml/zen_stores/schemas/step_run_schemas.py
class StepRunSchema(NamedSchema, table=True):
"""SQL Model for steps of pipeline runs."""
__tablename__ = "step_run"
pipeline_run_id: UUID = build_foreign_key_field(
source=__tablename__,
target=PipelineRunSchema.__tablename__,
source_column="pipeline_run_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
pipeline_run: "PipelineRunSchema" = Relationship(
back_populates="step_runs"
)
original_step_run_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=__tablename__,
source_column="original_step_run_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user_id: Optional[UUID] = build_foreign_key_field(
source=__tablename__,
target=UserSchema.__tablename__,
source_column="user_id",
target_column="id",
ondelete="SET NULL",
nullable=True,
)
user: Optional["UserSchema"] = Relationship(back_populates="step_runs")
workspace_id: UUID = build_foreign_key_field(
source=__tablename__,
target=WorkspaceSchema.__tablename__,
source_column="workspace_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
)
workspace: "WorkspaceSchema" = Relationship(back_populates="step_runs")
enable_cache: Optional[bool] = Field(nullable=True)
enable_artifact_metadata: Optional[bool] = Field(nullable=True)
code_hash: Optional[str] = Field(nullable=True)
cache_key: Optional[str] = Field(nullable=True)
start_time: Optional[datetime] = Field(nullable=True)
end_time: Optional[datetime] = Field(nullable=True)
status: ExecutionStatus
entrypoint_name: str
parameters: str = Field(sa_column=Column(TEXT, nullable=False))
step_configuration: str = Field(sa_column=Column(TEXT, nullable=False))
caching_parameters: Optional[str] = Field(
sa_column=Column(TEXT, nullable=True)
)
docstring: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
source_code: Optional[str] = Field(sa_column=Column(TEXT, nullable=True))
num_outputs: Optional[int]
run_metadata: List["RunMetadataSchema"] = Relationship(
back_populates="step_run", sa_relationship_kwargs={"cascade": "delete"}
)
input_artifacts: List["StepRunInputArtifactSchema"] = Relationship(
back_populates="step_run", sa_relationship_kwargs={"cascade": "delete"}
)
output_artifacts: List["StepRunOutputArtifactSchema"] = Relationship(
back_populates="step_run", sa_relationship_kwargs={"cascade": "delete"}
)
parents: List["StepRunParentsSchema"] = Relationship(
back_populates="child",
sa_relationship_kwargs={
"cascade": "delete",
"primaryjoin": "StepRunParentsSchema.child_id == StepRunSchema.id",
},
)
children: List["StepRunParentsSchema"] = Relationship(
back_populates="parent",
sa_relationship_kwargs={
"cascade": "delete",
"primaryjoin": "StepRunParentsSchema.parent_id == StepRunSchema.id",
},
)
@classmethod
def from_request(cls, request: StepRunRequestModel) -> "StepRunSchema":
"""Create a step run schema from a step run request model.
Args:
request: The step run request model.
Returns:
The step run schema.
"""
step_config = request.step.config
return cls(
name=request.name,
pipeline_run_id=request.pipeline_run_id,
original_step_run_id=request.original_step_run_id,
workspace_id=request.workspace,
user_id=request.user,
enable_cache=step_config.enable_cache,
enable_artifact_metadata=step_config.enable_artifact_metadata,
code_hash=step_config.caching_parameters.get(
STEP_SOURCE_PARAMETER_NAME
),
cache_key=request.cache_key,
start_time=request.start_time,
end_time=request.end_time,
entrypoint_name=step_config.name,
parameters=json.dumps(
step_config.parameters,
default=pydantic_encoder,
sort_keys=True,
),
step_configuration=request.step.json(sort_keys=True),
caching_parameters=json.dumps(
step_config.caching_parameters,
default=pydantic_encoder,
sort_keys=True,
),
docstring=request.docstring,
source_code=request.source_code,
num_outputs=len(step_config.outputs),
status=request.status,
)
def to_model(
self,
parent_step_ids: List[UUID],
input_artifacts: Dict[str, "ArtifactResponseModel"],
output_artifacts: Dict[str, "ArtifactResponseModel"],
) -> StepRunResponseModel:
"""Convert a `StepRunSchema` to a `StepRunModel`.
Args:
parent_step_ids: The parent step ids to link to the step.
input_artifacts: The input artifacts to link to the step.
output_artifacts: The output artifacts to link to the step.
Returns:
The created StepRunModel.
"""
metadata = {
metadata_schema.key: metadata_schema.to_model()
for metadata_schema in self.run_metadata
}
return StepRunResponseModel(
id=self.id,
name=self.name,
pipeline_run_id=self.pipeline_run_id,
original_step_run_id=self.original_step_run_id,
workspace=self.workspace.to_model(),
user=self.user.to_model() if self.user else None,
parent_step_ids=parent_step_ids,
enable_cache=self.enable_cache,
enable_artifact_metadata=self.enable_artifact_metadata,
cache_key=self.cache_key,
start_time=self.start_time,
end_time=self.end_time,
step=Step.parse_raw(self.step_configuration),
status=self.status,
docstring=self.docstring,
source_code=self.source_code,
created=self.created,
updated=self.updated,
input_artifacts=input_artifacts,
output_artifacts=output_artifacts,
metadata=metadata,
)
def update(self, step_update: StepRunUpdateModel) -> "StepRunSchema":
"""Update a step run schema with a step run update model.
Args:
step_update: The step run update model.
Returns:
The updated step run schema.
"""
for key, value in step_update.dict(
exclude_unset=True, exclude_none=True
).items():
if key == "status":
self.status = value
if key == "end_time":
self.end_time = value
self.updated = datetime.utcnow()
return self
from_request(request)
classmethod
Create a step run schema from a step run request model.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
request |
StepRunRequestModel |
The step run request model. |
required |
Returns:
| Type | Description |
|---|---|
StepRunSchema |
The step run schema. |
Source code in zenml/zen_stores/schemas/step_run_schemas.py
@classmethod
def from_request(cls, request: StepRunRequestModel) -> "StepRunSchema":
"""Create a step run schema from a step run request model.
Args:
request: The step run request model.
Returns:
The step run schema.
"""
step_config = request.step.config
return cls(
name=request.name,
pipeline_run_id=request.pipeline_run_id,
original_step_run_id=request.original_step_run_id,
workspace_id=request.workspace,
user_id=request.user,
enable_cache=step_config.enable_cache,
enable_artifact_metadata=step_config.enable_artifact_metadata,
code_hash=step_config.caching_parameters.get(
STEP_SOURCE_PARAMETER_NAME
),
cache_key=request.cache_key,
start_time=request.start_time,
end_time=request.end_time,
entrypoint_name=step_config.name,
parameters=json.dumps(
step_config.parameters,
default=pydantic_encoder,
sort_keys=True,
),
step_configuration=request.step.json(sort_keys=True),
caching_parameters=json.dumps(
step_config.caching_parameters,
default=pydantic_encoder,
sort_keys=True,
),
docstring=request.docstring,
source_code=request.source_code,
num_outputs=len(step_config.outputs),
status=request.status,
)
to_model(self, parent_step_ids, input_artifacts, output_artifacts)
Convert a StepRunSchema to a StepRunModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
parent_step_ids |
List[uuid.UUID] |
The parent step ids to link to the step. |
required |
input_artifacts |
Dict[str, ArtifactResponseModel] |
The input artifacts to link to the step. |
required |
output_artifacts |
Dict[str, ArtifactResponseModel] |
The output artifacts to link to the step. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The created StepRunModel. |
Source code in zenml/zen_stores/schemas/step_run_schemas.py
def to_model(
self,
parent_step_ids: List[UUID],
input_artifacts: Dict[str, "ArtifactResponseModel"],
output_artifacts: Dict[str, "ArtifactResponseModel"],
) -> StepRunResponseModel:
"""Convert a `StepRunSchema` to a `StepRunModel`.
Args:
parent_step_ids: The parent step ids to link to the step.
input_artifacts: The input artifacts to link to the step.
output_artifacts: The output artifacts to link to the step.
Returns:
The created StepRunModel.
"""
metadata = {
metadata_schema.key: metadata_schema.to_model()
for metadata_schema in self.run_metadata
}
return StepRunResponseModel(
id=self.id,
name=self.name,
pipeline_run_id=self.pipeline_run_id,
original_step_run_id=self.original_step_run_id,
workspace=self.workspace.to_model(),
user=self.user.to_model() if self.user else None,
parent_step_ids=parent_step_ids,
enable_cache=self.enable_cache,
enable_artifact_metadata=self.enable_artifact_metadata,
cache_key=self.cache_key,
start_time=self.start_time,
end_time=self.end_time,
step=Step.parse_raw(self.step_configuration),
status=self.status,
docstring=self.docstring,
source_code=self.source_code,
created=self.created,
updated=self.updated,
input_artifacts=input_artifacts,
output_artifacts=output_artifacts,
metadata=metadata,
)
update(self, step_update)
Update a step run schema with a step run update model.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_update |
StepRunUpdateModel |
The step run update model. |
required |
Returns:
| Type | Description |
|---|---|
StepRunSchema |
The updated step run schema. |
Source code in zenml/zen_stores/schemas/step_run_schemas.py
def update(self, step_update: StepRunUpdateModel) -> "StepRunSchema":
"""Update a step run schema with a step run update model.
Args:
step_update: The step run update model.
Returns:
The updated step run schema.
"""
for key, value in step_update.dict(
exclude_unset=True, exclude_none=True
).items():
if key == "status":
self.status = value
if key == "end_time":
self.end_time = value
self.updated = datetime.utcnow()
return self
team_schemas
SQLModel implementation of team tables.
TeamAssignmentSchema (SQLModel)
pydantic-model
SQL Model for team assignments.
Source code in zenml/zen_stores/schemas/team_schemas.py
class TeamAssignmentSchema(SQLModel, table=True):
"""SQL Model for team assignments."""
__tablename__ = "team_assignment"
user_id: UUID = build_foreign_key_field(
source=__tablename__,
target="user", # TODO: how to reference `UserSchema.__tablename__`?
source_column="user_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
team_id: UUID = build_foreign_key_field(
source=__tablename__,
target="team", # TODO: how to reference `TeamSchema.__tablename__`?
source_column="team_id",
target_column="id",
ondelete="CASCADE",
nullable=False,
primary_key=True,
)
TeamSchema (NamedSchema)
pydantic-model
SQL Model for teams.
Source code in zenml/zen_stores/schemas/team_schemas.py
class TeamSchema(NamedSchema, table=True):
"""SQL Model for teams."""
__tablename__ = "team"
users: List["UserSchema"] = Relationship(
back_populates="teams", link_model=TeamAssignmentSchema
)
assigned_roles: List["TeamRoleAssignmentSchema"] = Relationship(
back_populates="team", sa_relationship_kwargs={"cascade": "delete"}
)
def update(self, team_update: TeamUpdateModel) -> "TeamSchema":
"""Update a `TeamSchema` with a `TeamUpdateModel`.
Args:
team_update: The `TeamUpdateModel` to update the schema with.
Returns:
The updated `TeamSchema`.
"""
for field, value in team_update.dict(exclude_unset=True).items():
if field == "users":
pass
else:
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
def to_model(self, _block_recursion: bool = False) -> TeamResponseModel:
"""Convert a `TeamSchema` to a `TeamResponseModel`.
Args:
_block_recursion: Don't recursively fill attributes
Returns:
The converted `TeamResponseModel`.
"""
if _block_recursion:
return TeamResponseModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
users=[],
)
else:
return TeamResponseModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
users=[u.to_model(_block_recursion=False) for u in self.users],
)
to_model(self, _block_recursion=False)
Convert a TeamSchema to a TeamResponseModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
_block_recursion |
bool |
Don't recursively fill attributes |
False |
Returns:
| Type | Description |
|---|---|
TeamResponseModel |
The converted |
Source code in zenml/zen_stores/schemas/team_schemas.py
def to_model(self, _block_recursion: bool = False) -> TeamResponseModel:
"""Convert a `TeamSchema` to a `TeamResponseModel`.
Args:
_block_recursion: Don't recursively fill attributes
Returns:
The converted `TeamResponseModel`.
"""
if _block_recursion:
return TeamResponseModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
users=[],
)
else:
return TeamResponseModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
users=[u.to_model(_block_recursion=False) for u in self.users],
)
update(self, team_update)
Update a TeamSchema with a TeamUpdateModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_update |
TeamUpdateModel |
The |
required |
Returns:
| Type | Description |
|---|---|
TeamSchema |
The updated |
Source code in zenml/zen_stores/schemas/team_schemas.py
def update(self, team_update: TeamUpdateModel) -> "TeamSchema":
"""Update a `TeamSchema` with a `TeamUpdateModel`.
Args:
team_update: The `TeamUpdateModel` to update the schema with.
Returns:
The updated `TeamSchema`.
"""
for field, value in team_update.dict(exclude_unset=True).items():
if field == "users":
pass
else:
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
user_schemas
SQLModel implementation of user tables.
UserSchema (NamedSchema)
pydantic-model
SQL Model for users.
Source code in zenml/zen_stores/schemas/user_schemas.py
class UserSchema(NamedSchema, table=True):
"""SQL Model for users."""
__tablename__ = "user"
full_name: str
email: Optional[str] = Field(nullable=True)
active: bool
password: Optional[str] = Field(nullable=True)
activation_token: Optional[str] = Field(nullable=True)
email_opted_in: Optional[bool] = Field(nullable=True)
teams: List["TeamSchema"] = Relationship(
back_populates="users", link_model=TeamAssignmentSchema
)
assigned_roles: List["UserRoleAssignmentSchema"] = Relationship(
back_populates="user", sa_relationship_kwargs={"cascade": "delete"}
)
stacks: List["StackSchema"] = Relationship(back_populates="user")
components: List["StackComponentSchema"] = Relationship(
back_populates="user",
)
flavors: List["FlavorSchema"] = Relationship(back_populates="user")
pipelines: List["PipelineSchema"] = Relationship(back_populates="user")
schedules: List["ScheduleSchema"] = Relationship(
back_populates="user",
)
runs: List["PipelineRunSchema"] = Relationship(back_populates="user")
step_runs: List["StepRunSchema"] = Relationship(back_populates="user")
artifacts: List["ArtifactSchema"] = Relationship(back_populates="user")
run_metadata: List["RunMetadataSchema"] = Relationship(
back_populates="user"
)
@classmethod
def from_request(cls, model: UserRequestModel) -> "UserSchema":
"""Create a `UserSchema` from a `UserModel`.
Args:
model: The `UserModel` from which to create the schema.
Returns:
The created `UserSchema`.
"""
return cls(
name=model.name,
full_name=model.full_name,
active=model.active,
password=model.create_hashed_password(),
activation_token=model.create_hashed_activation_token(),
)
def update(self, user_update: UserUpdateModel) -> "UserSchema":
"""Update a `UserSchema` from a `UserUpdateModel`.
Args:
user_update: The `UserUpdateModel` from which to update the schema.
Returns:
The updated `UserSchema`.
"""
for field, value in user_update.dict(exclude_unset=True).items():
if field == "password":
setattr(self, field, user_update.create_hashed_password())
elif field == "activation_token":
setattr(
self, field, user_update.create_hashed_activation_token()
)
else:
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
def to_model(
self, _block_recursion: bool = False, include_private: bool = False
) -> UserResponseModel:
"""Convert a `UserSchema` to a `UserResponseModel`.
Args:
_block_recursion: Don't recursively fill attributes
include_private: Whether to include the user private information
this is to limit the amount of data one can get
about other users
Returns:
The converted `UserResponseModel`.
"""
if _block_recursion:
return UserResponseModel(
id=self.id,
name=self.name,
active=self.active,
email_opted_in=self.email_opted_in,
email=self.email if include_private else None,
full_name=self.full_name,
created=self.created,
updated=self.updated,
)
else:
return UserResponseModel(
id=self.id,
name=self.name,
active=self.active,
email_opted_in=self.email_opted_in,
email=self.email if include_private else None,
teams=[t.to_model(_block_recursion=True) for t in self.teams],
full_name=self.full_name,
created=self.created,
updated=self.updated,
roles=[ra.role.to_model() for ra in self.assigned_roles],
)
from_request(model)
classmethod
Create a UserSchema from a UserModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
UserRequestModel |
The |
required |
Returns:
| Type | Description |
|---|---|
UserSchema |
The created |
Source code in zenml/zen_stores/schemas/user_schemas.py
@classmethod
def from_request(cls, model: UserRequestModel) -> "UserSchema":
"""Create a `UserSchema` from a `UserModel`.
Args:
model: The `UserModel` from which to create the schema.
Returns:
The created `UserSchema`.
"""
return cls(
name=model.name,
full_name=model.full_name,
active=model.active,
password=model.create_hashed_password(),
activation_token=model.create_hashed_activation_token(),
)
to_model(self, _block_recursion=False, include_private=False)
Convert a UserSchema to a UserResponseModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
_block_recursion |
bool |
Don't recursively fill attributes |
False |
include_private |
bool |
Whether to include the user private information this is to limit the amount of data one can get about other users |
False |
Returns:
| Type | Description |
|---|---|
UserResponseModel |
The converted |
Source code in zenml/zen_stores/schemas/user_schemas.py
def to_model(
self, _block_recursion: bool = False, include_private: bool = False
) -> UserResponseModel:
"""Convert a `UserSchema` to a `UserResponseModel`.
Args:
_block_recursion: Don't recursively fill attributes
include_private: Whether to include the user private information
this is to limit the amount of data one can get
about other users
Returns:
The converted `UserResponseModel`.
"""
if _block_recursion:
return UserResponseModel(
id=self.id,
name=self.name,
active=self.active,
email_opted_in=self.email_opted_in,
email=self.email if include_private else None,
full_name=self.full_name,
created=self.created,
updated=self.updated,
)
else:
return UserResponseModel(
id=self.id,
name=self.name,
active=self.active,
email_opted_in=self.email_opted_in,
email=self.email if include_private else None,
teams=[t.to_model(_block_recursion=True) for t in self.teams],
full_name=self.full_name,
created=self.created,
updated=self.updated,
roles=[ra.role.to_model() for ra in self.assigned_roles],
)
update(self, user_update)
Update a UserSchema from a UserUpdateModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_update |
UserUpdateModel |
The |
required |
Returns:
| Type | Description |
|---|---|
UserSchema |
The updated |
Source code in zenml/zen_stores/schemas/user_schemas.py
def update(self, user_update: UserUpdateModel) -> "UserSchema":
"""Update a `UserSchema` from a `UserUpdateModel`.
Args:
user_update: The `UserUpdateModel` from which to update the schema.
Returns:
The updated `UserSchema`.
"""
for field, value in user_update.dict(exclude_unset=True).items():
if field == "password":
setattr(self, field, user_update.create_hashed_password())
elif field == "activation_token":
setattr(
self, field, user_update.create_hashed_activation_token()
)
else:
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
workspace_schemas
SQL Model Implementations for Workspaces.
WorkspaceSchema (NamedSchema)
pydantic-model
SQL Model for workspaces.
Source code in zenml/zen_stores/schemas/workspace_schemas.py
class WorkspaceSchema(NamedSchema, table=True):
"""SQL Model for workspaces."""
__tablename__ = "workspace"
description: str
user_role_assignments: List["UserRoleAssignmentSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
team_role_assignments: List["TeamRoleAssignmentSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "all, delete"},
)
stacks: List["StackSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
components: List["StackComponentSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
flavors: List["FlavorSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
pipelines: List["PipelineSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
schedules: List["ScheduleSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
runs: List["PipelineRunSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
step_runs: List["StepRunSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
artifacts: List["ArtifactSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
run_metadata: List["RunMetadataSchema"] = Relationship(
back_populates="workspace",
sa_relationship_kwargs={"cascade": "delete"},
)
@classmethod
def from_request(
cls, workspace: WorkspaceRequestModel
) -> "WorkspaceSchema":
"""Create a `WorkspaceSchema` from a `WorkspaceResponseModel`.
Args:
workspace: The `WorkspaceResponseModel` from which to create the schema.
Returns:
The created `WorkspaceSchema`.
"""
return cls(name=workspace.name, description=workspace.description)
def update(
self, workspace_update: WorkspaceUpdateModel
) -> "WorkspaceSchema":
"""Update a `WorkspaceSchema` from a `WorkspaceUpdateModel`.
Args:
workspace_update: The `WorkspaceUpdateModel` from which to update the
schema.
Returns:
The updated `WorkspaceSchema`.
"""
for field, value in workspace_update.dict(exclude_unset=True).items():
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
def to_model(self) -> WorkspaceResponseModel:
"""Convert a `WorkspaceSchema` to a `WorkspaceResponseModel`.
Returns:
The converted `WorkspaceResponseModel`.
"""
return WorkspaceResponseModel(
id=self.id,
name=self.name,
description=self.description,
created=self.created,
updated=self.updated,
)
from_request(workspace)
classmethod
Create a WorkspaceSchema from a WorkspaceResponseModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace |
WorkspaceRequestModel |
The |
required |
Returns:
| Type | Description |
|---|---|
WorkspaceSchema |
The created |
Source code in zenml/zen_stores/schemas/workspace_schemas.py
@classmethod
def from_request(
cls, workspace: WorkspaceRequestModel
) -> "WorkspaceSchema":
"""Create a `WorkspaceSchema` from a `WorkspaceResponseModel`.
Args:
workspace: The `WorkspaceResponseModel` from which to create the schema.
Returns:
The created `WorkspaceSchema`.
"""
return cls(name=workspace.name, description=workspace.description)
to_model(self)
Convert a WorkspaceSchema to a WorkspaceResponseModel.
Returns:
| Type | Description |
|---|---|
WorkspaceResponseModel |
The converted |
Source code in zenml/zen_stores/schemas/workspace_schemas.py
def to_model(self) -> WorkspaceResponseModel:
"""Convert a `WorkspaceSchema` to a `WorkspaceResponseModel`.
Returns:
The converted `WorkspaceResponseModel`.
"""
return WorkspaceResponseModel(
id=self.id,
name=self.name,
description=self.description,
created=self.created,
updated=self.updated,
)
update(self, workspace_update)
Update a WorkspaceSchema from a WorkspaceUpdateModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_update |
WorkspaceUpdateModel |
The |
required |
Returns:
| Type | Description |
|---|---|
WorkspaceSchema |
The updated |
Source code in zenml/zen_stores/schemas/workspace_schemas.py
def update(
self, workspace_update: WorkspaceUpdateModel
) -> "WorkspaceSchema":
"""Update a `WorkspaceSchema` from a `WorkspaceUpdateModel`.
Args:
workspace_update: The `WorkspaceUpdateModel` from which to update the
schema.
Returns:
The updated `WorkspaceSchema`.
"""
for field, value in workspace_update.dict(exclude_unset=True).items():
setattr(self, field, value)
self.updated = datetime.utcnow()
return self
sql_zen_store
SQL Zen Store implementation.
SQLDatabaseDriver (StrEnum)
SQL database drivers supported by the SQL ZenML store.
Source code in zenml/zen_stores/sql_zen_store.py
class SQLDatabaseDriver(StrEnum):
"""SQL database drivers supported by the SQL ZenML store."""
MYSQL = "mysql"
SQLITE = "sqlite"
SqlZenStore (BaseZenStore)
pydantic-model
Store Implementation that uses SQL database backend.
Attributes:
| Name | Type | Description |
|---|---|---|
config |
SqlZenStoreConfiguration |
The configuration of the SQL ZenML store. |
skip_migrations |
bool |
Whether to skip migrations when initializing the store. |
TYPE |
ClassVar[zenml.enums.StoreType] |
The type of the store. |
CONFIG_TYPE |
ClassVar[Type[zenml.config.store_config.StoreConfiguration]] |
The type of the store configuration. |
_engine |
Optional[sqlalchemy.engine.base.Engine] |
The SQLAlchemy engine. |
Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStore(BaseZenStore):
"""Store Implementation that uses SQL database backend.
Attributes:
config: The configuration of the SQL ZenML store.
skip_migrations: Whether to skip migrations when initializing the store.
TYPE: The type of the store.
CONFIG_TYPE: The type of the store configuration.
_engine: The SQLAlchemy engine.
"""
config: SqlZenStoreConfiguration
skip_migrations: bool = False
TYPE: ClassVar[StoreType] = StoreType.SQL
CONFIG_TYPE: ClassVar[Type[StoreConfiguration]] = SqlZenStoreConfiguration
_engine: Optional[Engine] = None
_alembic: Optional[Alembic] = None
@property
def engine(self) -> Engine:
"""The SQLAlchemy engine.
Returns:
The SQLAlchemy engine.
Raises:
ValueError: If the store is not initialized.
"""
if not self._engine:
raise ValueError("Store not initialized")
return self._engine
@property
def runs_inside_server(self) -> bool:
"""Whether the store is running inside a server.
Returns:
Whether the store is running inside a server.
"""
if ENV_ZENML_SERVER_DEPLOYMENT_TYPE in os.environ:
return True
return False
@property
def alembic(self) -> Alembic:
"""The Alembic wrapper.
Returns:
The Alembic wrapper.
Raises:
ValueError: If the store is not initialized.
"""
if not self._alembic:
raise ValueError("Store not initialized")
return self._alembic
@classmethod
def filter_and_paginate(
cls,
session: Session,
query: Union[Select[AnySchema], SelectOfScalar[AnySchema]],
table: Type[AnySchema],
filter_model: BaseFilterModel,
custom_schema_to_model_conversion: Optional[
Callable[[AnySchema], B]
] = None,
) -> Page[B]:
"""Given a query, return a Page instance with a list of filtered Models.
Args:
session: The SQLModel Session
query: The query to execute
table: The table to select from
filter_model: The filter to use, including pagination and sorting
custom_schema_to_model_conversion: Callable to convert the schema
into a model. This is used if the Model contains additional
data that is not explicitly stored as a field or relationship
on the model.
Returns:
The Domain Model representation of the DB resource
Raises:
ValueError: if the filtered page number is out of bounds.
RuntimeError: if the schema does not have a `to_model` method.
"""
# Filtering
filters = filter_model.generate_filter(table=table)
if filters is not None:
query = query.where(filters)
# Get the total amount of items in the database for a given query
total = session.scalar(
select([func.count("*")]).select_from(
query.options(noload("*")).subquery()
)
)
# Sorting
column, operand = filter_model.sorting_params
if operand == SorterOps.DESCENDING:
query = query.order_by(desc(getattr(table, column)))
else:
query = query.order_by(asc(getattr(table, column)))
# Get the total amount of pages in the database for a given query
if total == 0:
total_pages = 1
else:
total_pages = math.ceil(total / filter_model.size)
if filter_model.page > total_pages:
raise ValueError(
f"Invalid page {filter_model.page}. The requested page size is "
f"{filter_model.size} and there are a total of {total} items "
f"for this query. The maximum page value therefore is "
f"{total_pages}."
)
# Get a page of the actual data
item_schemas: List[AnySchema] = (
session.exec(
query.limit(filter_model.size).offset(filter_model.offset)
)
.unique()
.all()
)
# Convert this page of items from schemas to models.
items: List[B] = []
for schema in item_schemas:
# If a custom conversion function is provided, use it.
if custom_schema_to_model_conversion:
items.append(custom_schema_to_model_conversion(schema))
continue
# Otherwise, try to use the `to_model` method of the schema.
to_model = getattr(schema, "to_model", None)
if callable(to_model):
items.append(to_model())
continue
# If neither of the above work, raise an error.
raise RuntimeError(
f"Cannot convert schema `{schema.__class__.__name__}` to model "
"since it does not have a `to_model` method."
)
return Page(
total=total,
total_pages=total_pages,
items=items,
page=filter_model.page,
size=filter_model.size,
)
# ====================================
# ZenML Store interface implementation
# ====================================
# --------------------------------
# Initialization and configuration
# --------------------------------
def _initialize(self) -> None:
"""Initialize the SQL store.
Raises:
OperationalError: If connecting to the database failed.
"""
logger.debug("Initializing SqlZenStore at %s", self.config.url)
url, connect_args, engine_args = self.config.get_sqlmodel_config()
self._engine = create_engine(
url=url, connect_args=connect_args, **engine_args
)
# SQLite: As long as the parent directory exists, SQLAlchemy will
# automatically create the database.
if (
self.config.driver == SQLDatabaseDriver.SQLITE
and self.config.database
and not fileio.exists(self.config.database)
):
fileio.makedirs(os.path.dirname(self.config.database))
# MySQL: We might need to create the database manually.
# To do so, we create a new engine that connects to the `mysql` database
# and then create the desired database.
# See https://stackoverflow.com/a/8977109
if (
self.config.driver == SQLDatabaseDriver.MYSQL
and self.config.database
):
try:
self._engine.connect()
except OperationalError as e:
logger.debug(
"Failed to connect to mysql database `%s`.",
self._engine.url.database,
)
if _is_mysql_missing_database_error(e):
self._create_mysql_database(
url=self._engine.url,
connect_args=connect_args,
engine_args=engine_args,
)
else:
raise
self._alembic = Alembic(self.engine)
if (
not self.skip_migrations
and ENV_ZENML_DISABLE_DATABASE_MIGRATION not in os.environ
):
self.migrate_database()
def _create_mysql_database(
self,
url: URL,
connect_args: Dict[str, Any],
engine_args: Dict[str, Any],
) -> None:
"""Creates a mysql database.
Args:
url: The URL of the database to create.
connect_args: Connect arguments for the SQLAlchemy engine.
engine_args: Additional initialization arguments for the SQLAlchemy
engine
"""
logger.info("Trying to create database %s.", url.database)
master_url = url._replace(database=None)
master_engine = create_engine(
url=master_url, connect_args=connect_args, **engine_args
)
query = f"CREATE DATABASE IF NOT EXISTS {self.config.database}"
try:
connection = master_engine.connect()
connection.execute(text(query))
finally:
connection.close()
def migrate_database(self) -> None:
"""Migrate the database to the head as defined by the python package."""
alembic_logger = logging.getLogger("alembic")
# remove all existing handlers
while len(alembic_logger.handlers):
alembic_logger.removeHandler(alembic_logger.handlers[0])
logging_level = get_logging_level()
# suppress alembic info logging if the zenml logging level is not debug
if logging_level == LoggingLevels.DEBUG:
alembic_logger.setLevel(logging.DEBUG)
else:
alembic_logger.setLevel(logging.WARNING)
alembic_logger.addHandler(get_console_handler())
# We need to account for 3 distinct cases here:
# 1. the database is completely empty (not initialized)
# 2. the database is not empty, but has never been migrated with alembic
# before (i.e. was created with SQLModel back when alembic wasn't
# used)
# 3. the database is not empty and has been migrated with alembic before
revisions = self.alembic.current_revisions()
if len(revisions) >= 1:
if len(revisions) > 1:
logger.warning(
"The ZenML database has more than one migration head "
"revision. This is not expected and might indicate a "
"database migration problem. Please raise an issue on "
"GitHub if you encounter this."
)
# Case 3: the database has been migrated with alembic before. Just
# upgrade to the latest revision.
self.alembic.upgrade()
else:
if self.alembic.db_is_empty():
# Case 1: the database is empty. We can just create the
# tables from scratch with alembic.
self.alembic.upgrade()
else:
# Case 2: the database is not empty, but has never been
# migrated with alembic before. We need to create the alembic
# version table, initialize it with the first revision where we
# introduced alembic and then upgrade to the latest revision.
self.alembic.stamp(ZENML_ALEMBIC_START_REVISION)
self.alembic.upgrade()
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
Raises:
KeyError: If the deployment ID could not be loaded from the
database.
"""
model = super().get_store_info()
sql_url = make_url(self.config.url)
model.database_type = ServerDatabaseType(sql_url.drivername)
# Fetch the deployment ID from the database and use it to replace the one
# fetched from the global configuration
with Session(self.engine) as session:
identity = session.exec(select(IdentitySchema)).first()
if identity is None:
raise KeyError(
"The deployment ID could not be loaded from the database."
)
model.id = identity.id
return model
# ------
# Stacks
# ------
@track(AnalyticsEvent.REGISTERED_STACK)
def create_stack(self, stack: StackRequestModel) -> StackResponseModel:
"""Register a new stack.
Args:
stack: The stack to register.
Returns:
The registered stack.
"""
with Session(self.engine) as session:
self._fail_if_stack_with_name_exists_for_user(
stack=stack, session=session
)
if stack.is_shared:
self._fail_if_stack_with_name_already_shared(
stack=stack, session=session
)
# Get the Schemas of all components mentioned
component_ids = [
component_id
for list_of_component_ids in stack.components.values()
for component_id in list_of_component_ids
]
filters = [
(StackComponentSchema.id == component_id)
for component_id in component_ids
]
defined_components = session.exec(
select(StackComponentSchema).where(or_(*filters))
).all()
new_stack_schema = StackSchema(
workspace_id=stack.workspace,
user_id=stack.user,
is_shared=stack.is_shared,
name=stack.name,
description=stack.description,
components=defined_components,
)
session.add(new_stack_schema)
session.commit()
session.refresh(new_stack_schema)
return new_stack_schema.to_model()
def get_stack(self, stack_id: UUID) -> StackResponseModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
Raises:
KeyError: if the stack doesn't exist.
"""
with Session(self.engine) as session:
stack = session.exec(
select(StackSchema).where(StackSchema.id == stack_id)
).first()
if stack is None:
raise KeyError(f"Stack with ID {stack_id} not found.")
return stack.to_model()
def list_stacks(
self, stack_filter_model: StackFilterModel
) -> Page[StackResponseModel]:
"""List all stacks matching the given filter criteria.
Args:
stack_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stacks matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(StackSchema)
if stack_filter_model.component_id:
query = query.where(
StackCompositionSchema.stack_id == StackSchema.id
).where(
StackCompositionSchema.component_id
== stack_filter_model.component_id
)
return self.filter_and_paginate(
session=session,
query=query,
table=StackSchema,
filter_model=stack_filter_model,
)
@track(AnalyticsEvent.UPDATED_STACK)
def update_stack(
self, stack_id: UUID, stack_update: StackUpdateModel
) -> StackResponseModel:
"""Update a stack.
Args:
stack_id: The ID of the stack update.
stack_update: The update request on the stack.
Returns:
The updated stack.
Raises:
KeyError: if the stack doesn't exist.
IllegalOperationError: if the stack is a default stack.
"""
with Session(self.engine) as session:
# Check if stack with the domain key (name, workspace, owner) already
# exists
existing_stack = session.exec(
select(StackSchema).where(StackSchema.id == stack_id)
).first()
if existing_stack is None:
raise KeyError(
f"Unable to update stack with id '{stack_id}': Found no"
f"existing stack with this id."
)
if existing_stack.name == DEFAULT_STACK_NAME:
raise IllegalOperationError(
"The default stack cannot be modified."
)
# In case of a renaming update, make sure no stack already exists
# with that name
if stack_update.name:
if existing_stack.name != stack_update.name:
self._fail_if_stack_with_name_exists_for_user(
stack=stack_update, session=session
)
# Check if stack update makes the stack a shared stack. In that
# case, check if a stack with the same name is already shared
# within the workspace
if stack_update.is_shared:
if not existing_stack.is_shared and stack_update.is_shared:
self._fail_if_stack_with_name_already_shared(
stack=stack_update, session=session
)
components = []
if stack_update.components:
filters = [
(StackComponentSchema.id == component_id)
for list_of_component_ids in stack_update.components.values()
for component_id in list_of_component_ids
]
components = session.exec(
select(StackComponentSchema).where(or_(*filters))
).all()
existing_stack.update(
stack_update=stack_update,
components=components,
)
session.add(existing_stack)
session.commit()
session.refresh(existing_stack)
return existing_stack.to_model()
@track(AnalyticsEvent.DELETED_STACK)
def delete_stack(self, stack_id: UUID) -> None:
"""Delete a stack.
Args:
stack_id: The ID of the stack to delete.
Raises:
KeyError: if the stack doesn't exist.
IllegalOperationError: if the stack is a default stack.
"""
with Session(self.engine) as session:
try:
stack = session.exec(
select(StackSchema).where(StackSchema.id == stack_id)
).one()
if stack.name == DEFAULT_STACK_NAME:
raise IllegalOperationError(
"The default stack cannot be deleted."
)
session.delete(stack)
except NoResultFound as error:
raise KeyError from error
session.commit()
def _fail_if_stack_with_name_exists_for_user(
self,
stack: StackRequestModel,
session: Session,
) -> None:
"""Raise an exception if a Component with same name exists for user.
Args:
stack: The Stack
session: The Session
Returns:
None
Raises:
StackExistsError: If a Stack with the given name is already
owned by the user
"""
existing_domain_stack = session.exec(
select(StackSchema)
.where(StackSchema.name == stack.name)
.where(StackSchema.workspace_id == stack.workspace)
.where(StackSchema.user_id == stack.user)
).first()
if existing_domain_stack is not None:
workspace = self._get_workspace_schema(
workspace_name_or_id=stack.workspace, session=session
)
user = self._get_user_schema(
user_name_or_id=stack.user, session=session
)
raise StackExistsError(
f"Unable to register stack with name "
f"'{stack.name}': Found an existing stack with the same "
f"name in the active workspace, '{workspace.name}', owned by the "
f"same user, '{user.name}'."
)
return None
def _fail_if_stack_with_name_already_shared(
self,
stack: StackRequestModel,
session: Session,
) -> None:
"""Raise an exception if a Stack with same name is already shared.
Args:
stack: The Stack
session: The Session
Raises:
StackExistsError: If a stack with the given name is already shared
by a user.
"""
# Check if component with the same name, type is already shared
# within the workspace
existing_shared_stack = session.exec(
select(StackSchema)
.where(StackSchema.name == stack.name)
.where(StackSchema.workspace_id == stack.workspace)
.where(StackSchema.is_shared == stack.is_shared)
).first()
if existing_shared_stack is not None:
workspace = self._get_workspace_schema(
workspace_name_or_id=stack.workspace, session=session
)
error_msg = (
f"Unable to share stack with name '{stack.name}': Found an "
f"existing shared stack with the same name in workspace "
f"'{workspace.name}'"
)
if existing_shared_stack.user_id:
owner_of_shared = self._get_user_schema(
existing_shared_stack.user_id, session=session
)
error_msg += f" owned by '{owner_of_shared.name}'."
else:
error_msg += ", which is currently not owned by any user."
raise StackExistsError(error_msg)
# ----------------
# Stack components
# ----------------
@track(AnalyticsEvent.REGISTERED_STACK_COMPONENT)
def create_stack_component(
self,
component: ComponentRequestModel,
) -> ComponentResponseModel:
"""Create a stack component.
Args:
component: The stack component to create.
Returns:
The created stack component.
"""
with Session(self.engine) as session:
self._fail_if_component_with_name_type_exists_for_user(
name=component.name,
component_type=component.type,
user_id=component.user,
workspace_id=component.workspace,
session=session,
)
if component.is_shared:
self._fail_if_component_with_name_type_already_shared(
name=component.name,
component_type=component.type,
workspace_id=component.workspace,
session=session,
)
# Create the component
new_component = StackComponentSchema(
name=component.name,
workspace_id=component.workspace,
user_id=component.user,
is_shared=component.is_shared,
type=component.type,
flavor=component.flavor,
configuration=base64.b64encode(
json.dumps(component.configuration).encode("utf-8")
),
)
session.add(new_component)
session.commit()
session.refresh(new_component)
return new_component.to_model()
def get_stack_component(
self, component_id: UUID
) -> ComponentResponseModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
with Session(self.engine) as session:
stack_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component_id
)
).first()
if stack_component is None:
raise KeyError(
f"Stack component with ID {component_id} not found."
)
return stack_component.to_model()
def list_stack_components(
self, component_filter_model: ComponentFilterModel
) -> Page[ComponentResponseModel]:
"""List all stack components matching the given filter criteria.
Args:
component_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stack components matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(StackComponentSchema)
paged_components: Page[
ComponentResponseModel
] = self.filter_and_paginate(
session=session,
query=query,
table=StackComponentSchema,
filter_model=component_filter_model,
)
return paged_components
@track(AnalyticsEvent.UPDATED_STACK_COMPONENT)
def update_stack_component(
self, component_id: UUID, component_update: ComponentUpdateModel
) -> ComponentResponseModel:
"""Update an existing stack component.
Args:
component_id: The ID of the stack component to update.
component_update: The update to be applied to the stack component.
Returns:
The updated stack component.
Raises:
KeyError: if the stack component doesn't exist.
IllegalOperationError: if the stack component is a default stack
component.
"""
with Session(self.engine) as session:
existing_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component_id
)
).first()
if existing_component is None:
raise KeyError(
f"Unable to update component with id "
f"'{component_id}': Found no"
f"existing component with this id."
)
if (
existing_component.name == DEFAULT_STACK_COMPONENT_NAME
and existing_component.type
in [
StackComponentType.ORCHESTRATOR,
StackComponentType.ARTIFACT_STORE,
]
):
raise IllegalOperationError(
f"The default {existing_component.type} cannot be modified."
)
# In case of a renaming update, make sure no component of the same
# type already exists with that name
if component_update.name:
if (
existing_component.name != component_update.name
and existing_component.user_id is not None
):
self._fail_if_component_with_name_type_exists_for_user(
name=component_update.name,
component_type=existing_component.type,
workspace_id=existing_component.workspace_id,
user_id=existing_component.user_id,
session=session,
)
# Check if component update makes the component a shared component,
# In that case check if a component with the same name, type are
# already shared within the workspace
if component_update.is_shared:
if (
not existing_component.is_shared
and component_update.is_shared
):
self._fail_if_component_with_name_type_already_shared(
name=component_update.name or existing_component.name,
component_type=existing_component.type,
workspace_id=existing_component.workspace_id,
session=session,
)
existing_component.update(component_update=component_update)
session.add(existing_component)
session.commit()
return existing_component.to_model()
@track(AnalyticsEvent.DELETED_STACK_COMPONENT)
def delete_stack_component(self, component_id: UUID) -> None:
"""Delete a stack component.
Args:
component_id: The id of the stack component to delete.
Raises:
KeyError: if the stack component doesn't exist.
IllegalOperationError: if the stack component is part of one or
more stacks, or if it's a default stack component.
"""
with Session(self.engine) as session:
try:
stack_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component_id
)
).one()
if (
stack_component.name == DEFAULT_STACK_COMPONENT_NAME
and stack_component.type
in [
StackComponentType.ORCHESTRATOR,
StackComponentType.ARTIFACT_STORE,
]
):
raise IllegalOperationError(
f"The default {stack_component.type} cannot be deleted."
)
if len(stack_component.stacks) > 0:
raise IllegalOperationError(
f"Stack Component `{stack_component.name}` of type "
f"`{stack_component.type} cannot be "
f"deleted as it is part of "
f"{len(stack_component.stacks)} stacks. "
f"Before deleting this stack "
f"component, make sure to remove it "
f"from all stacks."
)
else:
session.delete(stack_component)
except NoResultFound as error:
raise KeyError from error
session.commit()
@staticmethod
def _fail_if_component_with_name_type_exists_for_user(
name: str,
component_type: StackComponentType,
workspace_id: UUID,
user_id: UUID,
session: Session,
) -> None:
"""Raise an exception if a Component with same name/type exists.
Args:
name: The name of the component
component_type: The type of the component
workspace_id: The ID of the workspace
user_id: The ID of the user
session: The Session
Returns:
None
Raises:
StackComponentExistsError: If a component with the given name and
type is already owned by the user
"""
assert user_id
# Check if component with the same domain key (name, type, workspace,
# owner) already exists
existing_domain_component = session.exec(
select(StackComponentSchema)
.where(StackComponentSchema.name == name)
.where(StackComponentSchema.workspace_id == workspace_id)
.where(StackComponentSchema.user_id == user_id)
.where(StackComponentSchema.type == component_type)
).first()
if existing_domain_component is not None:
# Theoretically the user schema is optional, in this case there is
# no way that it will be None
assert existing_domain_component.user
raise StackComponentExistsError(
f"Unable to register '{component_type.value}' component "
f"with name '{name}': Found an existing "
f"component with the same name and type in the same "
f" workspace, '{existing_domain_component.workspace.name}', "
f"owned by the same user, "
f"'{existing_domain_component.user.name}'."
)
return None
@staticmethod
def _fail_if_component_with_name_type_already_shared(
name: str,
component_type: StackComponentType,
workspace_id: UUID,
session: Session,
) -> None:
"""Raise an exception if a Component with same name/type already shared.
Args:
name: The name of the component
component_type: The type of the component
workspace_id: The ID of the workspace
session: The Session
Raises:
StackComponentExistsError: If a component with the given name and
type is already shared by a user
"""
# Check if component with the same name, type is already shared
# within the workspace
existing_shared_component = session.exec(
select(StackComponentSchema)
.where(StackComponentSchema.name == name)
.where(StackComponentSchema.workspace_id == workspace_id)
.where(StackComponentSchema.type == component_type)
.where(StackComponentSchema.is_shared is True)
).first()
if existing_shared_component is not None:
raise StackComponentExistsError(
f"Unable to shared component of type '{component_type.value}' "
f"with name '{name}': Found an existing shared "
f"component with the same name and type in workspace "
f"'{workspace_id}'."
)
# -----------------------
# Stack component flavors
# -----------------------
@track(AnalyticsEvent.CREATED_FLAVOR)
def create_flavor(self, flavor: FlavorRequestModel) -> FlavorResponseModel:
"""Creates a new stack component flavor.
Args:
flavor: The stack component flavor to create.
Returns:
The newly created flavor.
Raises:
EntityExistsError: If a flavor with the same name and type
is already owned by this user in this workspace.
"""
with Session(self.engine) as session:
# Check if component with the same domain key (name, type, workspace,
# owner) already exists
existing_flavor = session.exec(
select(FlavorSchema)
.where(FlavorSchema.name == flavor.name)
.where(FlavorSchema.type == flavor.type)
.where(FlavorSchema.workspace_id == flavor.workspace)
.where(FlavorSchema.user_id == flavor.user)
).first()
if existing_flavor is not None:
raise EntityExistsError(
f"Unable to register '{flavor.type.value}' flavor "
f"with name '{flavor.name}': Found an existing "
f"flavor with the same name and type in the same "
f"'{flavor.workspace}' workspace owned by the same "
f"'{flavor.user}' user."
)
new_flavor = FlavorSchema(
name=flavor.name,
type=flavor.type,
source=flavor.source,
config_schema=flavor.config_schema,
integration=flavor.integration,
workspace_id=flavor.workspace,
user_id=flavor.user,
)
session.add(new_flavor)
session.commit()
return new_flavor.to_model()
def get_flavor(self, flavor_id: UUID) -> FlavorResponseModel:
"""Get a flavor by ID.
Args:
flavor_id: The ID of the flavor to fetch.
Returns:
The stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
with Session(self.engine) as session:
flavor_in_db = session.exec(
select(FlavorSchema).where(FlavorSchema.id == flavor_id)
).first()
if flavor_in_db is None:
raise KeyError(f"Flavor with ID {flavor_id} not found.")
return flavor_in_db.to_model()
def list_flavors(
self, flavor_filter_model: FlavorFilterModel
) -> Page[FlavorResponseModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
flavor_filter_model: All filter parameters including pagination
params
Returns:
List of all the stack component flavors matching the given criteria.
"""
with Session(self.engine) as session:
query = select(FlavorSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=FlavorSchema,
filter_model=flavor_filter_model,
)
@track(AnalyticsEvent.DELETED_FLAVOR)
def delete_flavor(self, flavor_id: UUID) -> None:
"""Delete a flavor.
Args:
flavor_id: The id of the flavor to delete.
Raises:
KeyError: if the flavor doesn't exist.
IllegalOperationError: if the flavor is used by a stack component.
"""
with Session(self.engine) as session:
try:
flavor_in_db = session.exec(
select(FlavorSchema).where(FlavorSchema.id == flavor_id)
).one()
components_of_flavor = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.flavor == flavor_in_db.name
)
).all()
if len(components_of_flavor) > 0:
raise IllegalOperationError(
f"Stack Component `{flavor_in_db.name}` of type "
f"`{flavor_in_db.type} cannot be "
f"deleted as it is used by"
f"{len(components_of_flavor)} "
f"components. Before deleting this "
f"flavor, make sure to delete all "
f"associated components."
)
else:
session.delete(flavor_in_db)
except NoResultFound as error:
raise KeyError from error
session.commit()
# -----
# Users
# -----
@track(AnalyticsEvent.CREATED_USER)
def create_user(self, user: UserRequestModel) -> UserResponseModel:
"""Creates a new user.
Args:
user: User to be created.
Returns:
The newly created user.
Raises:
EntityExistsError: If a user with the given name already exists.
"""
with Session(self.engine) as session:
# Check if user with the given name already exists
existing_user = session.exec(
select(UserSchema).where(UserSchema.name == user.name)
).first()
if existing_user is not None:
raise EntityExistsError(
f"Unable to create user with name '{user.name}': "
f"Found existing user with this name."
)
# Create the user
new_user = UserSchema.from_request(user)
session.add(new_user)
session.commit()
return new_user.to_model()
def get_user(
self,
user_name_or_id: Optional[Union[str, UUID]] = None,
include_private: bool = False,
) -> UserResponseModel:
"""Gets a specific user, when no id is specified the active user is returned.
Raises a KeyError in case a user with that id does not exist.
Args:
user_name_or_id: The name or ID of the user to get.
include_private: Whether to include private user information
Returns:
The requested user, if it was found.
"""
if not user_name_or_id:
user_name_or_id = self._default_user_name
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
return user.to_model(include_private=include_private)
def get_auth_user(
self, user_name_or_id: Union[str, UUID]
) -> UserAuthModel:
"""Gets the auth model to a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
"""
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
return UserAuthModel(
id=user.id,
name=user.name,
full_name=user.full_name,
email_opted_in=user.email_opted_in,
active=user.active,
created=user.created,
updated=user.updated,
password=user.password,
activation_token=user.activation_token,
)
def list_users(
self, user_filter_model: UserFilterModel
) -> Page[UserResponseModel]:
"""List all users.
Args:
user_filter_model: All filter parameters including pagination
params.
Returns:
A list of all users.
"""
with Session(self.engine) as session:
query = select(UserSchema)
paged_user: Page[UserResponseModel] = self.filter_and_paginate(
session=session,
query=query,
table=UserSchema,
filter_model=user_filter_model,
)
return paged_user
@track(AnalyticsEvent.UPDATED_USER)
def update_user(
self, user_id: UUID, user_update: UserUpdateModel
) -> UserResponseModel:
"""Updates an existing user.
Args:
user_id: The id of the user to update.
user_update: The update to be applied to the user.
Returns:
The updated user.
Raises:
IllegalOperationError: If the request tries to update the username
for the default user account.
"""
with Session(self.engine) as session:
existing_user = self._get_user_schema(user_id, session=session)
if (
existing_user.name == self._default_user_name
and "name" in user_update.__fields_set__
and user_update.name != existing_user.name
):
raise IllegalOperationError(
"The username of the default user account cannot be "
"changed."
)
existing_user.update(user_update=user_update)
session.add(existing_user)
session.commit()
# Refresh the Model that was just created
session.refresh(existing_user)
return existing_user.to_model()
@track(AnalyticsEvent.DELETED_USER)
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
"""Deletes a user.
Args:
user_name_or_id: The name or the ID of the user to delete.
Raises:
IllegalOperationError: If the user is the default user account.
"""
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
if user.name == self._default_user_name:
raise IllegalOperationError(
"The default user account cannot be deleted."
)
session.delete(user)
session.commit()
# -----
# Teams
# -----
@track(AnalyticsEvent.CREATED_TEAM)
def create_team(self, team: TeamRequestModel) -> TeamResponseModel:
"""Creates a new team.
Args:
team: The team model to create.
Returns:
The newly created team.
Raises:
EntityExistsError: If a team with the given name already exists.
"""
with Session(self.engine) as session:
# Check if team with the given name already exists
existing_team = session.exec(
select(TeamSchema).where(TeamSchema.name == team.name)
).first()
if existing_team is not None:
raise EntityExistsError(
f"Unable to create team with name '{team.name}': "
f"Found existing team with this name."
)
defined_users = []
if team.users:
# Get the Schemas of all users mentioned
filters = [
(UserSchema.id == user_id) for user_id in team.users
]
defined_users = session.exec(
select(UserSchema).where(or_(*filters))
).all()
# Create the team
new_team = TeamSchema(name=team.name, users=defined_users)
session.add(new_team)
session.commit()
return new_team.to_model()
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamResponseModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
return team.to_model()
def list_teams(
self, team_filter_model: TeamFilterModel
) -> Page[TeamResponseModel]:
"""List all teams matching the given filter criteria.
Args:
team_filter_model: All filter parameters including pagination
params.
Returns:
A list of all teams matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(TeamSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=TeamSchema,
filter_model=team_filter_model,
)
@track(AnalyticsEvent.UPDATED_TEAM)
def update_team(
self, team_id: UUID, team_update: TeamUpdateModel
) -> TeamResponseModel:
"""Update an existing team.
Args:
team_id: The ID of the team to be updated.
team_update: The update to be applied to the team.
Returns:
The updated team.
Raises:
KeyError: if the team does not exist.
"""
with Session(self.engine) as session:
existing_team = session.exec(
select(TeamSchema).where(TeamSchema.id == team_id)
).first()
if existing_team is None:
raise KeyError(
f"Unable to update team with id "
f"'{team_id}': Found no"
f"existing teams with this id."
)
# Update the team
existing_team.update(team_update=team_update)
existing_team.users = []
if "users" in team_update.__fields_set__ and team_update.users:
for user in team_update.users:
existing_team.users.append(
self._get_user_schema(
user_name_or_id=user, session=session
)
)
session.add(existing_team)
session.commit()
# Refresh the Model that was just created
session.refresh(existing_team)
return existing_team.to_model()
@track(AnalyticsEvent.DELETED_TEAM)
def delete_team(self, team_name_or_id: Union[str, UUID]) -> None:
"""Deletes a team.
Args:
team_name_or_id: Name or ID of the team to delete.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
session.delete(team)
session.commit()
# -----
# Roles
# -----
@track(AnalyticsEvent.CREATED_ROLE)
def create_role(self, role: RoleRequestModel) -> RoleResponseModel:
"""Creates a new role.
Args:
role: The role model to create.
Returns:
The newly created role.
Raises:
EntityExistsError: If a role with the given name already exists.
"""
with Session(self.engine) as session:
# Check if role with the given name already exists
existing_role = session.exec(
select(RoleSchema).where(RoleSchema.name == role.name)
).first()
if existing_role is not None:
raise EntityExistsError(
f"Unable to create role '{role.name}': Role already exists."
)
# Create role
role_schema = RoleSchema.from_request(role)
session.add(role_schema)
session.commit()
# Add all permissions
for p in role.permissions:
session.add(
RolePermissionSchema(name=p, role_id=role_schema.id)
)
session.commit()
return role_schema.to_model()
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleResponseModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
"""
with Session(self.engine) as session:
role = self._get_role_schema(role_name_or_id, session=session)
return role.to_model()
def list_roles(
self, role_filter_model: RoleFilterModel
) -> Page[RoleResponseModel]:
"""List all roles matching the given filter criteria.
Args:
role_filter_model: All filter parameters including pagination
params.
Returns:
A list of all roles matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(RoleSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=RoleSchema,
filter_model=role_filter_model,
)
@track(AnalyticsEvent.UPDATED_ROLE)
def update_role(
self, role_id: UUID, role_update: RoleUpdateModel
) -> RoleResponseModel:
"""Update an existing role.
Args:
role_id: The ID of the role to be updated.
role_update: The update to be applied to the role.
Returns:
The updated role.
Raises:
KeyError: if the role does not exist.
IllegalOperationError: if the role is a system role.
"""
with Session(self.engine) as session:
existing_role = session.exec(
select(RoleSchema).where(RoleSchema.id == role_id)
).first()
if existing_role is None:
raise KeyError(
f"Unable to update role with id "
f"'{role_id}': Found no"
f"existing roles with this id."
)
if existing_role.name in [DEFAULT_ADMIN_ROLE, DEFAULT_GUEST_ROLE]:
raise IllegalOperationError(
f"The built-in role '{existing_role.name}' cannot be "
f"updated."
)
# The relationship table for roles behaves different from the other
# ones. As such the required updates on the permissions have to be
# done manually.
if "permissions" in role_update.__fields_set__:
existing_permissions = {
p.name for p in existing_role.permissions
}
diff = existing_permissions.symmetric_difference(
role_update.permissions
)
for permission in diff:
if permission not in role_update.permissions:
permission_to_delete = session.exec(
select(RolePermissionSchema)
.where(RolePermissionSchema.name == permission)
.where(
RolePermissionSchema.role_id
== existing_role.id
)
).one_or_none()
session.delete(permission_to_delete)
elif permission not in existing_permissions:
session.add(
RolePermissionSchema(
name=permission, role_id=existing_role.id
)
)
# Update the role
existing_role.update(role_update=role_update)
session.add(existing_role)
session.commit()
session.commit()
# Refresh the Model that was just created
session.refresh(existing_role)
return existing_role.to_model()
@track(AnalyticsEvent.DELETED_ROLE)
def delete_role(self, role_name_or_id: Union[str, UUID]) -> None:
"""Deletes a role.
Args:
role_name_or_id: Name or ID of the role to delete.
Raises:
IllegalOperationError: If the role is still assigned to users or
the role is one of the built-in roles.
"""
with Session(self.engine) as session:
role = self._get_role_schema(role_name_or_id, session=session)
if role.name in [DEFAULT_ADMIN_ROLE, DEFAULT_GUEST_ROLE]:
raise IllegalOperationError(
f"The built-in role '{role.name}' cannot be deleted."
)
user_role = session.exec(
select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.role_id == role.id
)
).all()
team_role = session.exec(
select(TeamRoleAssignmentSchema).where(
TeamRoleAssignmentSchema.role_id == role.id
)
).all()
if len(user_role) > 0 or len(team_role) > 0:
raise IllegalOperationError(
f"Role `{role.name}` of type cannot be "
f"deleted as it is in use by multiple users and teams. "
f"Before deleting this role make sure to remove all "
f"instances where this role is used."
)
else:
# Delete role
session.delete(role)
session.commit()
# ----------------
# Role assignments
# ----------------
def list_user_role_assignments(
self, user_role_assignment_filter_model: UserRoleAssignmentFilterModel
) -> Page[UserRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
user_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(UserRoleAssignmentSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=UserRoleAssignmentSchema,
filter_model=user_role_assignment_filter_model,
)
def create_user_role_assignment(
self, user_role_assignment: UserRoleAssignmentRequestModel
) -> UserRoleAssignmentResponseModel:
"""Assigns a role to a user or team, scoped to a specific workspace.
Args:
user_role_assignment: The role assignment to create.
Returns:
The created role assignment.
Raises:
EntityExistsError: if the role assignment already exists.
"""
with Session(self.engine) as session:
role = self._get_role_schema(
user_role_assignment.role, session=session
)
workspace: Optional[WorkspaceSchema] = None
if user_role_assignment.workspace:
workspace = self._get_workspace_schema(
user_role_assignment.workspace, session=session
)
user = self._get_user_schema(
user_role_assignment.user, session=session
)
query = select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.user_id == user.id,
UserRoleAssignmentSchema.role_id == role.id,
)
if workspace is not None:
query = query.where(
UserRoleAssignmentSchema.workspace_id == workspace.id
)
existing_role_assignment = session.exec(query).first()
if existing_role_assignment is not None:
raise EntityExistsError(
f"Unable to assign role '{role.name}' to user "
f"'{user.name}': Role already assigned in this workspace."
)
role_assignment = UserRoleAssignmentSchema(
role_id=role.id,
user_id=user.id,
workspace_id=workspace.id if workspace else None,
role=role,
user=user,
workspace=workspace,
)
session.add(role_assignment)
session.commit()
return role_assignment.to_model()
def get_user_role_assignment(
self, user_role_assignment_id: UUID
) -> UserRoleAssignmentResponseModel:
"""Gets a role assignment by ID.
Args:
user_role_assignment_id: ID of the role assignment to get.
Returns:
The role assignment.
Raises:
KeyError: If the role assignment does not exist.
"""
with Session(self.engine) as session:
user_role = session.exec(
select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.id == user_role_assignment_id
)
).one_or_none()
if user_role:
return user_role.to_model()
else:
raise KeyError(
f"Unable to get user role assignment with ID "
f"'{user_role_assignment_id}': No user role assignment "
f"with this ID found."
)
def delete_user_role_assignment(
self, user_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
user_role_assignment_id: The ID of the specific role assignment.
Raises:
KeyError: If the role assignment does not exist.
"""
with Session(self.engine) as session:
user_role = session.exec(
select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.id == user_role_assignment_id
)
).one_or_none()
if not user_role:
raise KeyError(
f"No user role assignment with id "
f"{user_role_assignment_id} exists."
)
session.delete(user_role)
session.commit()
# ---------------------
# Team Role assignments
# ---------------------
def create_team_role_assignment(
self, team_role_assignment: TeamRoleAssignmentRequestModel
) -> TeamRoleAssignmentResponseModel:
"""Creates a new team role assignment.
Args:
team_role_assignment: The role assignment model to create.
Returns:
The newly created role assignment.
Raises:
EntityExistsError: If the role assignment already exists.
"""
with Session(self.engine) as session:
role = self._get_role_schema(
team_role_assignment.role, session=session
)
workspace: Optional[WorkspaceSchema] = None
if team_role_assignment.workspace:
workspace = self._get_workspace_schema(
team_role_assignment.workspace, session=session
)
team = self._get_team_schema(
team_role_assignment.team, session=session
)
query = select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.user_id == team.id,
UserRoleAssignmentSchema.role_id == role.id,
)
if workspace is not None:
query = query.where(
UserRoleAssignmentSchema.workspace_id == workspace.id
)
existing_role_assignment = session.exec(query).first()
if existing_role_assignment is not None:
raise EntityExistsError(
f"Unable to assign role '{role.name}' to team "
f"'{team.name}': Role already assigned in this workspace."
)
role_assignment = TeamRoleAssignmentSchema(
role_id=role.id,
team_id=team.id,
workspace_id=workspace.id if workspace else None,
role=role,
team=team,
workspace=workspace,
)
session.add(role_assignment)
session.commit()
return role_assignment.to_model()
def get_team_role_assignment(
self, team_role_assignment_id: UUID
) -> TeamRoleAssignmentResponseModel:
"""Gets a specific role assignment.
Args:
team_role_assignment_id: ID of the role assignment to get.
Returns:
The requested role assignment.
Raises:
KeyError: If no role assignment with the given ID exists.
"""
with Session(self.engine) as session:
team_role = session.exec(
select(TeamRoleAssignmentSchema).where(
TeamRoleAssignmentSchema.id == team_role_assignment_id
)
).one_or_none()
if team_role:
return team_role.to_model()
else:
raise KeyError(
f"Unable to get team role assignment with ID "
f"'{team_role_assignment_id}': No team role assignment "
f"with this ID found."
)
def delete_team_role_assignment(
self, team_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
team_role_assignment_id: The ID of the specific role assignment
Raises:
KeyError: If the role assignment does not exist.
"""
with Session(self.engine) as session:
team_role = session.exec(
select(TeamRoleAssignmentSchema).where(
TeamRoleAssignmentSchema.id == team_role_assignment_id
)
).one_or_none()
if not team_role:
raise KeyError(
f"No team role assignment with id "
f"{team_role_assignment_id} exists."
)
session.delete(team_role)
session.commit()
def list_team_role_assignments(
self, team_role_assignment_filter_model: TeamRoleAssignmentFilterModel
) -> Page[TeamRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
team_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(TeamRoleAssignmentSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=TeamRoleAssignmentSchema,
filter_model=team_role_assignment_filter_model,
)
# --------
# Workspaces
# --------
@track(AnalyticsEvent.CREATED_WORKSPACE)
def create_workspace(
self, workspace: WorkspaceRequestModel
) -> WorkspaceResponseModel:
"""Creates a new workspace.
Args:
workspace: The workspace to create.
Returns:
The newly created workspace.
Raises:
EntityExistsError: If a workspace with the given name already exists.
"""
with Session(self.engine) as session:
# Check if workspace with the given name already exists
existing_workspace = session.exec(
select(WorkspaceSchema).where(
WorkspaceSchema.name == workspace.name
)
).first()
if existing_workspace is not None:
raise EntityExistsError(
f"Unable to create workspace {workspace.name}: "
"A workspace with this name already exists."
)
# Create the workspace
new_workspace = WorkspaceSchema.from_request(workspace)
session.add(new_workspace)
session.commit()
# Explicitly refresh the new_workspace schema
session.refresh(new_workspace)
return new_workspace.to_model()
def get_workspace(
self, workspace_name_or_id: Union[str, UUID]
) -> WorkspaceResponseModel:
"""Get an existing workspace by name or ID.
Args:
workspace_name_or_id: Name or ID of the workspace to get.
Returns:
The requested workspace if one was found.
"""
with Session(self.engine) as session:
workspace = self._get_workspace_schema(
workspace_name_or_id, session=session
)
return workspace.to_model()
def list_workspaces(
self, workspace_filter_model: WorkspaceFilterModel
) -> Page[WorkspaceResponseModel]:
"""List all workspace matching the given filter criteria.
Args:
workspace_filter_model: All filter parameters including pagination
params.
Returns:
A list of all workspace matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(WorkspaceSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=WorkspaceSchema,
filter_model=workspace_filter_model,
)
@track(AnalyticsEvent.UPDATED_WORKSPACE)
def update_workspace(
self, workspace_id: UUID, workspace_update: WorkspaceUpdateModel
) -> WorkspaceResponseModel:
"""Update an existing workspace.
Args:
workspace_id: The ID of the workspace to be updated.
workspace_update: The update to be applied to the workspace.
Returns:
The updated workspace.
Raises:
IllegalOperationError: if the workspace is the default workspace.
KeyError: if the workspace does not exist.
"""
with Session(self.engine) as session:
existing_workspace = session.exec(
select(WorkspaceSchema).where(
WorkspaceSchema.id == workspace_id
)
).first()
if existing_workspace is None:
raise KeyError(
f"Unable to update workspace with id "
f"'{workspace_id}': Found no"
f"existing workspaces with this id."
)
if (
existing_workspace.name == self._default_workspace_name
and "name" in workspace_update.__fields_set__
and workspace_update.name != existing_workspace.name
):
raise IllegalOperationError(
"The name of the default workspace cannot be changed."
)
# Update the workspace
existing_workspace.update(workspace_update=workspace_update)
session.add(existing_workspace)
session.commit()
# Refresh the Model that was just created
session.refresh(existing_workspace)
return existing_workspace.to_model()
@track(AnalyticsEvent.DELETED_WORKSPACE)
def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
"""Deletes a workspace.
Args:
workspace_name_or_id: Name or ID of the workspace to delete.
Raises:
IllegalOperationError: If the workspace is the default workspace.
"""
with Session(self.engine) as session:
# Check if workspace with the given name exists
workspace = self._get_workspace_schema(
workspace_name_or_id, session=session
)
if workspace.name == self._default_workspace_name:
raise IllegalOperationError(
"The default workspace cannot be deleted."
)
session.delete(workspace)
session.commit()
# ---------
# Pipelines
# ---------
@track(AnalyticsEvent.CREATE_PIPELINE)
def create_pipeline(
self,
pipeline: PipelineRequestModel,
) -> PipelineResponseModel:
"""Creates a new pipeline in a workspace.
Args:
pipeline: The pipeline to create.
Returns:
The newly created pipeline.
Raises:
EntityExistsError: If an identical pipeline already exists.
"""
with Session(self.engine) as session:
# Check if pipeline with the given name already exists
existing_pipeline = session.exec(
select(PipelineSchema)
.where(PipelineSchema.name == pipeline.name)
.where(PipelineSchema.workspace_id == pipeline.workspace)
).first()
if existing_pipeline is not None:
raise EntityExistsError(
f"Unable to create pipeline in workspace "
f"'{pipeline.workspace}': A pipeline with this name "
f"already exists."
)
# Create the pipeline
new_pipeline = PipelineSchema(
name=pipeline.name,
workspace_id=pipeline.workspace,
user_id=pipeline.user,
docstring=pipeline.docstring,
spec=pipeline.spec.json(sort_keys=True),
)
session.add(new_pipeline)
session.commit()
# Refresh the Model that was just created
session.refresh(new_pipeline)
return new_pipeline.to_model()
def get_pipeline(self, pipeline_id: UUID) -> PipelineResponseModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
with Session(self.engine) as session:
# Check if pipeline with the given ID exists
pipeline = session.exec(
select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
).first()
if pipeline is None:
raise KeyError(
f"Unable to get pipeline with ID '{pipeline_id}': "
"No pipeline with this ID found."
)
return pipeline.to_model()
def list_pipelines(
self, pipeline_filter_model: PipelineFilterModel
) -> Page[PipelineResponseModel]:
"""List all pipelines matching the given filter criteria.
Args:
pipeline_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipelines matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(PipelineSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=PipelineSchema,
filter_model=pipeline_filter_model,
)
@track(AnalyticsEvent.UPDATE_PIPELINE)
def update_pipeline(
self,
pipeline_id: UUID,
pipeline_update: PipelineUpdateModel,
) -> PipelineResponseModel:
"""Updates a pipeline.
Args:
pipeline_id: The ID of the pipeline to be updated.
pipeline_update: The update to be applied.
Returns:
The updated pipeline.
Raises:
KeyError: if the pipeline doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline with the given ID exists
existing_pipeline = session.exec(
select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
).first()
if existing_pipeline is None:
raise KeyError(
f"Unable to update pipeline with ID {pipeline_id}: "
f"No pipeline with this ID found."
)
# Update the pipeline
existing_pipeline.update(pipeline_update)
session.add(existing_pipeline)
session.commit()
return existing_pipeline.to_model()
@track(AnalyticsEvent.DELETE_PIPELINE)
def delete_pipeline(self, pipeline_id: UUID) -> None:
"""Deletes a pipeline.
Args:
pipeline_id: The ID of the pipeline to delete.
Raises:
KeyError: if the pipeline doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline with the given ID exists
pipeline = session.exec(
select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
).first()
if pipeline is None:
raise KeyError(
f"Unable to delete pipeline with ID {pipeline_id}: "
f"No pipeline with this ID found."
)
session.delete(pipeline)
session.commit()
# ---------
# Schedules
# ---------
def create_schedule(
self, schedule: ScheduleRequestModel
) -> ScheduleResponseModel:
"""Creates a new schedule.
Args:
schedule: The schedule to create.
Returns:
The newly created schedule.
"""
with Session(self.engine) as session:
new_schedule = ScheduleSchema.from_create_model(model=schedule)
session.add(new_schedule)
session.commit()
return new_schedule.to_model()
def get_schedule(self, schedule_id: UUID) -> ScheduleResponseModel:
"""Get a schedule with a given ID.
Args:
schedule_id: ID of the schedule.
Returns:
The schedule.
Raises:
KeyError: if the schedule does not exist.
"""
with Session(self.engine) as session:
# Check if schedule with the given ID exists
schedule = session.exec(
select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
).first()
if schedule is None:
raise KeyError(
f"Unable to get schedule with ID '{schedule_id}': "
"No schedule with this ID found."
)
return schedule.to_model()
def list_schedules(
self, schedule_filter_model: ScheduleFilterModel
) -> Page[ScheduleResponseModel]:
"""List all schedules in the workspace.
Args:
schedule_filter_model: All filter parameters including pagination
params
Returns:
A list of schedules.
"""
with Session(self.engine) as session:
query = select(ScheduleSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=ScheduleSchema,
filter_model=schedule_filter_model,
)
def update_schedule(
self,
schedule_id: UUID,
schedule_update: ScheduleUpdateModel,
) -> ScheduleResponseModel:
"""Updates a schedule.
Args:
schedule_id: The ID of the schedule to be updated.
schedule_update: The update to be applied.
Returns:
The updated schedule.
Raises:
KeyError: if the schedule doesn't exist.
"""
with Session(self.engine) as session:
# Check if schedule with the given ID exists
existing_schedule = session.exec(
select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
).first()
if existing_schedule is None:
raise KeyError(
f"Unable to update schedule with ID {schedule_id}: "
f"No schedule with this ID found."
)
# Update the schedule
existing_schedule = existing_schedule.from_update_model(
schedule_update
)
session.add(existing_schedule)
session.commit()
return existing_schedule.to_model()
def delete_schedule(self, schedule_id: UUID) -> None:
"""Deletes a schedule.
Args:
schedule_id: The ID of the schedule to delete.
Raises:
KeyError: if the schedule doesn't exist.
"""
with Session(self.engine) as session:
# Check if schedule with the given ID exists
schedule = session.exec(
select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
).first()
if schedule is None:
raise KeyError(
f"Unable to delete schedule with ID {schedule_id}: "
f"No schedule with this ID found."
)
# Delete the schedule
session.delete(schedule)
session.commit()
# --------------
# Pipeline runs
# --------------
def create_run(
self, pipeline_run: PipelineRunRequestModel
) -> PipelineRunResponseModel:
"""Creates a pipeline run.
Args:
pipeline_run: The pipeline run to create.
Returns:
The created pipeline run.
Raises:
EntityExistsError: If an identical pipeline run already exists.
"""
with Session(self.engine) as session:
# Check if pipeline run with same name already exists.
existing_domain_run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.name == pipeline_run.name
)
).first()
if existing_domain_run is not None:
raise EntityExistsError(
f"Unable to create pipeline run: A pipeline run with name "
f"'{pipeline_run.name}' already exists."
)
# Check if pipeline run with same ID already exists.
existing_id_run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.id == pipeline_run.id
)
).first()
if existing_id_run is not None:
raise EntityExistsError(
f"Unable to create pipeline run: A pipeline run with ID "
f"'{pipeline_run.id}' already exists."
)
# Query stack to ensure it exists in the DB
stack_id = None
if pipeline_run.stack is not None:
stack_id = session.exec(
select(StackSchema.id).where(
StackSchema.id == pipeline_run.stack
)
).first()
if stack_id is None:
logger.warning(
f"No stack found for this run. "
f"Creating pipeline run '{pipeline_run.name}' without "
"linked stack."
)
# Query pipeline to ensure it exists in the DB
pipeline_id = None
if pipeline_run.pipeline is not None:
pipeline_id = session.exec(
select(PipelineSchema.id).where(
PipelineSchema.id == pipeline_run.pipeline
)
).first()
if pipeline_id is None:
logger.warning(
f"No pipeline found. Creating pipeline run "
f"'{pipeline_run.name}' as unlisted run."
)
# Create the pipeline run
new_run = PipelineRunSchema.from_request(pipeline_run)
session.add(new_run)
session.commit()
return new_run.to_model()
def get_run(
self, run_name_or_id: Union[str, UUID]
) -> PipelineRunResponseModel:
"""Gets a pipeline run.
Args:
run_name_or_id: The name or ID of the pipeline run to get.
Returns:
The pipeline run.
"""
with Session(self.engine) as session:
run = self._get_run_schema(run_name_or_id, session=session)
return run.to_model()
def get_or_create_run(
self, pipeline_run: PipelineRunRequestModel
) -> Tuple[PipelineRunResponseModel, bool]:
"""Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned.
Otherwise, a new run is created.
Args:
pipeline_run: The pipeline run to get or create.
Returns:
The pipeline run, and a boolean indicating whether the run was
created or not.
"""
# We want to have the 'create' statement in the try block since running
# it first will reduce concurrency issues.
try:
return self.create_run(pipeline_run), True
except EntityExistsError:
# Currently, an `EntityExistsError` is raised if either the run ID
# or the run name already exists. Therefore, we need to have another
# try block since getting the run by ID might still fail.
try:
return self.get_run(pipeline_run.id), False
except KeyError:
return self.get_run(pipeline_run.name), False
def list_runs(
self, runs_filter_model: PipelineRunFilterModel
) -> Page[PipelineRunResponseModel]:
"""List all pipeline runs matching the given filter criteria.
Args:
runs_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipeline runs matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(PipelineRunSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=PipelineRunSchema,
filter_model=runs_filter_model,
)
def update_run(
self, run_id: UUID, run_update: PipelineRunUpdateModel
) -> PipelineRunResponseModel:
"""Updates a pipeline run.
Args:
run_id: The ID of the pipeline run to update.
run_update: The update to be applied to the pipeline run.
Returns:
The updated pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline run with the given ID exists
existing_run = session.exec(
select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
).first()
if existing_run is None:
raise KeyError(
f"Unable to update pipeline run with ID {run_id}: "
f"No pipeline run with this ID found."
)
# Update the pipeline run
existing_run.update(run_update=run_update)
session.add(existing_run)
session.commit()
session.refresh(existing_run)
return existing_run.to_model()
def delete_run(self, run_id: UUID) -> None:
"""Deletes a pipeline run.
Args:
run_id: The ID of the pipeline run to delete.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline run with the given ID exists
existing_run = session.exec(
select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
).first()
if existing_run is None:
raise KeyError(
f"Unable to delete pipeline run with ID {run_id}: "
f"No pipeline run with this ID found."
)
# Delete the pipeline run
session.delete(existing_run)
session.commit()
# ------------------
# Pipeline run steps
# ------------------
def create_run_step(
self, step_run: StepRunRequestModel
) -> StepRunResponseModel:
"""Creates a step run.
Args:
step_run: The step run to create.
Returns:
The created step run.
Raises:
EntityExistsError: if the step run already exists.
KeyError: if the pipeline run doesn't exist.
"""
with Session(self.engine) as session:
# Check if the pipeline run exists
run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.id == step_run.pipeline_run_id
)
).first()
if run is None:
raise KeyError(
f"Unable to create step '{step_run.name}': No pipeline run "
f"with ID '{step_run.pipeline_run_id}' found."
)
# Check if the step name already exists in the pipeline run
existing_step_run = session.exec(
select(StepRunSchema)
.where(StepRunSchema.name == step_run.name)
.where(
StepRunSchema.pipeline_run_id == step_run.pipeline_run_id
)
).first()
if existing_step_run is not None:
raise EntityExistsError(
f"Unable to create step '{step_run.name}': A step with this "
f"name already exists in the pipeline run with ID "
f"'{step_run.pipeline_run_id}'."
)
# Create the step
step_schema = StepRunSchema.from_request(step_run)
session.add(step_schema)
# Save parent step IDs into the database.
for parent_step_id in step_run.parent_step_ids:
self._set_run_step_parent_step(
child_id=step_schema.id,
parent_id=parent_step_id,
session=session,
)
# Save input artifact IDs into the database.
for input_name, artifact_id in step_run.input_artifacts.items():
self._set_run_step_input_artifact(
run_step_id=step_schema.id,
artifact_id=artifact_id,
name=input_name,
session=session,
)
# Save output artifact IDs into the database.
for output_name, artifact_id in step_run.output_artifacts.items():
self._set_run_step_output_artifact(
step_run_id=step_schema.id,
artifact_id=artifact_id,
name=output_name,
session=session,
)
session.commit()
return self._run_step_schema_to_model(step_schema)
def _set_run_step_parent_step(
self, child_id: UUID, parent_id: UUID, session: Session
) -> None:
"""Sets the parent step run for a step run.
Args:
child_id: The ID of the child step run to set the parent for.
parent_id: The ID of the parent step run to set a child for.
session: The database session to use.
Raises:
KeyError: if the child step run or parent step run doesn't exist.
"""
# Check if the child step exists.
child_step_run = session.exec(
select(StepRunSchema).where(StepRunSchema.id == child_id)
).first()
if child_step_run is None:
raise KeyError(
f"Unable to set parent step for step with ID "
f"{child_id}: No step with this ID found."
)
# Check if the parent step exists.
parent_step_run = session.exec(
select(StepRunSchema).where(StepRunSchema.id == parent_id)
).first()
if parent_step_run is None:
raise KeyError(
f"Unable to set parent step for step with ID "
f"{child_id}: No parent step with ID {parent_id} "
"found."
)
# Check if the parent step is already set.
assignment = session.exec(
select(StepRunParentsSchema)
.where(StepRunParentsSchema.child_id == child_id)
.where(StepRunParentsSchema.parent_id == parent_id)
).first()
if assignment is not None:
return
# Save the parent step assignment in the database.
assignment = StepRunParentsSchema(
child_id=child_id, parent_id=parent_id
)
session.add(assignment)
def _set_run_step_input_artifact(
self, run_step_id: UUID, artifact_id: UUID, name: str, session: Session
) -> None:
"""Sets an artifact as an input of a step run.
Args:
run_step_id: The ID of the step run.
artifact_id: The ID of the artifact.
name: The name of the input in the step run.
session: The database session to use.
Raises:
KeyError: if the step run or artifact doesn't exist.
"""
# Check if the step exists.
step_run = session.exec(
select(StepRunSchema).where(StepRunSchema.id == run_step_id)
).first()
if step_run is None:
raise KeyError(
f"Unable to set input artifact: No step run with ID "
f"'{run_step_id}' found."
)
# Check if the artifact exists.
artifact = session.exec(
select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
).first()
if artifact is None:
raise KeyError(
f"Unable to set input artifact: No artifact with ID "
f"'{artifact_id}' found."
)
# Check if the input is already set.
assignment = session.exec(
select(StepRunInputArtifactSchema)
.where(StepRunInputArtifactSchema.step_id == run_step_id)
.where(StepRunInputArtifactSchema.artifact_id == artifact_id)
).first()
if assignment is not None:
return
# Save the input assignment in the database.
assignment = StepRunInputArtifactSchema(
step_id=run_step_id, artifact_id=artifact_id, name=name
)
session.add(assignment)
def _set_run_step_output_artifact(
self,
step_run_id: UUID,
artifact_id: UUID,
name: str,
session: Session,
) -> None:
"""Sets an artifact as an output of a step run.
Args:
step_run_id: The ID of the step run.
artifact_id: The ID of the artifact.
name: The name of the output in the step run.
session: The database session to use.
Raises:
KeyError: if the step run or artifact doesn't exist.
"""
# Check if the step exists.
step_run = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_run_id)
).first()
if step_run is None:
raise KeyError(
f"Unable to set output artifact: No step run with ID "
f"'{step_run_id}' found."
)
# Check if the artifact exists.
artifact = session.exec(
select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
).first()
if artifact is None:
raise KeyError(
f"Unable to set output artifact: No artifact with ID "
f"'{artifact_id}' found."
)
# Check if the output is already set.
assignment = session.exec(
select(StepRunOutputArtifactSchema)
.where(StepRunOutputArtifactSchema.step_id == step_run_id)
.where(StepRunOutputArtifactSchema.artifact_id == artifact_id)
).first()
if assignment is not None:
return
# Save the output assignment in the database.
assignment = StepRunOutputArtifactSchema(
step_id=step_run_id,
artifact_id=artifact_id,
name=name,
)
session.add(assignment)
def get_run_step(self, step_run_id: UUID) -> StepRunResponseModel:
"""Get a step run by ID.
Args:
step_run_id: The ID of the step run to get.
Returns:
The step run.
Raises:
KeyError: if the step run doesn't exist.
"""
with Session(self.engine) as session:
step_run = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_run_id)
).first()
if step_run is None:
raise KeyError(
f"Unable to get step run with ID {step_run_id}: No step "
"run with this ID found."
)
return self._run_step_schema_to_model(step_run)
def _run_step_schema_to_model(
self, step_run: StepRunSchema
) -> StepRunResponseModel:
"""Converts a run step schema to a step model.
Args:
step_run: The run step schema to convert.
Returns:
The run step model.
"""
with Session(self.engine) as session:
# Get parent steps.
parent_steps = session.exec(
select(StepRunSchema)
.where(StepRunParentsSchema.child_id == step_run.id)
.where(StepRunParentsSchema.parent_id == StepRunSchema.id)
).all()
parent_step_ids = [parent_step.id for parent_step in parent_steps]
# Get input artifacts.
input_artifact_list = session.exec(
select(
ArtifactSchema,
StepRunInputArtifactSchema.name,
)
.where(
ArtifactSchema.id == StepRunInputArtifactSchema.artifact_id
)
.where(StepRunInputArtifactSchema.step_id == step_run.id)
).all()
input_artifacts = {
input_name: self._artifact_schema_to_model(artifact)
for (artifact, input_name) in input_artifact_list
}
# Get output artifacts.
output_artifact_list = session.exec(
select(
ArtifactSchema,
StepRunOutputArtifactSchema.name,
)
.where(
ArtifactSchema.id
== StepRunOutputArtifactSchema.artifact_id
)
.where(StepRunOutputArtifactSchema.step_id == step_run.id)
).all()
output_artifacts = {
output_name: self._artifact_schema_to_model(artifact)
for (artifact, output_name) in output_artifact_list
}
# Convert to model.
return step_run.to_model(
parent_step_ids=parent_step_ids,
input_artifacts=input_artifacts,
output_artifacts=output_artifacts,
)
def list_run_steps(
self, step_run_filter_model: StepRunFilterModel
) -> Page[StepRunResponseModel]:
"""List all step runs matching the given filter criteria.
Args:
step_run_filter_model: All filter parameters including pagination
params.
Returns:
A list of all step runs matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(StepRunSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=StepRunSchema,
filter_model=step_run_filter_model,
custom_schema_to_model_conversion=self._run_step_schema_to_model,
)
def update_run_step(
self,
step_run_id: UUID,
step_run_update: StepRunUpdateModel,
) -> StepRunResponseModel:
"""Updates a step run.
Args:
step_run_id: The ID of the step to update.
step_run_update: The update to be applied to the step.
Returns:
The updated step run.
Raises:
KeyError: if the step run doesn't exist.
"""
with Session(self.engine) as session:
# Check if the step exists
existing_step_run = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_run_id)
).first()
if existing_step_run is None:
raise KeyError(
f"Unable to update step with ID {step_run_id}: "
f"No step with this ID found."
)
# Update the step
existing_step_run.update(step_run_update)
session.add(existing_step_run)
# Update the output artifacts.
for name, artifact_id in step_run_update.output_artifacts.items():
self._set_run_step_output_artifact(
step_run_id=step_run_id,
artifact_id=artifact_id,
name=name,
session=session,
)
# Input artifacts and parent steps cannot be updated after the
# step has been created.
session.commit()
session.refresh(existing_step_run)
return self._run_step_schema_to_model(existing_step_run)
# ---------
# Artifacts
# ---------
def create_artifact(
self, artifact: ArtifactRequestModel
) -> ArtifactResponseModel:
"""Creates an artifact.
Args:
artifact: The artifact to create.
Returns:
The created artifact.
"""
with Session(self.engine) as session:
artifact_schema = ArtifactSchema.from_request(artifact)
session.add(artifact_schema)
session.commit()
return self._artifact_schema_to_model(artifact_schema)
def _artifact_schema_to_model(
self, artifact_schema: ArtifactSchema
) -> ArtifactResponseModel:
"""Converts an artifact schema to a model.
Args:
artifact_schema: The artifact schema to convert.
Returns:
The converted artifact model.
"""
# Find the producer step run ID.
with Session(self.engine) as session:
producer_step_run_id = session.exec(
select(StepRunOutputArtifactSchema.step_id)
.where(
StepRunOutputArtifactSchema.artifact_id
== artifact_schema.id
)
.where(StepRunOutputArtifactSchema.step_id == StepRunSchema.id)
.where(StepRunSchema.status != ExecutionStatus.CACHED)
).first()
# Convert the artifact schema to a model.
return artifact_schema.to_model(
producer_step_run_id=producer_step_run_id
)
def get_artifact(self, artifact_id: UUID) -> ArtifactResponseModel:
"""Gets an artifact.
Args:
artifact_id: The ID of the artifact to get.
Returns:
The artifact.
Raises:
KeyError: if the artifact doesn't exist.
"""
with Session(self.engine) as session:
artifact = session.exec(
select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
).first()
if artifact is None:
raise KeyError(
f"Unable to get artifact with ID {artifact_id}: "
f"No artifact with this ID found."
)
return self._artifact_schema_to_model(artifact)
def list_artifacts(
self, artifact_filter_model: ArtifactFilterModel
) -> Page[ArtifactResponseModel]:
"""List all artifacts matching the given filter criteria.
Args:
artifact_filter_model: All filter parameters including pagination
params.
Returns:
A list of all artifacts matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(ArtifactSchema)
if artifact_filter_model.only_unused:
query = query.where(
ArtifactSchema.id.notin_( # type: ignore[attr-defined]
select(StepRunOutputArtifactSchema.artifact_id)
)
)
query = query.where(
ArtifactSchema.id.notin_( # type: ignore[attr-defined]
select(StepRunInputArtifactSchema.artifact_id)
)
)
return self.filter_and_paginate(
session=session,
query=query,
table=ArtifactSchema,
filter_model=artifact_filter_model,
custom_schema_to_model_conversion=self._artifact_schema_to_model,
)
def delete_artifact(self, artifact_id: UUID) -> None:
"""Deletes an artifact.
Args:
artifact_id: The ID of the artifact to delete.
Raises:
KeyError: if the artifact doesn't exist.
"""
with Session(self.engine) as session:
artifact = session.exec(
select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
).first()
if artifact is None:
raise KeyError(
f"Unable to delete artifact with ID {artifact_id}: "
f"No artifact with this ID found."
)
session.delete(artifact)
session.commit()
# ------------
# Run Metadata
# ------------
def create_run_metadata(
self, run_metadata: RunMetadataRequestModel
) -> RunMetadataResponseModel:
"""Creates run metadata.
Args:
run_metadata: The run metadata to create.
Returns:
The created run metadata.
"""
with Session(self.engine) as session:
run_metadata_schema = RunMetadataSchema.from_request(run_metadata)
session.add(run_metadata_schema)
session.commit()
return run_metadata_schema.to_model()
def list_run_metadata(
self,
run_metadata_filter_model: RunMetadataFilterModel,
) -> Page[RunMetadataResponseModel]:
"""List run metadata.
Args:
run_metadata_filter_model: All filter parameters including
pagination params.
Returns:
The run metadata.
"""
with Session(self.engine) as session:
query = select(RunMetadataSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=RunMetadataSchema,
filter_model=run_metadata_filter_model,
)
# =======================
# Internal helper methods
# =======================
@staticmethod
def _get_schema_by_name_or_id(
object_name_or_id: Union[str, UUID],
schema_class: Type[AnyNamedSchema],
schema_name: str,
session: Session,
) -> AnyNamedSchema:
"""Query a schema by its 'name' or 'id' field.
Args:
object_name_or_id: The name or ID of the object to query.
schema_class: The schema class to query. E.g., `WorkspaceSchema`.
schema_name: The name of the schema used for error messages.
E.g., "workspace".
session: The database session to use.
Returns:
The schema object.
Raises:
KeyError: if the object couldn't be found.
ValueError: if the schema_name isn't provided.
"""
if object_name_or_id is None:
raise ValueError(
f"Unable to get {schema_name}: No {schema_name} ID or name "
"provided."
)
if uuid_utils.is_valid_uuid(object_name_or_id):
filter_params = schema_class.id == object_name_or_id
error_msg = (
f"Unable to get {schema_name} with name or ID "
f"'{object_name_or_id}': No {schema_name} with this ID found."
)
else:
filter_params = schema_class.name == object_name_or_id
error_msg = (
f"Unable to get {schema_name} with name or ID "
f"'{object_name_or_id}': '{object_name_or_id}' is not a valid "
f" UUID and no {schema_name} with this name exists."
)
schema = session.exec(
select(schema_class).where(filter_params)
).first()
if schema is None:
raise KeyError(error_msg)
return schema
def _get_workspace_schema(
self,
workspace_name_or_id: Union[str, UUID],
session: Session,
) -> WorkspaceSchema:
"""Gets a workspace schema by name or ID.
This is a helper method that is used in various places to find the
workspace associated to some other object.
Args:
workspace_name_or_id: The name or ID of the workspace to get.
session: The database session to use.
Returns:
The workspace schema.
"""
return self._get_schema_by_name_or_id(
object_name_or_id=workspace_name_or_id,
schema_class=WorkspaceSchema,
schema_name="workspace",
session=session,
)
def _get_user_schema(
self,
user_name_or_id: Union[str, UUID],
session: Session,
) -> UserSchema:
"""Gets a user schema by name or ID.
This is a helper method that is used in various places to find the
user associated to some other object.
Args:
user_name_or_id: The name or ID of the user to get.
session: The database session to use.
Returns:
The user schema.
"""
return self._get_schema_by_name_or_id(
object_name_or_id=user_name_or_id,
schema_class=UserSchema,
schema_name="user",
session=session,
)
def _get_team_schema(
self,
team_name_or_id: Union[str, UUID],
session: Session,
) -> TeamSchema:
"""Gets a team schema by name or ID.
This is a helper method that is used in various places to find a team
by its name or ID.
Args:
team_name_or_id: The name or ID of the team to get.
session: The database session to use.
Returns:
The team schema.
"""
return self._get_schema_by_name_or_id(
object_name_or_id=team_name_or_id,
schema_class=TeamSchema,
schema_name="team",
session=session,
)
def _get_role_schema(
self,
role_name_or_id: Union[str, UUID],
session: Session,
) -> RoleSchema:
"""Gets a role schema by name or ID.
This is a helper method that is used in various places to find a role
by its name or ID.
Args:
role_name_or_id: The name or ID of the role to get.
session: The database session to use.
Returns:
The role schema.
"""
return self._get_schema_by_name_or_id(
object_name_or_id=role_name_or_id,
schema_class=RoleSchema,
schema_name="role",
session=session,
)
def _get_run_schema(
self,
run_name_or_id: Union[str, UUID],
session: Session,
) -> PipelineRunSchema:
"""Gets a run schema by name or ID.
This is a helper method that is used in various places to find a run
by its name or ID.
Args:
run_name_or_id: The name or ID of the run to get.
session: The database session to use.
Returns:
The run schema.
"""
return self._get_schema_by_name_or_id(
object_name_or_id=run_name_or_id,
schema_class=PipelineRunSchema,
schema_name="run",
session=session,
)
alembic: Alembic
property
readonly
The Alembic wrapper.
Returns:
| Type | Description |
|---|---|
Alembic |
The Alembic wrapper. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the store is not initialized. |
engine: Engine
property
readonly
The SQLAlchemy engine.
Returns:
| Type | Description |
|---|---|
Engine |
The SQLAlchemy engine. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the store is not initialized. |
runs_inside_server: bool
property
readonly
Whether the store is running inside a server.
Returns:
| Type | Description |
|---|---|
bool |
Whether the store is running inside a server. |
CONFIG_TYPE (StoreConfiguration)
pydantic-model
SQL ZenML store configuration.
Attributes:
| Name | Type | Description |
|---|---|---|
type |
StoreType |
The type of the store. |
driver |
Optional[zenml.zen_stores.sql_zen_store.SQLDatabaseDriver] |
The SQL database driver. |
database |
Optional[str] |
database name. If not already present on the server, it will be created automatically on first access. |
username |
Optional[str] |
The database username. |
password |
Optional[str] |
The database password. |
ssl_ca |
Optional[str] |
certificate authority certificate. Required for SSL enabled authentication if the CA certificate is not part of the certificates shipped by the operating system. |
ssl_cert |
Optional[str] |
client certificate. Required for SSL enabled authentication if client certificates are used. |
ssl_key |
Optional[str] |
client certificate private key. Required for SSL enabled if client certificates are used. |
ssl_verify_server_cert |
bool |
set to verify the identity of the server against the provided server certificate. |
pool_size |
int |
The maximum number of connections to keep in the SQLAlchemy pool. |
max_overflow |
int |
The maximum number of connections to allow in the SQLAlchemy pool in addition to the pool_size. |
Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStoreConfiguration(StoreConfiguration):
"""SQL ZenML store configuration.
Attributes:
type: The type of the store.
driver: The SQL database driver.
database: database name. If not already present on the server, it will
be created automatically on first access.
username: The database username.
password: The database password.
ssl_ca: certificate authority certificate. Required for SSL
enabled authentication if the CA certificate is not part of the
certificates shipped by the operating system.
ssl_cert: client certificate. Required for SSL enabled
authentication if client certificates are used.
ssl_key: client certificate private key. Required for SSL
enabled if client certificates are used.
ssl_verify_server_cert: set to verify the identity of the server
against the provided server certificate.
pool_size: The maximum number of connections to keep in the SQLAlchemy
pool.
max_overflow: The maximum number of connections to allow in the
SQLAlchemy pool in addition to the pool_size.
"""
type: StoreType = StoreType.SQL
driver: Optional[SQLDatabaseDriver] = None
database: Optional[str] = None
username: Optional[str] = None
password: Optional[str] = None
ssl_ca: Optional[str] = None
ssl_cert: Optional[str] = None
ssl_key: Optional[str] = None
ssl_verify_server_cert: bool = False
pool_size: int = 20
max_overflow: int = 20
@root_validator(pre=True)
def _remove_grpc_attributes(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Removes old GRPC attributes.
Args:
values: All model attribute values.
Returns:
The model attribute values
"""
grpc_attribute_keys = [
"grpc_metadata_host",
"grpc_metadata_port",
"grpc_metadata_ssl_ca",
"grpc_metadata_ssl_key",
"grpc_metadata_ssl_cert",
]
grpc_values = [values.pop(key, None) for key in grpc_attribute_keys]
if any(grpc_values):
logger.warning(
"The GRPC attributes %s are unused and will be removed soon. "
"Please remove them from SQLZenStore configuration. This will "
"become an error in future versions of ZenML."
)
return values
@root_validator
def _validate_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validate the SQL URL.
The validator also moves the MySQL username, password and database
parameters from the URL into the other configuration arguments, if they
are present in the URL.
Args:
values: The values to validate.
Returns:
The validated values.
Raises:
ValueError: If the URL is invalid or the SQL driver is not
supported.
"""
url = values.get("url")
if url is None:
return values
# When running inside a container, if the URL uses localhost, the
# target service will not be available. We try to replace localhost
# with one of the special Docker or K3D internal hostnames.
url = replace_localhost_with_internal_hostname(url)
try:
sql_url = make_url(url)
except ArgumentError as e:
raise ValueError(
"Invalid SQL URL `%s`: %s. The URL must be in the format "
"`driver://[[username:password@]hostname:port]/database["
"?<extra-args>]`.",
url,
str(e),
)
if sql_url.drivername not in SQLDatabaseDriver.values():
raise ValueError(
"Invalid SQL driver value `%s`: The driver must be one of: %s.",
url,
", ".join(SQLDatabaseDriver.values()),
)
values["driver"] = SQLDatabaseDriver(sql_url.drivername)
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
if (
sql_url.username
or sql_url.password
or sql_url.query
or sql_url.database is None
):
raise ValueError(
"Invalid SQLite URL `%s`: The URL must be in the "
"format `sqlite:///path/to/database.db`.",
url,
)
if values.get("username") or values.get("password"):
raise ValueError(
"Invalid SQLite configuration: The username and password "
"must not be set",
url,
)
values["database"] = sql_url.database
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
if sql_url.username:
values["username"] = sql_url.username
sql_url = sql_url._replace(username=None)
if sql_url.password:
values["password"] = sql_url.password
sql_url = sql_url._replace(password=None)
if sql_url.database:
values["database"] = sql_url.database
sql_url = sql_url._replace(database=None)
if sql_url.query:
for k, v in sql_url.query.items():
if k == "ssl_ca":
values["ssl_ca"] = v
elif k == "ssl_cert":
values["ssl_cert"] = v
elif k == "ssl_key":
values["ssl_key"] = v
elif k == "ssl_verify_server_cert":
values["ssl_verify_server_cert"] = v
else:
raise ValueError(
"Invalid MySQL URL query parameter `%s`: The "
"parameter must be one of: ssl_ca, ssl_cert, "
"ssl_key, or ssl_verify_server_cert.",
k,
)
sql_url = sql_url._replace(query={})
database = values.get("database")
if (
not values.get("username")
or not values.get("password")
or not database
):
raise ValueError(
"Invalid MySQL configuration: The username, password and "
"database must be set in the URL or as configuration "
"attributes",
)
regexp = r"^[^\\/?%*:|\"<>.-]{1,64}$"
match = re.match(regexp, database)
if not match:
raise ValueError(
f"The database name does not conform to the required "
f"format "
f"rules ({regexp}): {database}"
)
# Save the certificates in a secure location on disk
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
content = values.get(key)
if content and not os.path.isfile(content):
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, f"{key}.pem")
with open(file_path, "w") as f:
f.write(content)
file_path.chmod(0o600)
values[key] = str(file_path)
values["url"] = str(sql_url)
return values
@staticmethod
def get_local_url(path: str) -> str:
"""Get a local SQL url for a given local path.
Args:
path: The path to the local sqlite file.
Returns:
The local SQL url for the given path.
"""
return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
"""Check if a URL scheme is supported by this store.
Args:
url: The URL to check.
Returns:
True if the URL scheme is supported, False otherwise.
"""
return make_url(url).drivername in SQLDatabaseDriver.values()
def expand_certificates(self) -> None:
"""Expands the certificates in the verify_ssl field."""
# Load the certificate values back into the configuration
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
file_path = getattr(self, key, None)
if file_path and os.path.isfile(file_path):
with open(file_path, "r") as f:
setattr(self, key, f.read())
@classmethod
def copy_configuration(
cls,
config: "StoreConfiguration",
config_path: str,
load_config_path: Optional[PurePath] = None,
) -> "StoreConfiguration":
"""Copy the store config using a different configuration path.
This method is used to create a copy of the store configuration that can
be loaded using a different configuration path or in the context of a
new environment, such as a container image.
The configuration files accompanying the store configuration are also
copied to the new configuration path (e.g. certificates etc.).
Args:
config: The store configuration to copy.
config_path: new path where the configuration copy will be loaded
from.
load_config_path: absolute path that will be used to load the copied
configuration. This can be set to a value different from
`config_path` if the configuration copy will be loaded from
a different environment, e.g. when the configuration is copied
to a container image and loaded using a different absolute path.
This will be reflected in the paths and URLs encoded in the
copied configuration.
Returns:
A new store configuration object that reflects the new configuration
path.
"""
assert isinstance(config, SqlZenStoreConfiguration)
config = config.copy()
if config.driver == SQLDatabaseDriver.MYSQL:
# Load the certificate values back into the configuration
config.expand_certificates()
elif config.driver == SQLDatabaseDriver.SQLITE:
if load_config_path:
config.url = cls.get_local_url(str(load_config_path))
else:
config.url = cls.get_local_url(config_path)
return config
def get_sqlmodel_config(
self,
) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
"""Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
The URL and connection arguments for the SQLModel engine.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
sql_url = make_url(self.url)
sqlalchemy_connect_args: Dict[str, Any] = {}
engine_args = {}
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
# The following default value is needed for sqlite to avoid the
# Error:
# sqlite3.ProgrammingError: SQLite objects created in a thread can
# only be used in that same thread.
sqlalchemy_connect_args = {"check_same_thread": False}
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
engine_args = {
"pool_size": self.pool_size,
"max_overflow": self.max_overflow,
}
sql_url = sql_url._replace(
drivername="mysql+pymysql",
username=self.username,
password=self.password,
database=self.database,
)
sqlalchemy_ssl_args: Dict[str, Any] = {}
# Handle SSL params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if not ssl_setting:
continue
if not os.path.isfile(ssl_setting):
logger.warning(
f"Database SSL setting `{key}` is not a file. "
)
sqlalchemy_ssl_args[key.lstrip("ssl_")] = ssl_setting
if len(sqlalchemy_ssl_args) > 0:
sqlalchemy_ssl_args[
"check_hostname"
] = self.ssl_verify_server_cert
sqlalchemy_connect_args["ssl"] = sqlalchemy_ssl_args
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return str(sql_url), sqlalchemy_connect_args, engine_args
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the certificate attributes can be expanded to the contents
# of the certificate files.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
Config
Pydantic configuration class.
Source code in zenml/zen_stores/sql_zen_store.py
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the certificate attributes can be expanded to the contents
# of the certificate files.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
copy_configuration(config, config_path, load_config_path=None)
classmethod
Copy the store config using a different configuration path.
This method is used to create a copy of the store configuration that can be loaded using a different configuration path or in the context of a new environment, such as a container image.
The configuration files accompanying the store configuration are also copied to the new configuration path (e.g. certificates etc.).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The store configuration to copy. |
required |
config_path |
str |
new path where the configuration copy will be loaded from. |
required |
load_config_path |
Optional[pathlib.PurePath] |
absolute path that will be used to load the copied
configuration. This can be set to a value different from
|
None |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
A new store configuration object that reflects the new configuration path. |
Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def copy_configuration(
cls,
config: "StoreConfiguration",
config_path: str,
load_config_path: Optional[PurePath] = None,
) -> "StoreConfiguration":
"""Copy the store config using a different configuration path.
This method is used to create a copy of the store configuration that can
be loaded using a different configuration path or in the context of a
new environment, such as a container image.
The configuration files accompanying the store configuration are also
copied to the new configuration path (e.g. certificates etc.).
Args:
config: The store configuration to copy.
config_path: new path where the configuration copy will be loaded
from.
load_config_path: absolute path that will be used to load the copied
configuration. This can be set to a value different from
`config_path` if the configuration copy will be loaded from
a different environment, e.g. when the configuration is copied
to a container image and loaded using a different absolute path.
This will be reflected in the paths and URLs encoded in the
copied configuration.
Returns:
A new store configuration object that reflects the new configuration
path.
"""
assert isinstance(config, SqlZenStoreConfiguration)
config = config.copy()
if config.driver == SQLDatabaseDriver.MYSQL:
# Load the certificate values back into the configuration
config.expand_certificates()
elif config.driver == SQLDatabaseDriver.SQLITE:
if load_config_path:
config.url = cls.get_local_url(str(load_config_path))
else:
config.url = cls.get_local_url(config_path)
return config
expand_certificates(self)
Expands the certificates in the verify_ssl field.
Source code in zenml/zen_stores/sql_zen_store.py
def expand_certificates(self) -> None:
"""Expands the certificates in the verify_ssl field."""
# Load the certificate values back into the configuration
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
file_path = getattr(self, key, None)
if file_path and os.path.isfile(file_path):
with open(file_path, "r") as f:
setattr(self, key, f.read())
get_local_url(path)
staticmethod
Get a local SQL url for a given local path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the local sqlite file. |
required |
Returns:
| Type | Description |
|---|---|
str |
The local SQL url for the given path. |
Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def get_local_url(path: str) -> str:
"""Get a local SQL url for a given local path.
Args:
path: The path to the local sqlite file.
Returns:
The local SQL url for the given path.
"""
return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"
get_sqlmodel_config(self)
Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
| Type | Description |
|---|---|
Tuple[str, Dict[str, Any], Dict[str, Any]] |
The URL and connection arguments for the SQLModel engine. |
Exceptions:
| Type | Description |
|---|---|
NotImplementedError |
If the SQL driver is not supported. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_sqlmodel_config(
self,
) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
"""Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
The URL and connection arguments for the SQLModel engine.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
sql_url = make_url(self.url)
sqlalchemy_connect_args: Dict[str, Any] = {}
engine_args = {}
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
# The following default value is needed for sqlite to avoid the
# Error:
# sqlite3.ProgrammingError: SQLite objects created in a thread can
# only be used in that same thread.
sqlalchemy_connect_args = {"check_same_thread": False}
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
engine_args = {
"pool_size": self.pool_size,
"max_overflow": self.max_overflow,
}
sql_url = sql_url._replace(
drivername="mysql+pymysql",
username=self.username,
password=self.password,
database=self.database,
)
sqlalchemy_ssl_args: Dict[str, Any] = {}
# Handle SSL params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if not ssl_setting:
continue
if not os.path.isfile(ssl_setting):
logger.warning(
f"Database SSL setting `{key}` is not a file. "
)
sqlalchemy_ssl_args[key.lstrip("ssl_")] = ssl_setting
if len(sqlalchemy_ssl_args) > 0:
sqlalchemy_ssl_args[
"check_hostname"
] = self.ssl_verify_server_cert
sqlalchemy_connect_args["ssl"] = sqlalchemy_ssl_args
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return str(sql_url), sqlalchemy_connect_args, engine_args
supports_url_scheme(url)
classmethod
Check if a URL scheme is supported by this store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The URL to check. |
required |
Returns:
| Type | Description |
|---|---|
bool |
True if the URL scheme is supported, False otherwise. |
Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
"""Check if a URL scheme is supported by this store.
Args:
url: The URL to check.
Returns:
True if the URL scheme is supported, False otherwise.
"""
return make_url(url).drivername in SQLDatabaseDriver.values()
create_artifact(self, artifact)
Creates an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact |
ArtifactRequestModel |
The artifact to create. |
required |
Returns:
| Type | Description |
|---|---|
ArtifactResponseModel |
The created artifact. |
Source code in zenml/zen_stores/sql_zen_store.py
def create_artifact(
self, artifact: ArtifactRequestModel
) -> ArtifactResponseModel:
"""Creates an artifact.
Args:
artifact: The artifact to create.
Returns:
The created artifact.
"""
with Session(self.engine) as session:
artifact_schema = ArtifactSchema.from_request(artifact)
session.add(artifact_schema)
session.commit()
return self._artifact_schema_to_model(artifact_schema)
create_flavor(*args, **kwargs)
Creates a new stack component flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor |
The stack component flavor to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created flavor. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a flavor with the same name and type is already owned by this user in this workspace. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_pipeline(*args, **kwargs)
Creates a new pipeline in a workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline |
The pipeline to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created pipeline. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If an identical pipeline already exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_role(*args, **kwargs)
Creates a new role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role |
The role model to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created role. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a role with the given name already exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_run(self, pipeline_run)
Creates a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_run |
PipelineRunRequestModel |
The pipeline run to create. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The created pipeline run. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If an identical pipeline run already exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def create_run(
self, pipeline_run: PipelineRunRequestModel
) -> PipelineRunResponseModel:
"""Creates a pipeline run.
Args:
pipeline_run: The pipeline run to create.
Returns:
The created pipeline run.
Raises:
EntityExistsError: If an identical pipeline run already exists.
"""
with Session(self.engine) as session:
# Check if pipeline run with same name already exists.
existing_domain_run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.name == pipeline_run.name
)
).first()
if existing_domain_run is not None:
raise EntityExistsError(
f"Unable to create pipeline run: A pipeline run with name "
f"'{pipeline_run.name}' already exists."
)
# Check if pipeline run with same ID already exists.
existing_id_run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.id == pipeline_run.id
)
).first()
if existing_id_run is not None:
raise EntityExistsError(
f"Unable to create pipeline run: A pipeline run with ID "
f"'{pipeline_run.id}' already exists."
)
# Query stack to ensure it exists in the DB
stack_id = None
if pipeline_run.stack is not None:
stack_id = session.exec(
select(StackSchema.id).where(
StackSchema.id == pipeline_run.stack
)
).first()
if stack_id is None:
logger.warning(
f"No stack found for this run. "
f"Creating pipeline run '{pipeline_run.name}' without "
"linked stack."
)
# Query pipeline to ensure it exists in the DB
pipeline_id = None
if pipeline_run.pipeline is not None:
pipeline_id = session.exec(
select(PipelineSchema.id).where(
PipelineSchema.id == pipeline_run.pipeline
)
).first()
if pipeline_id is None:
logger.warning(
f"No pipeline found. Creating pipeline run "
f"'{pipeline_run.name}' as unlisted run."
)
# Create the pipeline run
new_run = PipelineRunSchema.from_request(pipeline_run)
session.add(new_run)
session.commit()
return new_run.to_model()
create_run_metadata(self, run_metadata)
Creates run metadata.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_metadata |
RunMetadataRequestModel |
The run metadata to create. |
required |
Returns:
| Type | Description |
|---|---|
RunMetadataResponseModel |
The created run metadata. |
Source code in zenml/zen_stores/sql_zen_store.py
def create_run_metadata(
self, run_metadata: RunMetadataRequestModel
) -> RunMetadataResponseModel:
"""Creates run metadata.
Args:
run_metadata: The run metadata to create.
Returns:
The created run metadata.
"""
with Session(self.engine) as session:
run_metadata_schema = RunMetadataSchema.from_request(run_metadata)
session.add(run_metadata_schema)
session.commit()
return run_metadata_schema.to_model()
create_run_step(self, step_run)
Creates a step run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run |
StepRunRequestModel |
The step run to create. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The created step run. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
if the step run already exists. |
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def create_run_step(
self, step_run: StepRunRequestModel
) -> StepRunResponseModel:
"""Creates a step run.
Args:
step_run: The step run to create.
Returns:
The created step run.
Raises:
EntityExistsError: if the step run already exists.
KeyError: if the pipeline run doesn't exist.
"""
with Session(self.engine) as session:
# Check if the pipeline run exists
run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.id == step_run.pipeline_run_id
)
).first()
if run is None:
raise KeyError(
f"Unable to create step '{step_run.name}': No pipeline run "
f"with ID '{step_run.pipeline_run_id}' found."
)
# Check if the step name already exists in the pipeline run
existing_step_run = session.exec(
select(StepRunSchema)
.where(StepRunSchema.name == step_run.name)
.where(
StepRunSchema.pipeline_run_id == step_run.pipeline_run_id
)
).first()
if existing_step_run is not None:
raise EntityExistsError(
f"Unable to create step '{step_run.name}': A step with this "
f"name already exists in the pipeline run with ID "
f"'{step_run.pipeline_run_id}'."
)
# Create the step
step_schema = StepRunSchema.from_request(step_run)
session.add(step_schema)
# Save parent step IDs into the database.
for parent_step_id in step_run.parent_step_ids:
self._set_run_step_parent_step(
child_id=step_schema.id,
parent_id=parent_step_id,
session=session,
)
# Save input artifact IDs into the database.
for input_name, artifact_id in step_run.input_artifacts.items():
self._set_run_step_input_artifact(
run_step_id=step_schema.id,
artifact_id=artifact_id,
name=input_name,
session=session,
)
# Save output artifact IDs into the database.
for output_name, artifact_id in step_run.output_artifacts.items():
self._set_run_step_output_artifact(
step_run_id=step_schema.id,
artifact_id=artifact_id,
name=output_name,
session=session,
)
session.commit()
return self._run_step_schema_to_model(step_schema)
create_schedule(self, schedule)
Creates a new schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule |
ScheduleRequestModel |
The schedule to create. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The newly created schedule. |
Source code in zenml/zen_stores/sql_zen_store.py
def create_schedule(
self, schedule: ScheduleRequestModel
) -> ScheduleResponseModel:
"""Creates a new schedule.
Args:
schedule: The schedule to create.
Returns:
The newly created schedule.
"""
with Session(self.engine) as session:
new_schedule = ScheduleSchema.from_create_model(model=schedule)
session.add(new_schedule)
session.commit()
return new_schedule.to_model()
create_stack(*args, **kwargs)
Register a new stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack |
The stack to register. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The registered stack. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_stack_component(*args, **kwargs)
Create a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
The stack component to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The created stack component. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_team(*args, **kwargs)
Creates a new team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team |
The team model to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created team. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a team with the given name already exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_team_role_assignment(self, team_role_assignment)
Creates a new team role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment |
TeamRoleAssignmentRequestModel |
The role assignment model to create. |
required |
Returns:
| Type | Description |
|---|---|
TeamRoleAssignmentResponseModel |
The newly created role assignment. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If the role assignment already exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def create_team_role_assignment(
self, team_role_assignment: TeamRoleAssignmentRequestModel
) -> TeamRoleAssignmentResponseModel:
"""Creates a new team role assignment.
Args:
team_role_assignment: The role assignment model to create.
Returns:
The newly created role assignment.
Raises:
EntityExistsError: If the role assignment already exists.
"""
with Session(self.engine) as session:
role = self._get_role_schema(
team_role_assignment.role, session=session
)
workspace: Optional[WorkspaceSchema] = None
if team_role_assignment.workspace:
workspace = self._get_workspace_schema(
team_role_assignment.workspace, session=session
)
team = self._get_team_schema(
team_role_assignment.team, session=session
)
query = select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.user_id == team.id,
UserRoleAssignmentSchema.role_id == role.id,
)
if workspace is not None:
query = query.where(
UserRoleAssignmentSchema.workspace_id == workspace.id
)
existing_role_assignment = session.exec(query).first()
if existing_role_assignment is not None:
raise EntityExistsError(
f"Unable to assign role '{role.name}' to team "
f"'{team.name}': Role already assigned in this workspace."
)
role_assignment = TeamRoleAssignmentSchema(
role_id=role.id,
team_id=team.id,
workspace_id=workspace.id if workspace else None,
role=role,
team=team,
workspace=workspace,
)
session.add(role_assignment)
session.commit()
return role_assignment.to_model()
create_user(*args, **kwargs)
Creates a new user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user |
User to be created. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created user. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a user with the given name already exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_user_role_assignment(self, user_role_assignment)
Assigns a role to a user or team, scoped to a specific workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment |
UserRoleAssignmentRequestModel |
The role assignment to create. |
required |
Returns:
| Type | Description |
|---|---|
UserRoleAssignmentResponseModel |
The created role assignment. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
if the role assignment already exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def create_user_role_assignment(
self, user_role_assignment: UserRoleAssignmentRequestModel
) -> UserRoleAssignmentResponseModel:
"""Assigns a role to a user or team, scoped to a specific workspace.
Args:
user_role_assignment: The role assignment to create.
Returns:
The created role assignment.
Raises:
EntityExistsError: if the role assignment already exists.
"""
with Session(self.engine) as session:
role = self._get_role_schema(
user_role_assignment.role, session=session
)
workspace: Optional[WorkspaceSchema] = None
if user_role_assignment.workspace:
workspace = self._get_workspace_schema(
user_role_assignment.workspace, session=session
)
user = self._get_user_schema(
user_role_assignment.user, session=session
)
query = select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.user_id == user.id,
UserRoleAssignmentSchema.role_id == role.id,
)
if workspace is not None:
query = query.where(
UserRoleAssignmentSchema.workspace_id == workspace.id
)
existing_role_assignment = session.exec(query).first()
if existing_role_assignment is not None:
raise EntityExistsError(
f"Unable to assign role '{role.name}' to user "
f"'{user.name}': Role already assigned in this workspace."
)
role_assignment = UserRoleAssignmentSchema(
role_id=role.id,
user_id=user.id,
workspace_id=workspace.id if workspace else None,
role=role,
user=user,
workspace=workspace,
)
session.add(role_assignment)
session.commit()
return role_assignment.to_model()
create_workspace(*args, **kwargs)
Creates a new workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace |
The workspace to create. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The newly created workspace. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a workspace with the given name already exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_artifact(self, artifact_id)
Deletes an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_id |
UUID |
The ID of the artifact to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the artifact doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def delete_artifact(self, artifact_id: UUID) -> None:
"""Deletes an artifact.
Args:
artifact_id: The ID of the artifact to delete.
Raises:
KeyError: if the artifact doesn't exist.
"""
with Session(self.engine) as session:
artifact = session.exec(
select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
).first()
if artifact is None:
raise KeyError(
f"Unable to delete artifact with ID {artifact_id}: "
f"No artifact with this ID found."
)
session.delete(artifact)
session.commit()
delete_flavor(*args, **kwargs)
Delete a flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
The id of the flavor to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the flavor doesn't exist. |
IllegalOperationError |
if the flavor is used by a stack component. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_pipeline(*args, **kwargs)
Deletes a pipeline.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
The ID of the pipeline to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_role(*args, **kwargs)
Deletes a role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Name or ID of the role to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
IllegalOperationError |
If the role is still assigned to users or the role is one of the built-in roles. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_run(self, run_id)
Deletes a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def delete_run(self, run_id: UUID) -> None:
"""Deletes a pipeline run.
Args:
run_id: The ID of the pipeline run to delete.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline run with the given ID exists
existing_run = session.exec(
select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
).first()
if existing_run is None:
raise KeyError(
f"Unable to delete pipeline run with ID {run_id}: "
f"No pipeline run with this ID found."
)
# Delete the pipeline run
session.delete(existing_run)
session.commit()
delete_schedule(self, schedule_id)
Deletes a schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
The ID of the schedule to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the schedule doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def delete_schedule(self, schedule_id: UUID) -> None:
"""Deletes a schedule.
Args:
schedule_id: The ID of the schedule to delete.
Raises:
KeyError: if the schedule doesn't exist.
"""
with Session(self.engine) as session:
# Check if schedule with the given ID exists
schedule = session.exec(
select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
).first()
if schedule is None:
raise KeyError(
f"Unable to delete schedule with ID {schedule_id}: "
f"No schedule with this ID found."
)
# Delete the schedule
session.delete(schedule)
session.commit()
delete_stack(*args, **kwargs)
Delete a stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
The ID of the stack to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
IllegalOperationError |
if the stack is a default stack. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_stack_component(*args, **kwargs)
Delete a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
The id of the stack component to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
IllegalOperationError |
if the stack component is part of one or more stacks, or if it's a default stack component. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_team(*args, **kwargs)
Deletes a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Name or ID of the team to delete. |
required |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_team_role_assignment(self, team_role_assignment_id)
Delete a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_id |
UUID |
The ID of the specific role assignment |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If the role assignment does not exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def delete_team_role_assignment(
self, team_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
team_role_assignment_id: The ID of the specific role assignment
Raises:
KeyError: If the role assignment does not exist.
"""
with Session(self.engine) as session:
team_role = session.exec(
select(TeamRoleAssignmentSchema).where(
TeamRoleAssignmentSchema.id == team_role_assignment_id
)
).one_or_none()
if not team_role:
raise KeyError(
f"No team role assignment with id "
f"{team_role_assignment_id} exists."
)
session.delete(team_role)
session.commit()
delete_user(*args, **kwargs)
Deletes a user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
The name or the ID of the user to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
IllegalOperationError |
If the user is the default user account. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_user_role_assignment(self, user_role_assignment_id)
Delete a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_id |
UUID |
The ID of the specific role assignment. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If the role assignment does not exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def delete_user_role_assignment(
self, user_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
user_role_assignment_id: The ID of the specific role assignment.
Raises:
KeyError: If the role assignment does not exist.
"""
with Session(self.engine) as session:
user_role = session.exec(
select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.id == user_role_assignment_id
)
).one_or_none()
if not user_role:
raise KeyError(
f"No user role assignment with id "
f"{user_role_assignment_id} exists."
)
session.delete(user_role)
session.commit()
delete_workspace(*args, **kwargs)
Deletes a workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_name_or_id |
Name or ID of the workspace to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
IllegalOperationError |
If the workspace is the default workspace. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
filter_and_paginate(session, query, table, filter_model, custom_schema_to_model_conversion=None)
classmethod
Given a query, return a Page instance with a list of filtered Models.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
session |
Session |
The SQLModel Session |
required |
query |
Union[sqlmodel.sql.expression.Select, sqlmodel.sql.expression.SelectOfScalar] |
The query to execute |
required |
table |
Type[~AnySchema] |
The table to select from |
required |
filter_model |
BaseFilterModel |
The filter to use, including pagination and sorting |
required |
custom_schema_to_model_conversion |
Optional[Callable[[~AnySchema], ~B]] |
Callable to convert the schema into a model. This is used if the Model contains additional data that is not explicitly stored as a field or relationship on the model. |
None |
Returns:
| Type | Description |
|---|---|
Page[B] |
The Domain Model representation of the DB resource |
Exceptions:
| Type | Description |
|---|---|
ValueError |
if the filtered page number is out of bounds. |
RuntimeError |
if the schema does not have a |
Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def filter_and_paginate(
cls,
session: Session,
query: Union[Select[AnySchema], SelectOfScalar[AnySchema]],
table: Type[AnySchema],
filter_model: BaseFilterModel,
custom_schema_to_model_conversion: Optional[
Callable[[AnySchema], B]
] = None,
) -> Page[B]:
"""Given a query, return a Page instance with a list of filtered Models.
Args:
session: The SQLModel Session
query: The query to execute
table: The table to select from
filter_model: The filter to use, including pagination and sorting
custom_schema_to_model_conversion: Callable to convert the schema
into a model. This is used if the Model contains additional
data that is not explicitly stored as a field or relationship
on the model.
Returns:
The Domain Model representation of the DB resource
Raises:
ValueError: if the filtered page number is out of bounds.
RuntimeError: if the schema does not have a `to_model` method.
"""
# Filtering
filters = filter_model.generate_filter(table=table)
if filters is not None:
query = query.where(filters)
# Get the total amount of items in the database for a given query
total = session.scalar(
select([func.count("*")]).select_from(
query.options(noload("*")).subquery()
)
)
# Sorting
column, operand = filter_model.sorting_params
if operand == SorterOps.DESCENDING:
query = query.order_by(desc(getattr(table, column)))
else:
query = query.order_by(asc(getattr(table, column)))
# Get the total amount of pages in the database for a given query
if total == 0:
total_pages = 1
else:
total_pages = math.ceil(total / filter_model.size)
if filter_model.page > total_pages:
raise ValueError(
f"Invalid page {filter_model.page}. The requested page size is "
f"{filter_model.size} and there are a total of {total} items "
f"for this query. The maximum page value therefore is "
f"{total_pages}."
)
# Get a page of the actual data
item_schemas: List[AnySchema] = (
session.exec(
query.limit(filter_model.size).offset(filter_model.offset)
)
.unique()
.all()
)
# Convert this page of items from schemas to models.
items: List[B] = []
for schema in item_schemas:
# If a custom conversion function is provided, use it.
if custom_schema_to_model_conversion:
items.append(custom_schema_to_model_conversion(schema))
continue
# Otherwise, try to use the `to_model` method of the schema.
to_model = getattr(schema, "to_model", None)
if callable(to_model):
items.append(to_model())
continue
# If neither of the above work, raise an error.
raise RuntimeError(
f"Cannot convert schema `{schema.__class__.__name__}` to model "
"since it does not have a `to_model` method."
)
return Page(
total=total,
total_pages=total_pages,
items=items,
page=filter_model.page,
size=filter_model.size,
)
get_artifact(self, artifact_id)
Gets an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_id |
UUID |
The ID of the artifact to get. |
required |
Returns:
| Type | Description |
|---|---|
ArtifactResponseModel |
The artifact. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the artifact doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_artifact(self, artifact_id: UUID) -> ArtifactResponseModel:
"""Gets an artifact.
Args:
artifact_id: The ID of the artifact to get.
Returns:
The artifact.
Raises:
KeyError: if the artifact doesn't exist.
"""
with Session(self.engine) as session:
artifact = session.exec(
select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
).first()
if artifact is None:
raise KeyError(
f"Unable to get artifact with ID {artifact_id}: "
f"No artifact with this ID found."
)
return self._artifact_schema_to_model(artifact)
get_auth_user(self, user_name_or_id)
Gets the auth model to a specific user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
required |
Returns:
| Type | Description |
|---|---|
UserAuthModel |
The requested user, if it was found. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_auth_user(
self, user_name_or_id: Union[str, UUID]
) -> UserAuthModel:
"""Gets the auth model to a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
"""
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
return UserAuthModel(
id=user.id,
name=user.name,
full_name=user.full_name,
email_opted_in=user.email_opted_in,
active=user.active,
created=user.created,
updated=user.updated,
password=user.password,
activation_token=user.activation_token,
)
get_flavor(self, flavor_id)
Get a flavor by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
UUID |
The ID of the flavor to fetch. |
required |
Returns:
| Type | Description |
|---|---|
FlavorResponseModel |
The stack component flavor. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component flavor doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_flavor(self, flavor_id: UUID) -> FlavorResponseModel:
"""Get a flavor by ID.
Args:
flavor_id: The ID of the flavor to fetch.
Returns:
The stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
with Session(self.engine) as session:
flavor_in_db = session.exec(
select(FlavorSchema).where(FlavorSchema.id == flavor_id)
).first()
if flavor_in_db is None:
raise KeyError(f"Flavor with ID {flavor_id} not found.")
return flavor_in_db.to_model()
get_or_create_run(self, pipeline_run)
Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned. Otherwise, a new run is created.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_run |
PipelineRunRequestModel |
The pipeline run to get or create. |
required |
Returns:
| Type | Description |
|---|---|
Tuple[zenml.models.pipeline_run_models.PipelineRunResponseModel, bool] |
The pipeline run, and a boolean indicating whether the run was created or not. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_or_create_run(
self, pipeline_run: PipelineRunRequestModel
) -> Tuple[PipelineRunResponseModel, bool]:
"""Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned.
Otherwise, a new run is created.
Args:
pipeline_run: The pipeline run to get or create.
Returns:
The pipeline run, and a boolean indicating whether the run was
created or not.
"""
# We want to have the 'create' statement in the try block since running
# it first will reduce concurrency issues.
try:
return self.create_run(pipeline_run), True
except EntityExistsError:
# Currently, an `EntityExistsError` is raised if either the run ID
# or the run name already exists. Therefore, we need to have another
# try block since getting the run by ID might still fail.
try:
return self.get_run(pipeline_run.id), False
except KeyError:
return self.get_run(pipeline_run.name), False
get_pipeline(self, pipeline_id)
Get a pipeline with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
ID of the pipeline. |
required |
Returns:
| Type | Description |
|---|---|
PipelineResponseModel |
The pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline does not exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_pipeline(self, pipeline_id: UUID) -> PipelineResponseModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
with Session(self.engine) as session:
# Check if pipeline with the given ID exists
pipeline = session.exec(
select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
).first()
if pipeline is None:
raise KeyError(
f"Unable to get pipeline with ID '{pipeline_id}': "
"No pipeline with this ID found."
)
return pipeline.to_model()
get_role(self, role_name_or_id)
Gets a specific role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to get. |
required |
Returns:
| Type | Description |
|---|---|
RoleResponseModel |
The requested role. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleResponseModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
"""
with Session(self.engine) as session:
role = self._get_role_schema(role_name_or_id, session=session)
return role.to_model()
get_run(self, run_name_or_id)
Gets a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the pipeline run to get. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The pipeline run. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run(
self, run_name_or_id: Union[str, UUID]
) -> PipelineRunResponseModel:
"""Gets a pipeline run.
Args:
run_name_or_id: The name or ID of the pipeline run to get.
Returns:
The pipeline run.
"""
with Session(self.engine) as session:
run = self._get_run_schema(run_name_or_id, session=session)
return run.to_model()
get_run_step(self, step_run_id)
Get a step run by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_id |
UUID |
The ID of the step run to get. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The step run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the step run doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run_step(self, step_run_id: UUID) -> StepRunResponseModel:
"""Get a step run by ID.
Args:
step_run_id: The ID of the step run to get.
Returns:
The step run.
Raises:
KeyError: if the step run doesn't exist.
"""
with Session(self.engine) as session:
step_run = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_run_id)
).first()
if step_run is None:
raise KeyError(
f"Unable to get step run with ID {step_run_id}: No step "
"run with this ID found."
)
return self._run_step_schema_to_model(step_run)
get_schedule(self, schedule_id)
Get a schedule with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
ID of the schedule. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The schedule. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the schedule does not exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_schedule(self, schedule_id: UUID) -> ScheduleResponseModel:
"""Get a schedule with a given ID.
Args:
schedule_id: ID of the schedule.
Returns:
The schedule.
Raises:
KeyError: if the schedule does not exist.
"""
with Session(self.engine) as session:
# Check if schedule with the given ID exists
schedule = session.exec(
select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
).first()
if schedule is None:
raise KeyError(
f"Unable to get schedule with ID '{schedule_id}': "
"No schedule with this ID found."
)
return schedule.to_model()
get_stack(self, stack_id)
Get a stack by its unique ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack to get. |
required |
Returns:
| Type | Description |
|---|---|
StackResponseModel |
The stack with the given ID. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_stack(self, stack_id: UUID) -> StackResponseModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
Raises:
KeyError: if the stack doesn't exist.
"""
with Session(self.engine) as session:
stack = session.exec(
select(StackSchema).where(StackSchema.id == stack_id)
).first()
if stack is None:
raise KeyError(f"Stack with ID {stack_id} not found.")
return stack.to_model()
get_stack_component(self, component_id)
Get a stack component by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to get. |
required |
Returns:
| Type | Description |
|---|---|
ComponentResponseModel |
The stack component. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_stack_component(
self, component_id: UUID
) -> ComponentResponseModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
with Session(self.engine) as session:
stack_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component_id
)
).first()
if stack_component is None:
raise KeyError(
f"Stack component with ID {component_id} not found."
)
return stack_component.to_model()
get_store_info(self)
Get information about the store.
Returns:
| Type | Description |
|---|---|
ServerModel |
Information about the store. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If the deployment ID could not be loaded from the database. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
Raises:
KeyError: If the deployment ID could not be loaded from the
database.
"""
model = super().get_store_info()
sql_url = make_url(self.config.url)
model.database_type = ServerDatabaseType(sql_url.drivername)
# Fetch the deployment ID from the database and use it to replace the one
# fetched from the global configuration
with Session(self.engine) as session:
identity = session.exec(select(IdentitySchema)).first()
if identity is None:
raise KeyError(
"The deployment ID could not be loaded from the database."
)
model.id = identity.id
return model
get_team(self, team_name_or_id)
Gets a specific team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamResponseModel |
The requested team. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamResponseModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
return team.to_model()
get_team_role_assignment(self, team_role_assignment_id)
Gets a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_id |
UUID |
ID of the role assignment to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamRoleAssignmentResponseModel |
The requested role assignment. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no role assignment with the given ID exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_team_role_assignment(
self, team_role_assignment_id: UUID
) -> TeamRoleAssignmentResponseModel:
"""Gets a specific role assignment.
Args:
team_role_assignment_id: ID of the role assignment to get.
Returns:
The requested role assignment.
Raises:
KeyError: If no role assignment with the given ID exists.
"""
with Session(self.engine) as session:
team_role = session.exec(
select(TeamRoleAssignmentSchema).where(
TeamRoleAssignmentSchema.id == team_role_assignment_id
)
).one_or_none()
if team_role:
return team_role.to_model()
else:
raise KeyError(
f"Unable to get team role assignment with ID "
f"'{team_role_assignment_id}': No team role assignment "
f"with this ID found."
)
get_user(self, user_name_or_id=None, include_private=False)
Gets a specific user, when no id is specified the active user is returned.
Raises a KeyError in case a user with that id does not exist.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
None |
include_private |
bool |
Whether to include private user information |
False |
Returns:
| Type | Description |
|---|---|
UserResponseModel |
The requested user, if it was found. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_user(
self,
user_name_or_id: Optional[Union[str, UUID]] = None,
include_private: bool = False,
) -> UserResponseModel:
"""Gets a specific user, when no id is specified the active user is returned.
Raises a KeyError in case a user with that id does not exist.
Args:
user_name_or_id: The name or ID of the user to get.
include_private: Whether to include private user information
Returns:
The requested user, if it was found.
"""
if not user_name_or_id:
user_name_or_id = self._default_user_name
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
return user.to_model(include_private=include_private)
get_user_role_assignment(self, user_role_assignment_id)
Gets a role assignment by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_id |
UUID |
ID of the role assignment to get. |
required |
Returns:
| Type | Description |
|---|---|
UserRoleAssignmentResponseModel |
The role assignment. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If the role assignment does not exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_user_role_assignment(
self, user_role_assignment_id: UUID
) -> UserRoleAssignmentResponseModel:
"""Gets a role assignment by ID.
Args:
user_role_assignment_id: ID of the role assignment to get.
Returns:
The role assignment.
Raises:
KeyError: If the role assignment does not exist.
"""
with Session(self.engine) as session:
user_role = session.exec(
select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.id == user_role_assignment_id
)
).one_or_none()
if user_role:
return user_role.to_model()
else:
raise KeyError(
f"Unable to get user role assignment with ID "
f"'{user_role_assignment_id}': No user role assignment "
f"with this ID found."
)
get_workspace(self, workspace_name_or_id)
Get an existing workspace by name or ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the workspace to get. |
required |
Returns:
| Type | Description |
|---|---|
WorkspaceResponseModel |
The requested workspace if one was found. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_workspace(
self, workspace_name_or_id: Union[str, UUID]
) -> WorkspaceResponseModel:
"""Get an existing workspace by name or ID.
Args:
workspace_name_or_id: Name or ID of the workspace to get.
Returns:
The requested workspace if one was found.
"""
with Session(self.engine) as session:
workspace = self._get_workspace_schema(
workspace_name_or_id, session=session
)
return workspace.to_model()
list_artifacts(self, artifact_filter_model)
List all artifacts matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_filter_model |
ArtifactFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[ArtifactResponseModel] |
A list of all artifacts matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_artifacts(
self, artifact_filter_model: ArtifactFilterModel
) -> Page[ArtifactResponseModel]:
"""List all artifacts matching the given filter criteria.
Args:
artifact_filter_model: All filter parameters including pagination
params.
Returns:
A list of all artifacts matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(ArtifactSchema)
if artifact_filter_model.only_unused:
query = query.where(
ArtifactSchema.id.notin_( # type: ignore[attr-defined]
select(StepRunOutputArtifactSchema.artifact_id)
)
)
query = query.where(
ArtifactSchema.id.notin_( # type: ignore[attr-defined]
select(StepRunInputArtifactSchema.artifact_id)
)
)
return self.filter_and_paginate(
session=session,
query=query,
table=ArtifactSchema,
filter_model=artifact_filter_model,
custom_schema_to_model_conversion=self._artifact_schema_to_model,
)
list_flavors(self, flavor_filter_model)
List all stack component flavors matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_filter_model |
FlavorFilterModel |
All filter parameters including pagination params |
required |
Returns:
| Type | Description |
|---|---|
Page[FlavorResponseModel] |
List of all the stack component flavors matching the given criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_flavors(
self, flavor_filter_model: FlavorFilterModel
) -> Page[FlavorResponseModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
flavor_filter_model: All filter parameters including pagination
params
Returns:
List of all the stack component flavors matching the given criteria.
"""
with Session(self.engine) as session:
query = select(FlavorSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=FlavorSchema,
filter_model=flavor_filter_model,
)
list_pipelines(self, pipeline_filter_model)
List all pipelines matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_filter_model |
PipelineFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[PipelineResponseModel] |
A list of all pipelines matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_pipelines(
self, pipeline_filter_model: PipelineFilterModel
) -> Page[PipelineResponseModel]:
"""List all pipelines matching the given filter criteria.
Args:
pipeline_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipelines matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(PipelineSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=PipelineSchema,
filter_model=pipeline_filter_model,
)
list_roles(self, role_filter_model)
List all roles matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_filter_model |
RoleFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[RoleResponseModel] |
A list of all roles matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_roles(
self, role_filter_model: RoleFilterModel
) -> Page[RoleResponseModel]:
"""List all roles matching the given filter criteria.
Args:
role_filter_model: All filter parameters including pagination
params.
Returns:
A list of all roles matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(RoleSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=RoleSchema,
filter_model=role_filter_model,
)
list_run_metadata(self, run_metadata_filter_model)
List run metadata.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_metadata_filter_model |
RunMetadataFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[RunMetadataResponseModel] |
The run metadata. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_run_metadata(
self,
run_metadata_filter_model: RunMetadataFilterModel,
) -> Page[RunMetadataResponseModel]:
"""List run metadata.
Args:
run_metadata_filter_model: All filter parameters including
pagination params.
Returns:
The run metadata.
"""
with Session(self.engine) as session:
query = select(RunMetadataSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=RunMetadataSchema,
filter_model=run_metadata_filter_model,
)
list_run_steps(self, step_run_filter_model)
List all step runs matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_filter_model |
StepRunFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[StepRunResponseModel] |
A list of all step runs matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_run_steps(
self, step_run_filter_model: StepRunFilterModel
) -> Page[StepRunResponseModel]:
"""List all step runs matching the given filter criteria.
Args:
step_run_filter_model: All filter parameters including pagination
params.
Returns:
A list of all step runs matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(StepRunSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=StepRunSchema,
filter_model=step_run_filter_model,
custom_schema_to_model_conversion=self._run_step_schema_to_model,
)
list_runs(self, runs_filter_model)
List all pipeline runs matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
runs_filter_model |
PipelineRunFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[PipelineRunResponseModel] |
A list of all pipeline runs matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_runs(
self, runs_filter_model: PipelineRunFilterModel
) -> Page[PipelineRunResponseModel]:
"""List all pipeline runs matching the given filter criteria.
Args:
runs_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipeline runs matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(PipelineRunSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=PipelineRunSchema,
filter_model=runs_filter_model,
)
list_schedules(self, schedule_filter_model)
List all schedules in the workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_filter_model |
ScheduleFilterModel |
All filter parameters including pagination params |
required |
Returns:
| Type | Description |
|---|---|
Page[ScheduleResponseModel] |
A list of schedules. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_schedules(
self, schedule_filter_model: ScheduleFilterModel
) -> Page[ScheduleResponseModel]:
"""List all schedules in the workspace.
Args:
schedule_filter_model: All filter parameters including pagination
params
Returns:
A list of schedules.
"""
with Session(self.engine) as session:
query = select(ScheduleSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=ScheduleSchema,
filter_model=schedule_filter_model,
)
list_stack_components(self, component_filter_model)
List all stack components matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_filter_model |
ComponentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[ComponentResponseModel] |
A list of all stack components matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_stack_components(
self, component_filter_model: ComponentFilterModel
) -> Page[ComponentResponseModel]:
"""List all stack components matching the given filter criteria.
Args:
component_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stack components matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(StackComponentSchema)
paged_components: Page[
ComponentResponseModel
] = self.filter_and_paginate(
session=session,
query=query,
table=StackComponentSchema,
filter_model=component_filter_model,
)
return paged_components
list_stacks(self, stack_filter_model)
List all stacks matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_filter_model |
StackFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[StackResponseModel] |
A list of all stacks matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_stacks(
self, stack_filter_model: StackFilterModel
) -> Page[StackResponseModel]:
"""List all stacks matching the given filter criteria.
Args:
stack_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stacks matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(StackSchema)
if stack_filter_model.component_id:
query = query.where(
StackCompositionSchema.stack_id == StackSchema.id
).where(
StackCompositionSchema.component_id
== stack_filter_model.component_id
)
return self.filter_and_paginate(
session=session,
query=query,
table=StackSchema,
filter_model=stack_filter_model,
)
list_team_role_assignments(self, team_role_assignment_filter_model)
List all roles assignments matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_filter_model |
TeamRoleAssignmentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[TeamRoleAssignmentResponseModel] |
A list of all roles assignments matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_team_role_assignments(
self, team_role_assignment_filter_model: TeamRoleAssignmentFilterModel
) -> Page[TeamRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
team_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(TeamRoleAssignmentSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=TeamRoleAssignmentSchema,
filter_model=team_role_assignment_filter_model,
)
list_teams(self, team_filter_model)
List all teams matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_filter_model |
TeamFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[TeamResponseModel] |
A list of all teams matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_teams(
self, team_filter_model: TeamFilterModel
) -> Page[TeamResponseModel]:
"""List all teams matching the given filter criteria.
Args:
team_filter_model: All filter parameters including pagination
params.
Returns:
A list of all teams matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(TeamSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=TeamSchema,
filter_model=team_filter_model,
)
list_user_role_assignments(self, user_role_assignment_filter_model)
List all roles assignments matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_filter_model |
UserRoleAssignmentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[UserRoleAssignmentResponseModel] |
A list of all roles assignments matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_user_role_assignments(
self, user_role_assignment_filter_model: UserRoleAssignmentFilterModel
) -> Page[UserRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
user_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(UserRoleAssignmentSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=UserRoleAssignmentSchema,
filter_model=user_role_assignment_filter_model,
)
list_users(self, user_filter_model)
List all users.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_filter_model |
UserFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[UserResponseModel] |
A list of all users. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_users(
self, user_filter_model: UserFilterModel
) -> Page[UserResponseModel]:
"""List all users.
Args:
user_filter_model: All filter parameters including pagination
params.
Returns:
A list of all users.
"""
with Session(self.engine) as session:
query = select(UserSchema)
paged_user: Page[UserResponseModel] = self.filter_and_paginate(
session=session,
query=query,
table=UserSchema,
filter_model=user_filter_model,
)
return paged_user
list_workspaces(self, workspace_filter_model)
List all workspace matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_filter_model |
WorkspaceFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[WorkspaceResponseModel] |
A list of all workspace matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_workspaces(
self, workspace_filter_model: WorkspaceFilterModel
) -> Page[WorkspaceResponseModel]:
"""List all workspace matching the given filter criteria.
Args:
workspace_filter_model: All filter parameters including pagination
params.
Returns:
A list of all workspace matching the filter criteria.
"""
with Session(self.engine) as session:
query = select(WorkspaceSchema)
return self.filter_and_paginate(
session=session,
query=query,
table=WorkspaceSchema,
filter_model=workspace_filter_model,
)
migrate_database(self)
Migrate the database to the head as defined by the python package.
Source code in zenml/zen_stores/sql_zen_store.py
def migrate_database(self) -> None:
"""Migrate the database to the head as defined by the python package."""
alembic_logger = logging.getLogger("alembic")
# remove all existing handlers
while len(alembic_logger.handlers):
alembic_logger.removeHandler(alembic_logger.handlers[0])
logging_level = get_logging_level()
# suppress alembic info logging if the zenml logging level is not debug
if logging_level == LoggingLevels.DEBUG:
alembic_logger.setLevel(logging.DEBUG)
else:
alembic_logger.setLevel(logging.WARNING)
alembic_logger.addHandler(get_console_handler())
# We need to account for 3 distinct cases here:
# 1. the database is completely empty (not initialized)
# 2. the database is not empty, but has never been migrated with alembic
# before (i.e. was created with SQLModel back when alembic wasn't
# used)
# 3. the database is not empty and has been migrated with alembic before
revisions = self.alembic.current_revisions()
if len(revisions) >= 1:
if len(revisions) > 1:
logger.warning(
"The ZenML database has more than one migration head "
"revision. This is not expected and might indicate a "
"database migration problem. Please raise an issue on "
"GitHub if you encounter this."
)
# Case 3: the database has been migrated with alembic before. Just
# upgrade to the latest revision.
self.alembic.upgrade()
else:
if self.alembic.db_is_empty():
# Case 1: the database is empty. We can just create the
# tables from scratch with alembic.
self.alembic.upgrade()
else:
# Case 2: the database is not empty, but has never been
# migrated with alembic before. We need to create the alembic
# version table, initialize it with the first revision where we
# introduced alembic and then upgrade to the latest revision.
self.alembic.stamp(ZENML_ALEMBIC_START_REVISION)
self.alembic.upgrade()
update_pipeline(*args, **kwargs)
Updates a pipeline.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
The ID of the pipeline to be updated. |
required | |
pipeline_update |
The update to be applied. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_role(*args, **kwargs)
Update an existing role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_id |
The ID of the role to be updated. |
required | |
role_update |
The update to be applied to the role. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated role. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the role does not exist. |
IllegalOperationError |
if the role is a system role. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_run(self, run_id, run_update)
Updates a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to update. |
required |
run_update |
PipelineRunUpdateModel |
The update to be applied to the pipeline run. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The updated pipeline run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def update_run(
self, run_id: UUID, run_update: PipelineRunUpdateModel
) -> PipelineRunResponseModel:
"""Updates a pipeline run.
Args:
run_id: The ID of the pipeline run to update.
run_update: The update to be applied to the pipeline run.
Returns:
The updated pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline run with the given ID exists
existing_run = session.exec(
select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
).first()
if existing_run is None:
raise KeyError(
f"Unable to update pipeline run with ID {run_id}: "
f"No pipeline run with this ID found."
)
# Update the pipeline run
existing_run.update(run_update=run_update)
session.add(existing_run)
session.commit()
session.refresh(existing_run)
return existing_run.to_model()
update_run_step(self, step_run_id, step_run_update)
Updates a step run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_id |
UUID |
The ID of the step to update. |
required |
step_run_update |
StepRunUpdateModel |
The update to be applied to the step. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The updated step run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the step run doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def update_run_step(
self,
step_run_id: UUID,
step_run_update: StepRunUpdateModel,
) -> StepRunResponseModel:
"""Updates a step run.
Args:
step_run_id: The ID of the step to update.
step_run_update: The update to be applied to the step.
Returns:
The updated step run.
Raises:
KeyError: if the step run doesn't exist.
"""
with Session(self.engine) as session:
# Check if the step exists
existing_step_run = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_run_id)
).first()
if existing_step_run is None:
raise KeyError(
f"Unable to update step with ID {step_run_id}: "
f"No step with this ID found."
)
# Update the step
existing_step_run.update(step_run_update)
session.add(existing_step_run)
# Update the output artifacts.
for name, artifact_id in step_run_update.output_artifacts.items():
self._set_run_step_output_artifact(
step_run_id=step_run_id,
artifact_id=artifact_id,
name=name,
session=session,
)
# Input artifacts and parent steps cannot be updated after the
# step has been created.
session.commit()
session.refresh(existing_step_run)
return self._run_step_schema_to_model(existing_step_run)
update_schedule(self, schedule_id, schedule_update)
Updates a schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
The ID of the schedule to be updated. |
required |
schedule_update |
ScheduleUpdateModel |
The update to be applied. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The updated schedule. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the schedule doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def update_schedule(
self,
schedule_id: UUID,
schedule_update: ScheduleUpdateModel,
) -> ScheduleResponseModel:
"""Updates a schedule.
Args:
schedule_id: The ID of the schedule to be updated.
schedule_update: The update to be applied.
Returns:
The updated schedule.
Raises:
KeyError: if the schedule doesn't exist.
"""
with Session(self.engine) as session:
# Check if schedule with the given ID exists
existing_schedule = session.exec(
select(ScheduleSchema).where(ScheduleSchema.id == schedule_id)
).first()
if existing_schedule is None:
raise KeyError(
f"Unable to update schedule with ID {schedule_id}: "
f"No schedule with this ID found."
)
# Update the schedule
existing_schedule = existing_schedule.from_update_model(
schedule_update
)
session.add(existing_schedule)
session.commit()
return existing_schedule.to_model()
update_stack(*args, **kwargs)
Update a stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
The ID of the stack update. |
required | |
stack_update |
The update request on the stack. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated stack. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
IllegalOperationError |
if the stack is a default stack. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_stack_component(*args, **kwargs)
Update an existing stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
The ID of the stack component to update. |
required | |
component_update |
The update to be applied to the stack component. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated stack component. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
IllegalOperationError |
if the stack component is a default stack component. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_team(*args, **kwargs)
Update an existing team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_id |
The ID of the team to be updated. |
required | |
team_update |
The update to be applied to the team. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated team. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the team does not exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_user(*args, **kwargs)
Updates an existing user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_id |
The id of the user to update. |
required | |
user_update |
The update to be applied to the user. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated user. |
Exceptions:
| Type | Description |
|---|---|
IllegalOperationError |
If the request tries to update the username for the default user account. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_workspace(*args, **kwargs)
Update an existing workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_id |
The ID of the workspace to be updated. |
required | |
workspace_update |
The update to be applied to the workspace. |
required |
Returns:
| Type | Description |
|---|---|
Any |
The updated workspace. |
Exceptions:
| Type | Description |
|---|---|
IllegalOperationError |
if the workspace is the default workspace. |
KeyError |
if the workspace does not exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
with event_handler(event) as handler:
try:
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
handler.tracker = args[0]
for obj in list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
handler.metadata = obj.get_analytics_metadata()
break
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
result = func(*args, **kwargs)
try:
if isinstance(result, AnalyticsTrackedModelMixin):
handler.metadata = result.get_analytics_metadata()
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
SqlZenStoreConfiguration (StoreConfiguration)
pydantic-model
SQL ZenML store configuration.
Attributes:
| Name | Type | Description |
|---|---|---|
type |
StoreType |
The type of the store. |
driver |
Optional[zenml.zen_stores.sql_zen_store.SQLDatabaseDriver] |
The SQL database driver. |
database |
Optional[str] |
database name. If not already present on the server, it will be created automatically on first access. |
username |
Optional[str] |
The database username. |
password |
Optional[str] |
The database password. |
ssl_ca |
Optional[str] |
certificate authority certificate. Required for SSL enabled authentication if the CA certificate is not part of the certificates shipped by the operating system. |
ssl_cert |
Optional[str] |
client certificate. Required for SSL enabled authentication if client certificates are used. |
ssl_key |
Optional[str] |
client certificate private key. Required for SSL enabled if client certificates are used. |
ssl_verify_server_cert |
bool |
set to verify the identity of the server against the provided server certificate. |
pool_size |
int |
The maximum number of connections to keep in the SQLAlchemy pool. |
max_overflow |
int |
The maximum number of connections to allow in the SQLAlchemy pool in addition to the pool_size. |
Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStoreConfiguration(StoreConfiguration):
"""SQL ZenML store configuration.
Attributes:
type: The type of the store.
driver: The SQL database driver.
database: database name. If not already present on the server, it will
be created automatically on first access.
username: The database username.
password: The database password.
ssl_ca: certificate authority certificate. Required for SSL
enabled authentication if the CA certificate is not part of the
certificates shipped by the operating system.
ssl_cert: client certificate. Required for SSL enabled
authentication if client certificates are used.
ssl_key: client certificate private key. Required for SSL
enabled if client certificates are used.
ssl_verify_server_cert: set to verify the identity of the server
against the provided server certificate.
pool_size: The maximum number of connections to keep in the SQLAlchemy
pool.
max_overflow: The maximum number of connections to allow in the
SQLAlchemy pool in addition to the pool_size.
"""
type: StoreType = StoreType.SQL
driver: Optional[SQLDatabaseDriver] = None
database: Optional[str] = None
username: Optional[str] = None
password: Optional[str] = None
ssl_ca: Optional[str] = None
ssl_cert: Optional[str] = None
ssl_key: Optional[str] = None
ssl_verify_server_cert: bool = False
pool_size: int = 20
max_overflow: int = 20
@root_validator(pre=True)
def _remove_grpc_attributes(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Removes old GRPC attributes.
Args:
values: All model attribute values.
Returns:
The model attribute values
"""
grpc_attribute_keys = [
"grpc_metadata_host",
"grpc_metadata_port",
"grpc_metadata_ssl_ca",
"grpc_metadata_ssl_key",
"grpc_metadata_ssl_cert",
]
grpc_values = [values.pop(key, None) for key in grpc_attribute_keys]
if any(grpc_values):
logger.warning(
"The GRPC attributes %s are unused and will be removed soon. "
"Please remove them from SQLZenStore configuration. This will "
"become an error in future versions of ZenML."
)
return values
@root_validator
def _validate_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validate the SQL URL.
The validator also moves the MySQL username, password and database
parameters from the URL into the other configuration arguments, if they
are present in the URL.
Args:
values: The values to validate.
Returns:
The validated values.
Raises:
ValueError: If the URL is invalid or the SQL driver is not
supported.
"""
url = values.get("url")
if url is None:
return values
# When running inside a container, if the URL uses localhost, the
# target service will not be available. We try to replace localhost
# with one of the special Docker or K3D internal hostnames.
url = replace_localhost_with_internal_hostname(url)
try:
sql_url = make_url(url)
except ArgumentError as e:
raise ValueError(
"Invalid SQL URL `%s`: %s. The URL must be in the format "
"`driver://[[username:password@]hostname:port]/database["
"?<extra-args>]`.",
url,
str(e),
)
if sql_url.drivername not in SQLDatabaseDriver.values():
raise ValueError(
"Invalid SQL driver value `%s`: The driver must be one of: %s.",
url,
", ".join(SQLDatabaseDriver.values()),
)
values["driver"] = SQLDatabaseDriver(sql_url.drivername)
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
if (
sql_url.username
or sql_url.password
or sql_url.query
or sql_url.database is None
):
raise ValueError(
"Invalid SQLite URL `%s`: The URL must be in the "
"format `sqlite:///path/to/database.db`.",
url,
)
if values.get("username") or values.get("password"):
raise ValueError(
"Invalid SQLite configuration: The username and password "
"must not be set",
url,
)
values["database"] = sql_url.database
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
if sql_url.username:
values["username"] = sql_url.username
sql_url = sql_url._replace(username=None)
if sql_url.password:
values["password"] = sql_url.password
sql_url = sql_url._replace(password=None)
if sql_url.database:
values["database"] = sql_url.database
sql_url = sql_url._replace(database=None)
if sql_url.query:
for k, v in sql_url.query.items():
if k == "ssl_ca":
values["ssl_ca"] = v
elif k == "ssl_cert":
values["ssl_cert"] = v
elif k == "ssl_key":
values["ssl_key"] = v
elif k == "ssl_verify_server_cert":
values["ssl_verify_server_cert"] = v
else:
raise ValueError(
"Invalid MySQL URL query parameter `%s`: The "
"parameter must be one of: ssl_ca, ssl_cert, "
"ssl_key, or ssl_verify_server_cert.",
k,
)
sql_url = sql_url._replace(query={})
database = values.get("database")
if (
not values.get("username")
or not values.get("password")
or not database
):
raise ValueError(
"Invalid MySQL configuration: The username, password and "
"database must be set in the URL or as configuration "
"attributes",
)
regexp = r"^[^\\/?%*:|\"<>.-]{1,64}$"
match = re.match(regexp, database)
if not match:
raise ValueError(
f"The database name does not conform to the required "
f"format "
f"rules ({regexp}): {database}"
)
# Save the certificates in a secure location on disk
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
content = values.get(key)
if content and not os.path.isfile(content):
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, f"{key}.pem")
with open(file_path, "w") as f:
f.write(content)
file_path.chmod(0o600)
values[key] = str(file_path)
values["url"] = str(sql_url)
return values
@staticmethod
def get_local_url(path: str) -> str:
"""Get a local SQL url for a given local path.
Args:
path: The path to the local sqlite file.
Returns:
The local SQL url for the given path.
"""
return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
"""Check if a URL scheme is supported by this store.
Args:
url: The URL to check.
Returns:
True if the URL scheme is supported, False otherwise.
"""
return make_url(url).drivername in SQLDatabaseDriver.values()
def expand_certificates(self) -> None:
"""Expands the certificates in the verify_ssl field."""
# Load the certificate values back into the configuration
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
file_path = getattr(self, key, None)
if file_path and os.path.isfile(file_path):
with open(file_path, "r") as f:
setattr(self, key, f.read())
@classmethod
def copy_configuration(
cls,
config: "StoreConfiguration",
config_path: str,
load_config_path: Optional[PurePath] = None,
) -> "StoreConfiguration":
"""Copy the store config using a different configuration path.
This method is used to create a copy of the store configuration that can
be loaded using a different configuration path or in the context of a
new environment, such as a container image.
The configuration files accompanying the store configuration are also
copied to the new configuration path (e.g. certificates etc.).
Args:
config: The store configuration to copy.
config_path: new path where the configuration copy will be loaded
from.
load_config_path: absolute path that will be used to load the copied
configuration. This can be set to a value different from
`config_path` if the configuration copy will be loaded from
a different environment, e.g. when the configuration is copied
to a container image and loaded using a different absolute path.
This will be reflected in the paths and URLs encoded in the
copied configuration.
Returns:
A new store configuration object that reflects the new configuration
path.
"""
assert isinstance(config, SqlZenStoreConfiguration)
config = config.copy()
if config.driver == SQLDatabaseDriver.MYSQL:
# Load the certificate values back into the configuration
config.expand_certificates()
elif config.driver == SQLDatabaseDriver.SQLITE:
if load_config_path:
config.url = cls.get_local_url(str(load_config_path))
else:
config.url = cls.get_local_url(config_path)
return config
def get_sqlmodel_config(
self,
) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
"""Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
The URL and connection arguments for the SQLModel engine.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
sql_url = make_url(self.url)
sqlalchemy_connect_args: Dict[str, Any] = {}
engine_args = {}
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
# The following default value is needed for sqlite to avoid the
# Error:
# sqlite3.ProgrammingError: SQLite objects created in a thread can
# only be used in that same thread.
sqlalchemy_connect_args = {"check_same_thread": False}
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
engine_args = {
"pool_size": self.pool_size,
"max_overflow": self.max_overflow,
}
sql_url = sql_url._replace(
drivername="mysql+pymysql",
username=self.username,
password=self.password,
database=self.database,
)
sqlalchemy_ssl_args: Dict[str, Any] = {}
# Handle SSL params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if not ssl_setting:
continue
if not os.path.isfile(ssl_setting):
logger.warning(
f"Database SSL setting `{key}` is not a file. "
)
sqlalchemy_ssl_args[key.lstrip("ssl_")] = ssl_setting
if len(sqlalchemy_ssl_args) > 0:
sqlalchemy_ssl_args[
"check_hostname"
] = self.ssl_verify_server_cert
sqlalchemy_connect_args["ssl"] = sqlalchemy_ssl_args
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return str(sql_url), sqlalchemy_connect_args, engine_args
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the certificate attributes can be expanded to the contents
# of the certificate files.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
Config
Pydantic configuration class.
Source code in zenml/zen_stores/sql_zen_store.py
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the certificate attributes can be expanded to the contents
# of the certificate files.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
copy_configuration(config, config_path, load_config_path=None)
classmethod
Copy the store config using a different configuration path.
This method is used to create a copy of the store configuration that can be loaded using a different configuration path or in the context of a new environment, such as a container image.
The configuration files accompanying the store configuration are also copied to the new configuration path (e.g. certificates etc.).
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The store configuration to copy. |
required |
config_path |
str |
new path where the configuration copy will be loaded from. |
required |
load_config_path |
Optional[pathlib.PurePath] |
absolute path that will be used to load the copied
configuration. This can be set to a value different from
|
None |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
A new store configuration object that reflects the new configuration path. |
Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def copy_configuration(
cls,
config: "StoreConfiguration",
config_path: str,
load_config_path: Optional[PurePath] = None,
) -> "StoreConfiguration":
"""Copy the store config using a different configuration path.
This method is used to create a copy of the store configuration that can
be loaded using a different configuration path or in the context of a
new environment, such as a container image.
The configuration files accompanying the store configuration are also
copied to the new configuration path (e.g. certificates etc.).
Args:
config: The store configuration to copy.
config_path: new path where the configuration copy will be loaded
from.
load_config_path: absolute path that will be used to load the copied
configuration. This can be set to a value different from
`config_path` if the configuration copy will be loaded from
a different environment, e.g. when the configuration is copied
to a container image and loaded using a different absolute path.
This will be reflected in the paths and URLs encoded in the
copied configuration.
Returns:
A new store configuration object that reflects the new configuration
path.
"""
assert isinstance(config, SqlZenStoreConfiguration)
config = config.copy()
if config.driver == SQLDatabaseDriver.MYSQL:
# Load the certificate values back into the configuration
config.expand_certificates()
elif config.driver == SQLDatabaseDriver.SQLITE:
if load_config_path:
config.url = cls.get_local_url(str(load_config_path))
else:
config.url = cls.get_local_url(config_path)
return config
expand_certificates(self)
Expands the certificates in the verify_ssl field.
Source code in zenml/zen_stores/sql_zen_store.py
def expand_certificates(self) -> None:
"""Expands the certificates in the verify_ssl field."""
# Load the certificate values back into the configuration
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
file_path = getattr(self, key, None)
if file_path and os.path.isfile(file_path):
with open(file_path, "r") as f:
setattr(self, key, f.read())
get_local_url(path)
staticmethod
Get a local SQL url for a given local path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the local sqlite file. |
required |
Returns:
| Type | Description |
|---|---|
str |
The local SQL url for the given path. |
Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def get_local_url(path: str) -> str:
"""Get a local SQL url for a given local path.
Args:
path: The path to the local sqlite file.
Returns:
The local SQL url for the given path.
"""
return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"
get_sqlmodel_config(self)
Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
| Type | Description |
|---|---|
Tuple[str, Dict[str, Any], Dict[str, Any]] |
The URL and connection arguments for the SQLModel engine. |
Exceptions:
| Type | Description |
|---|---|
NotImplementedError |
If the SQL driver is not supported. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_sqlmodel_config(
self,
) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
"""Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
The URL and connection arguments for the SQLModel engine.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
sql_url = make_url(self.url)
sqlalchemy_connect_args: Dict[str, Any] = {}
engine_args = {}
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
# The following default value is needed for sqlite to avoid the
# Error:
# sqlite3.ProgrammingError: SQLite objects created in a thread can
# only be used in that same thread.
sqlalchemy_connect_args = {"check_same_thread": False}
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
engine_args = {
"pool_size": self.pool_size,
"max_overflow": self.max_overflow,
}
sql_url = sql_url._replace(
drivername="mysql+pymysql",
username=self.username,
password=self.password,
database=self.database,
)
sqlalchemy_ssl_args: Dict[str, Any] = {}
# Handle SSL params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if not ssl_setting:
continue
if not os.path.isfile(ssl_setting):
logger.warning(
f"Database SSL setting `{key}` is not a file. "
)
sqlalchemy_ssl_args[key.lstrip("ssl_")] = ssl_setting
if len(sqlalchemy_ssl_args) > 0:
sqlalchemy_ssl_args[
"check_hostname"
] = self.ssl_verify_server_cert
sqlalchemy_connect_args["ssl"] = sqlalchemy_ssl_args
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return str(sql_url), sqlalchemy_connect_args, engine_args
supports_url_scheme(url)
classmethod
Check if a URL scheme is supported by this store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The URL to check. |
required |
Returns:
| Type | Description |
|---|---|
bool |
True if the URL scheme is supported, False otherwise. |
Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def supports_url_scheme(cls, url: str) -> bool:
"""Check if a URL scheme is supported by this store.
Args:
url: The URL to check.
Returns:
True if the URL scheme is supported, False otherwise.
"""
return make_url(url).drivername in SQLDatabaseDriver.values()
zen_store_interface
ZenML Store interface.
ZenStoreInterface (ABC)
ZenML store interface.
All ZenML stores must implement the methods in this interface.
The methods in this interface are organized in the following way:
-
they are grouped into categories based on the type of resource that they operate on (e.g. stacks, stack components, etc.)
-
each category has a set of CRUD methods (create, read, update, delete) that operate on the resources in that category. The order of the methods in each category should be:
-
create methods - store a new resource. These methods should fill in generated fields (e.g. UUIDs, creation timestamps) in the resource and return the updated resource.
- get methods - retrieve a single existing resource identified by a unique key or identifier from the store. These methods should always return a resource and raise an exception if the resource does not exist.
- list methods - retrieve a list of resources from the store. These methods should accept a set of filter parameters that can be used to filter the list of resources retrieved from the store.
- update methods - update an existing resource in the store. These methods should expect the updated resource to be correctly identified by its unique key or identifier and raise an exception if the resource does not exist.
- delete methods - delete an existing resource from the store. These methods should expect the resource to be correctly identified by its unique key or identifier. If the resource does not exist, an exception should be raised.
Best practices for implementing and keeping this interface clean and easy to maintain and extend:
- keep methods organized by resource type and ordered by CRUD operation
- for resources with multiple keys, don't implement multiple get or list methods here if the same functionality can be achieved by a single get or list method. Instead, implement them in the BaseZenStore class and have them call the generic get or list method in this interface.
- keep the logic required to convert between ZenML domain Model classes and internal store representations outside the ZenML domain Model classes
- methods for resources that have two or more unique keys (e.g. a Workspace
is uniquely identified by its name as well as its UUID) should reflect
that in the method variants and/or method arguments:
- methods that take in a resource identifier as argument should accept
all variants of the identifier (e.g.
workspace_name_or_uuidfor methods that get/list/update/delete Workspaces) - if a compound key is involved, separate get methods should be
implemented (e.g.
get_pipelineto get a pipeline by ID andget_pipeline_in_workspaceto get a pipeline by its name and the ID of the workspace it belongs to)
- methods that take in a resource identifier as argument should accept
all variants of the identifier (e.g.
- methods for resources that are scoped as children of other resources
(e.g. a Stack is always owned by a Workspace) should reflect the
key(s) of the parent resource in the provided methods and method
arguments:
- create methods should take the parent resource UUID(s) as an argument
(e.g.
create_stacktakes in the workspace ID) - get methods should be provided to retrieve a resource by the compound key that includes the parent resource key(s)
- list methods should feature optional filter arguments that reflect the parent resource key(s)
- create methods should take the parent resource UUID(s) as an argument
(e.g.
Source code in zenml/zen_stores/zen_store_interface.py
class ZenStoreInterface(ABC):
"""ZenML store interface.
All ZenML stores must implement the methods in this interface.
The methods in this interface are organized in the following way:
* they are grouped into categories based on the type of resource
that they operate on (e.g. stacks, stack components, etc.)
* each category has a set of CRUD methods (create, read, update, delete)
that operate on the resources in that category. The order of the methods
in each category should be:
* create methods - store a new resource. These methods
should fill in generated fields (e.g. UUIDs, creation timestamps) in
the resource and return the updated resource.
* get methods - retrieve a single existing resource identified by a
unique key or identifier from the store. These methods should always
return a resource and raise an exception if the resource does not
exist.
* list methods - retrieve a list of resources from the store. These
methods should accept a set of filter parameters that can be used to
filter the list of resources retrieved from the store.
* update methods - update an existing resource in the store. These
methods should expect the updated resource to be correctly identified
by its unique key or identifier and raise an exception if the resource
does not exist.
* delete methods - delete an existing resource from the store. These
methods should expect the resource to be correctly identified by its
unique key or identifier. If the resource does not exist,
an exception should be raised.
Best practices for implementing and keeping this interface clean and easy to
maintain and extend:
* keep methods organized by resource type and ordered by CRUD operation
* for resources with multiple keys, don't implement multiple get or list
methods here if the same functionality can be achieved by a single get or
list method. Instead, implement them in the BaseZenStore class and have
them call the generic get or list method in this interface.
* keep the logic required to convert between ZenML domain Model classes
and internal store representations outside the ZenML domain Model classes
* methods for resources that have two or more unique keys (e.g. a Workspace
is uniquely identified by its name as well as its UUID) should reflect
that in the method variants and/or method arguments:
* methods that take in a resource identifier as argument should accept
all variants of the identifier (e.g. `workspace_name_or_uuid` for methods
that get/list/update/delete Workspaces)
* if a compound key is involved, separate get methods should be
implemented (e.g. `get_pipeline` to get a pipeline by ID and
`get_pipeline_in_workspace` to get a pipeline by its name and the ID of
the workspace it belongs to)
* methods for resources that are scoped as children of other resources
(e.g. a Stack is always owned by a Workspace) should reflect the
key(s) of the parent resource in the provided methods and method
arguments:
* create methods should take the parent resource UUID(s) as an argument
(e.g. `create_stack` takes in the workspace ID)
* get methods should be provided to retrieve a resource by the compound
key that includes the parent resource key(s)
* list methods should feature optional filter arguments that reflect
the parent resource key(s)
"""
# ---------------------------------
# Initialization and configuration
# ---------------------------------
@abstractmethod
def _initialize(self) -> None:
"""Initialize the store.
This method is called immediately after the store is created. It should
be used to set up the backend (database, connection etc.).
"""
@abstractmethod
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
# ------
# Stacks
# ------
@abstractmethod
def create_stack(self, stack: StackRequestModel) -> StackResponseModel:
"""Create a new stack.
Args:
stack: The stack to create.
Returns:
The created stack.
Raises:
StackExistsError: If a stack with the same name is already owned
by this user in this workspace.
"""
@abstractmethod
def get_stack(self, stack_id: UUID) -> StackResponseModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
Raises:
KeyError: if the stack doesn't exist.
"""
@abstractmethod
def list_stacks(
self, stack_filter_model: StackFilterModel
) -> Page[StackResponseModel]:
"""List all stacks matching the given filter criteria.
Args:
stack_filter_model: All filter parameters including pagination
params
Returns:
A list of all stacks matching the filter criteria.
"""
@abstractmethod
def update_stack(
self, stack_id: UUID, stack_update: StackUpdateModel
) -> StackResponseModel:
"""Update a stack.
Args:
stack_id: The ID of the stack update.
stack_update: The update request on the stack.
Returns:
The updated stack.
Raises:
KeyError: if the stack doesn't exist.
"""
@abstractmethod
def delete_stack(self, stack_id: UUID) -> None:
"""Delete a stack.
Args:
stack_id: The ID of the stack to delete.
Raises:
KeyError: if the stack doesn't exist.
"""
# ----------------
# Stack components
# ----------------
@abstractmethod
def create_stack_component(
self, component: ComponentRequestModel
) -> ComponentResponseModel:
"""Create a stack component.
Args:
component: The stack component to create.
Returns:
The created stack component.
Raises:
StackComponentExistsError: If a stack component with the same name
and type is already owned by this user in this workspace.
"""
@abstractmethod
def list_stack_components(
self, component_filter_model: ComponentFilterModel
) -> Page[ComponentResponseModel]:
"""List all stack components matching the given filter criteria.
Args:
component_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stack components matching the filter criteria.
"""
@abstractmethod
def get_stack_component(
self, component_id: UUID
) -> ComponentResponseModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
@abstractmethod
def update_stack_component(
self,
component_id: UUID,
component_update: ComponentUpdateModel,
) -> ComponentResponseModel:
"""Update an existing stack component.
Args:
component_id: The ID of the stack component to update.
component_update: The update to be applied to the stack component.
Returns:
The updated stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
@abstractmethod
def delete_stack_component(self, component_id: UUID) -> None:
"""Delete a stack component.
Args:
component_id: The ID of the stack component to delete.
Raises:
KeyError: if the stack component doesn't exist.
ValueError: if the stack component is part of one or more stacks.
"""
# -----------------------
# Stack component flavors
# -----------------------
@abstractmethod
def create_flavor(
self,
flavor: FlavorRequestModel,
) -> FlavorResponseModel:
"""Creates a new stack component flavor.
Args:
flavor: The stack component flavor to create.
Returns:
The newly created flavor.
Raises:
EntityExistsError: If a flavor with the same name and type
is already owned by this user in this workspace.
"""
@abstractmethod
def get_flavor(self, flavor_id: UUID) -> FlavorResponseModel:
"""Get a stack component flavor by ID.
Args:
flavor_id: The ID of the flavor to get.
Returns:
The stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
@abstractmethod
def list_flavors(
self, flavor_filter_model: FlavorFilterModel
) -> Page[FlavorResponseModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
flavor_filter_model: All filter parameters including pagination
params.
Returns:
List of all the stack component flavors matching the given criteria.
"""
@abstractmethod
def delete_flavor(self, flavor_id: UUID) -> None:
"""Delete a stack component flavor.
Args:
flavor_id: The ID of the stack component flavor to delete.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
# -----
# Users
# -----
@abstractmethod
def create_user(self, user: UserRequestModel) -> UserResponseModel:
"""Creates a new user.
Args:
user: User to be created.
Returns:
The newly created user.
Raises:
EntityExistsError: If a user with the given name already exists.
"""
@abstractmethod
def get_user(
self,
user_name_or_id: Optional[Union[str, UUID]] = None,
include_private: bool = False,
) -> UserResponseModel:
"""Gets a specific user, when no id is specified the active user is returned.
Args:
user_name_or_id: The name or ID of the user to get.
include_private: Whether to include private user information
Returns:
The requested user, if it was found.
Raises:
KeyError: If no user with the given name or ID exists.
"""
@abstractmethod
def get_auth_user(
self, user_name_or_id: Union[str, UUID]
) -> UserAuthModel:
"""Gets the auth model to a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
"""
@abstractmethod
def list_users(
self, user_filter_model: UserFilterModel
) -> Page[UserResponseModel]:
"""List all users.
Args:
user_filter_model: All filter parameters including pagination
params.
Returns:
A list of all users.
"""
@abstractmethod
def update_user(
self, user_id: UUID, user_update: UserUpdateModel
) -> UserResponseModel:
"""Updates an existing user.
Args:
user_id: The id of the user to update.
user_update: The update to be applied to the user.
Returns:
The updated user.
Raises:
KeyError: If no user with the given name exists.
"""
@abstractmethod
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
"""Deletes a user.
Args:
user_name_or_id: The name or ID of the user to delete.
Raises:
KeyError: If no user with the given ID exists.
"""
# -----
# Teams
# -----
@abstractmethod
def create_team(self, team: TeamRequestModel) -> TeamResponseModel:
"""Creates a new team.
Args:
team: The team model to create.
Returns:
The newly created team.
"""
@abstractmethod
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamResponseModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
Raises:
KeyError: If no team with the given name or ID exists.
"""
@abstractmethod
def list_teams(
self, team_filter_model: TeamFilterModel
) -> Page[TeamResponseModel]:
"""List all teams matching the given filter criteria.
Args:
team_filter_model: All filter parameters including pagination
params.
Returns:
A list of all teams matching the filter criteria.
"""
@abstractmethod
def update_team(
self, team_id: UUID, team_update: TeamUpdateModel
) -> TeamResponseModel:
"""Update an existing team.
Args:
team_id: The ID of the team to be updated.
team_update: The update to be applied to the team.
Returns:
The updated team.
Raises:
KeyError: if the team does not exist.
"""
@abstractmethod
def delete_team(self, team_name_or_id: Union[str, UUID]) -> None:
"""Deletes a team.
Args:
team_name_or_id: Name or ID of the team to delete.
Raises:
KeyError: If no team with the given ID exists.
"""
# -----
# Roles
# -----
@abstractmethod
def create_role(self, role: RoleRequestModel) -> RoleResponseModel:
"""Creates a new role.
Args:
role: The role model to create.
Returns:
The newly created role.
Raises:
EntityExistsError: If a role with the given name already exists.
"""
@abstractmethod
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleResponseModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
Raises:
KeyError: If no role with the given name exists.
"""
@abstractmethod
def list_roles(
self, role_filter_model: RoleFilterModel
) -> Page[RoleResponseModel]:
"""List all roles matching the given filter criteria.
Args:
role_filter_model: All filter parameters including pagination
params.
Returns:
A list of all roles matching the filter criteria.
"""
@abstractmethod
def update_role(
self, role_id: UUID, role_update: RoleUpdateModel
) -> RoleResponseModel:
"""Update an existing role.
Args:
role_id: The ID of the role to be updated.
role_update: The update to be applied to the role.
Returns:
The updated role.
Raises:
KeyError: if the role does not exist.
"""
@abstractmethod
def delete_role(self, role_name_or_id: Union[str, UUID]) -> None:
"""Deletes a role.
Args:
role_name_or_id: Name or ID of the role to delete.
Raises:
KeyError: If no role with the given ID exists.
"""
# ---------------------
# User Role assignments
# ---------------------
@abstractmethod
def create_user_role_assignment(
self, user_role_assignment: UserRoleAssignmentRequestModel
) -> UserRoleAssignmentResponseModel:
"""Creates a new role assignment.
Args:
user_role_assignment: The role assignment model to create.
Returns:
The newly created role assignment.
"""
@abstractmethod
def get_user_role_assignment(
self, user_role_assignment_id: UUID
) -> UserRoleAssignmentResponseModel:
"""Gets a specific role assignment.
Args:
user_role_assignment_id: ID of the role assignment to get.
Returns:
The requested role assignment.
Raises:
KeyError: If no role assignment with the given ID exists.
"""
@abstractmethod
def delete_user_role_assignment(
self, user_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
user_role_assignment_id: The ID of the specific role assignment
"""
@abstractmethod
def list_user_role_assignments(
self, user_role_assignment_filter_model: UserRoleAssignmentFilterModel
) -> Page[UserRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
user_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
# ---------------------
# Team Role assignments
# ---------------------
@abstractmethod
def create_team_role_assignment(
self, team_role_assignment: TeamRoleAssignmentRequestModel
) -> TeamRoleAssignmentResponseModel:
"""Creates a new team role assignment.
Args:
team_role_assignment: The role assignment model to create.
Returns:
The newly created role assignment.
"""
@abstractmethod
def get_team_role_assignment(
self, team_role_assignment_id: UUID
) -> TeamRoleAssignmentResponseModel:
"""Gets a specific role assignment.
Args:
team_role_assignment_id: ID of the role assignment to get.
Returns:
The requested role assignment.
Raises:
KeyError: If no role assignment with the given ID exists.
"""
@abstractmethod
def delete_team_role_assignment(
self, team_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
team_role_assignment_id: The ID of the specific role assignment
"""
@abstractmethod
def list_team_role_assignments(
self, team_role_assignment_filter_model: TeamRoleAssignmentFilterModel
) -> Page[TeamRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
team_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
# --------
# Workspaces
# --------
@abstractmethod
def create_workspace(
self, workspace: WorkspaceRequestModel
) -> WorkspaceResponseModel:
"""Creates a new workspace.
Args:
workspace: The workspace to create.
Returns:
The newly created workspace.
Raises:
EntityExistsError: If a workspace with the given name already exists.
"""
@abstractmethod
def get_workspace(
self, workspace_name_or_id: Union[UUID, str]
) -> WorkspaceResponseModel:
"""Get an existing workspace by name or ID.
Args:
workspace_name_or_id: Name or ID of the workspace to get.
Returns:
The requested workspace.
Raises:
KeyError: If there is no such workspace.
"""
@abstractmethod
def list_workspaces(
self, workspace_filter_model: WorkspaceFilterModel
) -> Page[WorkspaceResponseModel]:
"""List all workspace matching the given filter criteria.
Args:
workspace_filter_model: All filter parameters including pagination
params.
Returns:
A list of all workspace matching the filter criteria.
"""
@abstractmethod
def update_workspace(
self, workspace_id: UUID, workspace_update: WorkspaceUpdateModel
) -> WorkspaceResponseModel:
"""Update an existing workspace.
Args:
workspace_id: The ID of the workspace to be updated.
workspace_update: The update to be applied to the workspace.
Returns:
The updated workspace.
Raises:
KeyError: if the workspace does not exist.
"""
@abstractmethod
def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
"""Deletes a workspace.
Args:
workspace_name_or_id: Name or ID of the workspace to delete.
Raises:
KeyError: If no workspace with the given name exists.
"""
# ---------
# Pipelines
# ---------
@abstractmethod
def create_pipeline(
self,
pipeline: PipelineRequestModel,
) -> PipelineResponseModel:
"""Creates a new pipeline in a workspace.
Args:
pipeline: The pipeline to create.
Returns:
The newly created pipeline.
Raises:
KeyError: if the workspace does not exist.
EntityExistsError: If an identical pipeline already exists.
"""
@abstractmethod
def get_pipeline(self, pipeline_id: UUID) -> PipelineResponseModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
@abstractmethod
def list_pipelines(
self, pipeline_filter_model: PipelineFilterModel
) -> Page[PipelineResponseModel]:
"""List all pipelines matching the given filter criteria.
Args:
pipeline_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipelines matching the filter criteria.
"""
@abstractmethod
def update_pipeline(
self,
pipeline_id: UUID,
pipeline_update: PipelineUpdateModel,
) -> PipelineResponseModel:
"""Updates a pipeline.
Args:
pipeline_id: The ID of the pipeline to be updated.
pipeline_update: The update to be applied.
Returns:
The updated pipeline.
Raises:
KeyError: if the pipeline doesn't exist.
"""
@abstractmethod
def delete_pipeline(self, pipeline_id: UUID) -> None:
"""Deletes a pipeline.
Args:
pipeline_id: The ID of the pipeline to delete.
Raises:
KeyError: if the pipeline doesn't exist.
"""
# ---------
# Schedules
# ---------
@abstractmethod
def create_schedule(
self, schedule: ScheduleRequestModel
) -> ScheduleResponseModel:
"""Creates a new schedule.
Args:
schedule: The schedule to create.
Returns:
The newly created schedule.
"""
@abstractmethod
def get_schedule(self, schedule_id: UUID) -> ScheduleResponseModel:
"""Get a schedule with a given ID.
Args:
schedule_id: ID of the schedule.
Returns:
The schedule.
Raises:
KeyError: if the schedule does not exist.
"""
@abstractmethod
def list_schedules(
self, schedule_filter_model: ScheduleFilterModel
) -> Page[ScheduleResponseModel]:
"""List all schedules in the workspace.
Args:
schedule_filter_model: All filter parameters including pagination
params
Returns:
A list of schedules.
"""
@abstractmethod
def update_schedule(
self,
schedule_id: UUID,
schedule_update: ScheduleUpdateModel,
) -> ScheduleResponseModel:
"""Updates a schedule.
Args:
schedule_id: The ID of the schedule to be updated.
schedule_update: The update to be applied.
Returns:
The updated schedule.
Raises:
KeyError: if the schedule doesn't exist.
"""
@abstractmethod
def delete_schedule(self, schedule_id: UUID) -> None:
"""Deletes a schedule.
Args:
schedule_id: The ID of the schedule to delete.
Raises:
KeyError: if the schedule doesn't exist.
"""
# --------------
# Pipeline runs
# --------------
@abstractmethod
def create_run(
self, pipeline_run: PipelineRunRequestModel
) -> PipelineRunResponseModel:
"""Creates a pipeline run.
Args:
pipeline_run: The pipeline run to create.
Returns:
The created pipeline run.
Raises:
EntityExistsError: If an identical pipeline run already exists.
KeyError: If the pipeline does not exist.
"""
@abstractmethod
def get_run(
self, run_name_or_id: Union[str, UUID]
) -> PipelineRunResponseModel:
"""Gets a pipeline run.
Args:
run_name_or_id: The name or ID of the pipeline run to get.
Returns:
The pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
@abstractmethod
def get_or_create_run(
self, pipeline_run: PipelineRunRequestModel
) -> Tuple[PipelineRunResponseModel, bool]:
"""Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned.
Otherwise, a new run is created.
Args:
pipeline_run: The pipeline run to get or create.
Returns:
The pipeline run, and a boolean indicating whether the run was
created or not.
"""
@abstractmethod
def list_runs(
self, runs_filter_model: PipelineRunFilterModel
) -> Page[PipelineRunResponseModel]:
"""List all pipeline runs matching the given filter criteria.
Args:
runs_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipeline runs matching the filter criteria.
"""
@abstractmethod
def update_run(
self, run_id: UUID, run_update: PipelineRunUpdateModel
) -> PipelineRunResponseModel:
"""Updates a pipeline run.
Args:
run_id: The ID of the pipeline run to update.
run_update: The update to be applied to the pipeline run.
Returns:
The updated pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
@abstractmethod
def delete_run(self, run_id: UUID) -> None:
"""Deletes a pipeline run.
Args:
run_id: The ID of the pipeline run to delete.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
# ------------------
# Pipeline run steps
# ------------------
@abstractmethod
def create_run_step(
self, step_run: StepRunRequestModel
) -> StepRunResponseModel:
"""Creates a step run.
Args:
step_run: The step run to create.
Returns:
The created step run.
Raises:
EntityExistsError: if the step run already exists.
KeyError: if the pipeline run doesn't exist.
"""
@abstractmethod
def get_run_step(self, step_run_id: UUID) -> StepRunResponseModel:
"""Get a step run by ID.
Args:
step_run_id: The ID of the step run to get.
Returns:
The step run.
Raises:
KeyError: if the step run doesn't exist.
"""
@abstractmethod
def list_run_steps(
self, step_run_filter_model: StepRunFilterModel
) -> Page[StepRunResponseModel]:
"""List all step runs matching the given filter criteria.
Args:
step_run_filter_model: All filter parameters including pagination
params.
Returns:
A list of all step runs matching the filter criteria.
"""
@abstractmethod
def update_run_step(
self,
step_run_id: UUID,
step_run_update: StepRunUpdateModel,
) -> StepRunResponseModel:
"""Updates a step run.
Args:
step_run_id: The ID of the step to update.
step_run_update: The update to be applied to the step.
Returns:
The updated step run.
Raises:
KeyError: if the step run doesn't exist.
"""
# ---------
# Artifacts
# ---------
@abstractmethod
def create_artifact(
self, artifact: ArtifactRequestModel
) -> ArtifactResponseModel:
"""Creates an artifact.
Args:
artifact: The artifact to create.
Returns:
The created artifact.
"""
@abstractmethod
def get_artifact(self, artifact_id: UUID) -> ArtifactResponseModel:
"""Gets an artifact.
Args:
artifact_id: The ID of the artifact to get.
Returns:
The artifact.
Raises:
KeyError: if the artifact doesn't exist.
"""
@abstractmethod
def list_artifacts(
self, artifact_filter_model: ArtifactFilterModel
) -> Page[ArtifactResponseModel]:
"""List all artifacts matching the given filter criteria.
Args:
artifact_filter_model: All filter parameters including pagination
params.
Returns:
A list of all artifacts matching the filter criteria.
"""
@abstractmethod
def delete_artifact(self, artifact_id: UUID) -> None:
"""Deletes an artifact.
Args:
artifact_id: The ID of the artifact to delete.
Raises:
KeyError: if the artifact doesn't exist.
"""
# ------------
# Run Metadata
# ------------
@abstractmethod
def create_run_metadata(
self, run_metadata: RunMetadataRequestModel
) -> RunMetadataResponseModel:
"""Creates run metadata.
Args:
run_metadata: The run metadata to create.
Returns:
The created run metadata.
"""
@abstractmethod
def list_run_metadata(
self,
run_metadata_filter_model: RunMetadataFilterModel,
) -> Page[RunMetadataResponseModel]:
"""List run metadata.
Args:
run_metadata_filter_model: All filter parameters including
pagination params.
Returns:
The run metadata.
"""
create_artifact(self, artifact)
Creates an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact |
ArtifactRequestModel |
The artifact to create. |
required |
Returns:
| Type | Description |
|---|---|
ArtifactResponseModel |
The created artifact. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_artifact(
self, artifact: ArtifactRequestModel
) -> ArtifactResponseModel:
"""Creates an artifact.
Args:
artifact: The artifact to create.
Returns:
The created artifact.
"""
create_flavor(self, flavor)
Creates a new stack component flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor |
FlavorRequestModel |
The stack component flavor to create. |
required |
Returns:
| Type | Description |
|---|---|
FlavorResponseModel |
The newly created flavor. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a flavor with the same name and type is already owned by this user in this workspace. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_flavor(
self,
flavor: FlavorRequestModel,
) -> FlavorResponseModel:
"""Creates a new stack component flavor.
Args:
flavor: The stack component flavor to create.
Returns:
The newly created flavor.
Raises:
EntityExistsError: If a flavor with the same name and type
is already owned by this user in this workspace.
"""
create_pipeline(self, pipeline)
Creates a new pipeline in a workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline |
PipelineRequestModel |
The pipeline to create. |
required |
Returns:
| Type | Description |
|---|---|
PipelineResponseModel |
The newly created pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the workspace does not exist. |
EntityExistsError |
If an identical pipeline already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_pipeline(
self,
pipeline: PipelineRequestModel,
) -> PipelineResponseModel:
"""Creates a new pipeline in a workspace.
Args:
pipeline: The pipeline to create.
Returns:
The newly created pipeline.
Raises:
KeyError: if the workspace does not exist.
EntityExistsError: If an identical pipeline already exists.
"""
create_role(self, role)
Creates a new role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role |
RoleRequestModel |
The role model to create. |
required |
Returns:
| Type | Description |
|---|---|
RoleResponseModel |
The newly created role. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a role with the given name already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_role(self, role: RoleRequestModel) -> RoleResponseModel:
"""Creates a new role.
Args:
role: The role model to create.
Returns:
The newly created role.
Raises:
EntityExistsError: If a role with the given name already exists.
"""
create_run(self, pipeline_run)
Creates a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_run |
PipelineRunRequestModel |
The pipeline run to create. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The created pipeline run. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If an identical pipeline run already exists. |
KeyError |
If the pipeline does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_run(
self, pipeline_run: PipelineRunRequestModel
) -> PipelineRunResponseModel:
"""Creates a pipeline run.
Args:
pipeline_run: The pipeline run to create.
Returns:
The created pipeline run.
Raises:
EntityExistsError: If an identical pipeline run already exists.
KeyError: If the pipeline does not exist.
"""
create_run_metadata(self, run_metadata)
Creates run metadata.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_metadata |
RunMetadataRequestModel |
The run metadata to create. |
required |
Returns:
| Type | Description |
|---|---|
RunMetadataResponseModel |
The created run metadata. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_run_metadata(
self, run_metadata: RunMetadataRequestModel
) -> RunMetadataResponseModel:
"""Creates run metadata.
Args:
run_metadata: The run metadata to create.
Returns:
The created run metadata.
"""
create_run_step(self, step_run)
Creates a step run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run |
StepRunRequestModel |
The step run to create. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The created step run. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
if the step run already exists. |
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_run_step(
self, step_run: StepRunRequestModel
) -> StepRunResponseModel:
"""Creates a step run.
Args:
step_run: The step run to create.
Returns:
The created step run.
Raises:
EntityExistsError: if the step run already exists.
KeyError: if the pipeline run doesn't exist.
"""
create_schedule(self, schedule)
Creates a new schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule |
ScheduleRequestModel |
The schedule to create. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The newly created schedule. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_schedule(
self, schedule: ScheduleRequestModel
) -> ScheduleResponseModel:
"""Creates a new schedule.
Args:
schedule: The schedule to create.
Returns:
The newly created schedule.
"""
create_stack(self, stack)
Create a new stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack |
StackRequestModel |
The stack to create. |
required |
Returns:
| Type | Description |
|---|---|
StackResponseModel |
The created stack. |
Exceptions:
| Type | Description |
|---|---|
StackExistsError |
If a stack with the same name is already owned by this user in this workspace. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_stack(self, stack: StackRequestModel) -> StackResponseModel:
"""Create a new stack.
Args:
stack: The stack to create.
Returns:
The created stack.
Raises:
StackExistsError: If a stack with the same name is already owned
by this user in this workspace.
"""
create_stack_component(self, component)
Create a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
ComponentRequestModel |
The stack component to create. |
required |
Returns:
| Type | Description |
|---|---|
ComponentResponseModel |
The created stack component. |
Exceptions:
| Type | Description |
|---|---|
StackComponentExistsError |
If a stack component with the same name and type is already owned by this user in this workspace. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_stack_component(
self, component: ComponentRequestModel
) -> ComponentResponseModel:
"""Create a stack component.
Args:
component: The stack component to create.
Returns:
The created stack component.
Raises:
StackComponentExistsError: If a stack component with the same name
and type is already owned by this user in this workspace.
"""
create_team(self, team)
Creates a new team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team |
TeamRequestModel |
The team model to create. |
required |
Returns:
| Type | Description |
|---|---|
TeamResponseModel |
The newly created team. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_team(self, team: TeamRequestModel) -> TeamResponseModel:
"""Creates a new team.
Args:
team: The team model to create.
Returns:
The newly created team.
"""
create_team_role_assignment(self, team_role_assignment)
Creates a new team role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment |
TeamRoleAssignmentRequestModel |
The role assignment model to create. |
required |
Returns:
| Type | Description |
|---|---|
TeamRoleAssignmentResponseModel |
The newly created role assignment. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_team_role_assignment(
self, team_role_assignment: TeamRoleAssignmentRequestModel
) -> TeamRoleAssignmentResponseModel:
"""Creates a new team role assignment.
Args:
team_role_assignment: The role assignment model to create.
Returns:
The newly created role assignment.
"""
create_user(self, user)
Creates a new user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user |
UserRequestModel |
User to be created. |
required |
Returns:
| Type | Description |
|---|---|
UserResponseModel |
The newly created user. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a user with the given name already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_user(self, user: UserRequestModel) -> UserResponseModel:
"""Creates a new user.
Args:
user: User to be created.
Returns:
The newly created user.
Raises:
EntityExistsError: If a user with the given name already exists.
"""
create_user_role_assignment(self, user_role_assignment)
Creates a new role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment |
UserRoleAssignmentRequestModel |
The role assignment model to create. |
required |
Returns:
| Type | Description |
|---|---|
UserRoleAssignmentResponseModel |
The newly created role assignment. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_user_role_assignment(
self, user_role_assignment: UserRoleAssignmentRequestModel
) -> UserRoleAssignmentResponseModel:
"""Creates a new role assignment.
Args:
user_role_assignment: The role assignment model to create.
Returns:
The newly created role assignment.
"""
create_workspace(self, workspace)
Creates a new workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace |
WorkspaceRequestModel |
The workspace to create. |
required |
Returns:
| Type | Description |
|---|---|
WorkspaceResponseModel |
The newly created workspace. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a workspace with the given name already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_workspace(
self, workspace: WorkspaceRequestModel
) -> WorkspaceResponseModel:
"""Creates a new workspace.
Args:
workspace: The workspace to create.
Returns:
The newly created workspace.
Raises:
EntityExistsError: If a workspace with the given name already exists.
"""
delete_artifact(self, artifact_id)
Deletes an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_id |
UUID |
The ID of the artifact to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the artifact doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_artifact(self, artifact_id: UUID) -> None:
"""Deletes an artifact.
Args:
artifact_id: The ID of the artifact to delete.
Raises:
KeyError: if the artifact doesn't exist.
"""
delete_flavor(self, flavor_id)
Delete a stack component flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
UUID |
The ID of the stack component flavor to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component flavor doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_flavor(self, flavor_id: UUID) -> None:
"""Delete a stack component flavor.
Args:
flavor_id: The ID of the stack component flavor to delete.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
delete_pipeline(self, pipeline_id)
Deletes a pipeline.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
The ID of the pipeline to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_pipeline(self, pipeline_id: UUID) -> None:
"""Deletes a pipeline.
Args:
pipeline_id: The ID of the pipeline to delete.
Raises:
KeyError: if the pipeline doesn't exist.
"""
delete_role(self, role_name_or_id)
Deletes a role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no role with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_role(self, role_name_or_id: Union[str, UUID]) -> None:
"""Deletes a role.
Args:
role_name_or_id: Name or ID of the role to delete.
Raises:
KeyError: If no role with the given ID exists.
"""
delete_run(self, run_id)
Deletes a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_run(self, run_id: UUID) -> None:
"""Deletes a pipeline run.
Args:
run_id: The ID of the pipeline run to delete.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
delete_schedule(self, schedule_id)
Deletes a schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
The ID of the schedule to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the schedule doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_schedule(self, schedule_id: UUID) -> None:
"""Deletes a schedule.
Args:
schedule_id: The ID of the schedule to delete.
Raises:
KeyError: if the schedule doesn't exist.
"""
delete_stack(self, stack_id)
Delete a stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_stack(self, stack_id: UUID) -> None:
"""Delete a stack.
Args:
stack_id: The ID of the stack to delete.
Raises:
KeyError: if the stack doesn't exist.
"""
delete_stack_component(self, component_id)
Delete a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
ValueError |
if the stack component is part of one or more stacks. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_stack_component(self, component_id: UUID) -> None:
"""Delete a stack component.
Args:
component_id: The ID of the stack component to delete.
Raises:
KeyError: if the stack component doesn't exist.
ValueError: if the stack component is part of one or more stacks.
"""
delete_team(self, team_name_or_id)
Deletes a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no team with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_team(self, team_name_or_id: Union[str, UUID]) -> None:
"""Deletes a team.
Args:
team_name_or_id: Name or ID of the team to delete.
Raises:
KeyError: If no team with the given ID exists.
"""
delete_team_role_assignment(self, team_role_assignment_id)
Delete a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_id |
UUID |
The ID of the specific role assignment |
required |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_team_role_assignment(
self, team_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
team_role_assignment_id: The ID of the specific role assignment
"""
delete_user(self, user_name_or_id)
Deletes a user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
"""Deletes a user.
Args:
user_name_or_id: The name or ID of the user to delete.
Raises:
KeyError: If no user with the given ID exists.
"""
delete_user_role_assignment(self, user_role_assignment_id)
Delete a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_id |
UUID |
The ID of the specific role assignment |
required |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_user_role_assignment(
self, user_role_assignment_id: UUID
) -> None:
"""Delete a specific role assignment.
Args:
user_role_assignment_id: The ID of the specific role assignment
"""
delete_workspace(self, workspace_name_or_id)
Deletes a workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the workspace to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no workspace with the given name exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_workspace(self, workspace_name_or_id: Union[str, UUID]) -> None:
"""Deletes a workspace.
Args:
workspace_name_or_id: Name or ID of the workspace to delete.
Raises:
KeyError: If no workspace with the given name exists.
"""
get_artifact(self, artifact_id)
Gets an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_id |
UUID |
The ID of the artifact to get. |
required |
Returns:
| Type | Description |
|---|---|
ArtifactResponseModel |
The artifact. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the artifact doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_artifact(self, artifact_id: UUID) -> ArtifactResponseModel:
"""Gets an artifact.
Args:
artifact_id: The ID of the artifact to get.
Returns:
The artifact.
Raises:
KeyError: if the artifact doesn't exist.
"""
get_auth_user(self, user_name_or_id)
Gets the auth model to a specific user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
required |
Returns:
| Type | Description |
|---|---|
UserAuthModel |
The requested user, if it was found. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_auth_user(
self, user_name_or_id: Union[str, UUID]
) -> UserAuthModel:
"""Gets the auth model to a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
"""
get_flavor(self, flavor_id)
Get a stack component flavor by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
UUID |
The ID of the flavor to get. |
required |
Returns:
| Type | Description |
|---|---|
FlavorResponseModel |
The stack component flavor. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component flavor doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_flavor(self, flavor_id: UUID) -> FlavorResponseModel:
"""Get a stack component flavor by ID.
Args:
flavor_id: The ID of the flavor to get.
Returns:
The stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
get_or_create_run(self, pipeline_run)
Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned. Otherwise, a new run is created.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_run |
PipelineRunRequestModel |
The pipeline run to get or create. |
required |
Returns:
| Type | Description |
|---|---|
Tuple[zenml.models.pipeline_run_models.PipelineRunResponseModel, bool] |
The pipeline run, and a boolean indicating whether the run was created or not. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_or_create_run(
self, pipeline_run: PipelineRunRequestModel
) -> Tuple[PipelineRunResponseModel, bool]:
"""Gets or creates a pipeline run.
If a run with the same ID or name already exists, it is returned.
Otherwise, a new run is created.
Args:
pipeline_run: The pipeline run to get or create.
Returns:
The pipeline run, and a boolean indicating whether the run was
created or not.
"""
get_pipeline(self, pipeline_id)
Get a pipeline with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
ID of the pipeline. |
required |
Returns:
| Type | Description |
|---|---|
PipelineResponseModel |
The pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_pipeline(self, pipeline_id: UUID) -> PipelineResponseModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
get_role(self, role_name_or_id)
Gets a specific role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to get. |
required |
Returns:
| Type | Description |
|---|---|
RoleResponseModel |
The requested role. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no role with the given name exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleResponseModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
Raises:
KeyError: If no role with the given name exists.
"""
get_run(self, run_name_or_id)
Gets a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the pipeline run to get. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The pipeline run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run(
self, run_name_or_id: Union[str, UUID]
) -> PipelineRunResponseModel:
"""Gets a pipeline run.
Args:
run_name_or_id: The name or ID of the pipeline run to get.
Returns:
The pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
get_run_step(self, step_run_id)
Get a step run by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_id |
UUID |
The ID of the step run to get. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The step run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the step run doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_step(self, step_run_id: UUID) -> StepRunResponseModel:
"""Get a step run by ID.
Args:
step_run_id: The ID of the step run to get.
Returns:
The step run.
Raises:
KeyError: if the step run doesn't exist.
"""
get_schedule(self, schedule_id)
Get a schedule with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
ID of the schedule. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The schedule. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the schedule does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_schedule(self, schedule_id: UUID) -> ScheduleResponseModel:
"""Get a schedule with a given ID.
Args:
schedule_id: ID of the schedule.
Returns:
The schedule.
Raises:
KeyError: if the schedule does not exist.
"""
get_stack(self, stack_id)
Get a stack by its unique ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack to get. |
required |
Returns:
| Type | Description |
|---|---|
StackResponseModel |
The stack with the given ID. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_stack(self, stack_id: UUID) -> StackResponseModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
Raises:
KeyError: if the stack doesn't exist.
"""
get_stack_component(self, component_id)
Get a stack component by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to get. |
required |
Returns:
| Type | Description |
|---|---|
ComponentResponseModel |
The stack component. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_stack_component(
self, component_id: UUID
) -> ComponentResponseModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
get_store_info(self)
Get information about the store.
Returns:
| Type | Description |
|---|---|
ServerModel |
Information about the store. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
get_team(self, team_name_or_id)
Gets a specific team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamResponseModel |
The requested team. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no team with the given name or ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamResponseModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
Raises:
KeyError: If no team with the given name or ID exists.
"""
get_team_role_assignment(self, team_role_assignment_id)
Gets a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_id |
UUID |
ID of the role assignment to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamRoleAssignmentResponseModel |
The requested role assignment. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no role assignment with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_team_role_assignment(
self, team_role_assignment_id: UUID
) -> TeamRoleAssignmentResponseModel:
"""Gets a specific role assignment.
Args:
team_role_assignment_id: ID of the role assignment to get.
Returns:
The requested role assignment.
Raises:
KeyError: If no role assignment with the given ID exists.
"""
get_user(self, user_name_or_id=None, include_private=False)
Gets a specific user, when no id is specified the active user is returned.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
None |
include_private |
bool |
Whether to include private user information |
False |
Returns:
| Type | Description |
|---|---|
UserResponseModel |
The requested user, if it was found. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given name or ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_user(
self,
user_name_or_id: Optional[Union[str, UUID]] = None,
include_private: bool = False,
) -> UserResponseModel:
"""Gets a specific user, when no id is specified the active user is returned.
Args:
user_name_or_id: The name or ID of the user to get.
include_private: Whether to include private user information
Returns:
The requested user, if it was found.
Raises:
KeyError: If no user with the given name or ID exists.
"""
get_user_role_assignment(self, user_role_assignment_id)
Gets a specific role assignment.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_id |
UUID |
ID of the role assignment to get. |
required |
Returns:
| Type | Description |
|---|---|
UserRoleAssignmentResponseModel |
The requested role assignment. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no role assignment with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_user_role_assignment(
self, user_role_assignment_id: UUID
) -> UserRoleAssignmentResponseModel:
"""Gets a specific role assignment.
Args:
user_role_assignment_id: ID of the role assignment to get.
Returns:
The requested role assignment.
Raises:
KeyError: If no role assignment with the given ID exists.
"""
get_workspace(self, workspace_name_or_id)
Get an existing workspace by name or ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_name_or_id |
Union[uuid.UUID, str] |
Name or ID of the workspace to get. |
required |
Returns:
| Type | Description |
|---|---|
WorkspaceResponseModel |
The requested workspace. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If there is no such workspace. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_workspace(
self, workspace_name_or_id: Union[UUID, str]
) -> WorkspaceResponseModel:
"""Get an existing workspace by name or ID.
Args:
workspace_name_or_id: Name or ID of the workspace to get.
Returns:
The requested workspace.
Raises:
KeyError: If there is no such workspace.
"""
list_artifacts(self, artifact_filter_model)
List all artifacts matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_filter_model |
ArtifactFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[ArtifactResponseModel] |
A list of all artifacts matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_artifacts(
self, artifact_filter_model: ArtifactFilterModel
) -> Page[ArtifactResponseModel]:
"""List all artifacts matching the given filter criteria.
Args:
artifact_filter_model: All filter parameters including pagination
params.
Returns:
A list of all artifacts matching the filter criteria.
"""
list_flavors(self, flavor_filter_model)
List all stack component flavors matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_filter_model |
FlavorFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[FlavorResponseModel] |
List of all the stack component flavors matching the given criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_flavors(
self, flavor_filter_model: FlavorFilterModel
) -> Page[FlavorResponseModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
flavor_filter_model: All filter parameters including pagination
params.
Returns:
List of all the stack component flavors matching the given criteria.
"""
list_pipelines(self, pipeline_filter_model)
List all pipelines matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_filter_model |
PipelineFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[PipelineResponseModel] |
A list of all pipelines matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_pipelines(
self, pipeline_filter_model: PipelineFilterModel
) -> Page[PipelineResponseModel]:
"""List all pipelines matching the given filter criteria.
Args:
pipeline_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipelines matching the filter criteria.
"""
list_roles(self, role_filter_model)
List all roles matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_filter_model |
RoleFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[RoleResponseModel] |
A list of all roles matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_roles(
self, role_filter_model: RoleFilterModel
) -> Page[RoleResponseModel]:
"""List all roles matching the given filter criteria.
Args:
role_filter_model: All filter parameters including pagination
params.
Returns:
A list of all roles matching the filter criteria.
"""
list_run_metadata(self, run_metadata_filter_model)
List run metadata.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_metadata_filter_model |
RunMetadataFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[RunMetadataResponseModel] |
The run metadata. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_run_metadata(
self,
run_metadata_filter_model: RunMetadataFilterModel,
) -> Page[RunMetadataResponseModel]:
"""List run metadata.
Args:
run_metadata_filter_model: All filter parameters including
pagination params.
Returns:
The run metadata.
"""
list_run_steps(self, step_run_filter_model)
List all step runs matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_filter_model |
StepRunFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[StepRunResponseModel] |
A list of all step runs matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_run_steps(
self, step_run_filter_model: StepRunFilterModel
) -> Page[StepRunResponseModel]:
"""List all step runs matching the given filter criteria.
Args:
step_run_filter_model: All filter parameters including pagination
params.
Returns:
A list of all step runs matching the filter criteria.
"""
list_runs(self, runs_filter_model)
List all pipeline runs matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
runs_filter_model |
PipelineRunFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[PipelineRunResponseModel] |
A list of all pipeline runs matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_runs(
self, runs_filter_model: PipelineRunFilterModel
) -> Page[PipelineRunResponseModel]:
"""List all pipeline runs matching the given filter criteria.
Args:
runs_filter_model: All filter parameters including pagination
params.
Returns:
A list of all pipeline runs matching the filter criteria.
"""
list_schedules(self, schedule_filter_model)
List all schedules in the workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_filter_model |
ScheduleFilterModel |
All filter parameters including pagination params |
required |
Returns:
| Type | Description |
|---|---|
Page[ScheduleResponseModel] |
A list of schedules. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_schedules(
self, schedule_filter_model: ScheduleFilterModel
) -> Page[ScheduleResponseModel]:
"""List all schedules in the workspace.
Args:
schedule_filter_model: All filter parameters including pagination
params
Returns:
A list of schedules.
"""
list_stack_components(self, component_filter_model)
List all stack components matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_filter_model |
ComponentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[ComponentResponseModel] |
A list of all stack components matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_stack_components(
self, component_filter_model: ComponentFilterModel
) -> Page[ComponentResponseModel]:
"""List all stack components matching the given filter criteria.
Args:
component_filter_model: All filter parameters including pagination
params.
Returns:
A list of all stack components matching the filter criteria.
"""
list_stacks(self, stack_filter_model)
List all stacks matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_filter_model |
StackFilterModel |
All filter parameters including pagination params |
required |
Returns:
| Type | Description |
|---|---|
Page[StackResponseModel] |
A list of all stacks matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_stacks(
self, stack_filter_model: StackFilterModel
) -> Page[StackResponseModel]:
"""List all stacks matching the given filter criteria.
Args:
stack_filter_model: All filter parameters including pagination
params
Returns:
A list of all stacks matching the filter criteria.
"""
list_team_role_assignments(self, team_role_assignment_filter_model)
List all roles assignments matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_role_assignment_filter_model |
TeamRoleAssignmentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[TeamRoleAssignmentResponseModel] |
A list of all roles assignments matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_team_role_assignments(
self, team_role_assignment_filter_model: TeamRoleAssignmentFilterModel
) -> Page[TeamRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
team_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
list_teams(self, team_filter_model)
List all teams matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_filter_model |
TeamFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[TeamResponseModel] |
A list of all teams matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_teams(
self, team_filter_model: TeamFilterModel
) -> Page[TeamResponseModel]:
"""List all teams matching the given filter criteria.
Args:
team_filter_model: All filter parameters including pagination
params.
Returns:
A list of all teams matching the filter criteria.
"""
list_user_role_assignments(self, user_role_assignment_filter_model)
List all roles assignments matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_role_assignment_filter_model |
UserRoleAssignmentFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[UserRoleAssignmentResponseModel] |
A list of all roles assignments matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_user_role_assignments(
self, user_role_assignment_filter_model: UserRoleAssignmentFilterModel
) -> Page[UserRoleAssignmentResponseModel]:
"""List all roles assignments matching the given filter criteria.
Args:
user_role_assignment_filter_model: All filter parameters including
pagination params.
Returns:
A list of all roles assignments matching the filter criteria.
"""
list_users(self, user_filter_model)
List all users.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_filter_model |
UserFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[UserResponseModel] |
A list of all users. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_users(
self, user_filter_model: UserFilterModel
) -> Page[UserResponseModel]:
"""List all users.
Args:
user_filter_model: All filter parameters including pagination
params.
Returns:
A list of all users.
"""
list_workspaces(self, workspace_filter_model)
List all workspace matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_filter_model |
WorkspaceFilterModel |
All filter parameters including pagination params. |
required |
Returns:
| Type | Description |
|---|---|
Page[WorkspaceResponseModel] |
A list of all workspace matching the filter criteria. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_workspaces(
self, workspace_filter_model: WorkspaceFilterModel
) -> Page[WorkspaceResponseModel]:
"""List all workspace matching the given filter criteria.
Args:
workspace_filter_model: All filter parameters including pagination
params.
Returns:
A list of all workspace matching the filter criteria.
"""
update_pipeline(self, pipeline_id, pipeline_update)
Updates a pipeline.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
The ID of the pipeline to be updated. |
required |
pipeline_update |
PipelineUpdateModel |
The update to be applied. |
required |
Returns:
| Type | Description |
|---|---|
PipelineResponseModel |
The updated pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_pipeline(
self,
pipeline_id: UUID,
pipeline_update: PipelineUpdateModel,
) -> PipelineResponseModel:
"""Updates a pipeline.
Args:
pipeline_id: The ID of the pipeline to be updated.
pipeline_update: The update to be applied.
Returns:
The updated pipeline.
Raises:
KeyError: if the pipeline doesn't exist.
"""
update_role(self, role_id, role_update)
Update an existing role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_id |
UUID |
The ID of the role to be updated. |
required |
role_update |
RoleUpdateModel |
The update to be applied to the role. |
required |
Returns:
| Type | Description |
|---|---|
RoleResponseModel |
The updated role. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the role does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_role(
self, role_id: UUID, role_update: RoleUpdateModel
) -> RoleResponseModel:
"""Update an existing role.
Args:
role_id: The ID of the role to be updated.
role_update: The update to be applied to the role.
Returns:
The updated role.
Raises:
KeyError: if the role does not exist.
"""
update_run(self, run_id, run_update)
Updates a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to update. |
required |
run_update |
PipelineRunUpdateModel |
The update to be applied to the pipeline run. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunResponseModel |
The updated pipeline run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_run(
self, run_id: UUID, run_update: PipelineRunUpdateModel
) -> PipelineRunResponseModel:
"""Updates a pipeline run.
Args:
run_id: The ID of the pipeline run to update.
run_update: The update to be applied to the pipeline run.
Returns:
The updated pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
update_run_step(self, step_run_id, step_run_update)
Updates a step run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_run_id |
UUID |
The ID of the step to update. |
required |
step_run_update |
StepRunUpdateModel |
The update to be applied to the step. |
required |
Returns:
| Type | Description |
|---|---|
StepRunResponseModel |
The updated step run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the step run doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_run_step(
self,
step_run_id: UUID,
step_run_update: StepRunUpdateModel,
) -> StepRunResponseModel:
"""Updates a step run.
Args:
step_run_id: The ID of the step to update.
step_run_update: The update to be applied to the step.
Returns:
The updated step run.
Raises:
KeyError: if the step run doesn't exist.
"""
update_schedule(self, schedule_id, schedule_update)
Updates a schedule.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
schedule_id |
UUID |
The ID of the schedule to be updated. |
required |
schedule_update |
ScheduleUpdateModel |
The update to be applied. |
required |
Returns:
| Type | Description |
|---|---|
ScheduleResponseModel |
The updated schedule. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the schedule doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_schedule(
self,
schedule_id: UUID,
schedule_update: ScheduleUpdateModel,
) -> ScheduleResponseModel:
"""Updates a schedule.
Args:
schedule_id: The ID of the schedule to be updated.
schedule_update: The update to be applied.
Returns:
The updated schedule.
Raises:
KeyError: if the schedule doesn't exist.
"""
update_stack(self, stack_id, stack_update)
Update a stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack update. |
required |
stack_update |
StackUpdateModel |
The update request on the stack. |
required |
Returns:
| Type | Description |
|---|---|
StackResponseModel |
The updated stack. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_stack(
self, stack_id: UUID, stack_update: StackUpdateModel
) -> StackResponseModel:
"""Update a stack.
Args:
stack_id: The ID of the stack update.
stack_update: The update request on the stack.
Returns:
The updated stack.
Raises:
KeyError: if the stack doesn't exist.
"""
update_stack_component(self, component_id, component_update)
Update an existing stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to update. |
required |
component_update |
ComponentUpdateModel |
The update to be applied to the stack component. |
required |
Returns:
| Type | Description |
|---|---|
ComponentResponseModel |
The updated stack component. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_stack_component(
self,
component_id: UUID,
component_update: ComponentUpdateModel,
) -> ComponentResponseModel:
"""Update an existing stack component.
Args:
component_id: The ID of the stack component to update.
component_update: The update to be applied to the stack component.
Returns:
The updated stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
update_team(self, team_id, team_update)
Update an existing team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_id |
UUID |
The ID of the team to be updated. |
required |
team_update |
TeamUpdateModel |
The update to be applied to the team. |
required |
Returns:
| Type | Description |
|---|---|
TeamResponseModel |
The updated team. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the team does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_team(
self, team_id: UUID, team_update: TeamUpdateModel
) -> TeamResponseModel:
"""Update an existing team.
Args:
team_id: The ID of the team to be updated.
team_update: The update to be applied to the team.
Returns:
The updated team.
Raises:
KeyError: if the team does not exist.
"""
update_user(self, user_id, user_update)
Updates an existing user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_id |
UUID |
The id of the user to update. |
required |
user_update |
UserUpdateModel |
The update to be applied to the user. |
required |
Returns:
| Type | Description |
|---|---|
UserResponseModel |
The updated user. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given name exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_user(
self, user_id: UUID, user_update: UserUpdateModel
) -> UserResponseModel:
"""Updates an existing user.
Args:
user_id: The id of the user to update.
user_update: The update to be applied to the user.
Returns:
The updated user.
Raises:
KeyError: If no user with the given name exists.
"""
update_workspace(self, workspace_id, workspace_update)
Update an existing workspace.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
workspace_id |
UUID |
The ID of the workspace to be updated. |
required |
workspace_update |
WorkspaceUpdateModel |
The update to be applied to the workspace. |
required |
Returns:
| Type | Description |
|---|---|
WorkspaceResponseModel |
The updated workspace. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the workspace does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_workspace(
self, workspace_id: UUID, workspace_update: WorkspaceUpdateModel
) -> WorkspaceResponseModel:
"""Update an existing workspace.
Args:
workspace_id: The ID of the workspace to be updated.
workspace_update: The update to be applied to the workspace.
Returns:
The updated workspace.
Raises:
KeyError: if the workspace does not exist.
"""