Zen Stores
zenml.zen_stores
special
ZenStores define ways to store ZenML relevant data locally or remotely.
base_zen_store
Base Zen Store implementation.
BaseZenStore (BaseModel, ZenStoreInterface, AnalyticsTrackerMixin)
pydantic-model
Base class for accessing and persisting ZenML core objects.
Attributes:
| Name | Type | Description |
|---|---|---|
config |
StoreConfiguration |
The configuration of the store. |
track_analytics |
bool |
Only send analytics if set to |
Source code in zenml/zen_stores/base_zen_store.py
class BaseZenStore(BaseModel, ZenStoreInterface, AnalyticsTrackerMixin):
"""Base class for accessing and persisting ZenML core objects.
Attributes:
config: The configuration of the store.
track_analytics: Only send analytics if set to `True`.
"""
config: StoreConfiguration
track_analytics: bool = True
_active_user: Optional[UserModel] = None
TYPE: ClassVar[StoreType]
CONFIG_TYPE: ClassVar[Type[StoreConfiguration]]
# ---------------------------------
# Initialization and configuration
# ---------------------------------
def __init__(
self,
skip_default_registrations: bool = False,
**kwargs: Any,
) -> None:
"""Create and initialize a store.
Args:
skip_default_registrations: If `True`, the creation of the default
stack and user in the store will be skipped.
**kwargs: Additional keyword arguments to pass to the Pydantic
constructor.
Raises:
RuntimeError: If the store cannot be initialized.
"""
super().__init__(**kwargs)
try:
self._initialize()
except Exception as e:
raise RuntimeError(
f"Error initializing {self.type.value} store with URL "
f"'{self.url}': {str(e)}"
) from e
if not skip_default_registrations:
logger.debug("Initializing database")
self._initialize_database()
else:
logger.debug("Skipping database initialization")
@staticmethod
def get_store_class(store_type: StoreType) -> Type["BaseZenStore"]:
"""Returns the class of the given store type.
Args:
store_type: The type of the store to get the class for.
Returns:
The class of the given store type or None if the type is unknown.
Raises:
TypeError: If the store type is unsupported.
"""
if store_type == StoreType.SQL:
from zenml.zen_stores.sql_zen_store import SqlZenStore
return SqlZenStore
elif store_type == StoreType.REST:
from zenml.zen_stores.rest_zen_store import RestZenStore
return RestZenStore
else:
raise TypeError(
f"No store implementation found for store type "
f"`{store_type.value}`."
)
@staticmethod
def create_store(
config: StoreConfiguration,
skip_default_registrations: bool = False,
**kwargs: Any,
) -> "BaseZenStore":
"""Create and initialize a store from a store configuration.
Args:
config: The store configuration to use.
skip_default_registrations: If `True`, the creation of the default
stack and user in the store will be skipped.
**kwargs: Additional keyword arguments to pass to the store class
Returns:
The initialized store.
"""
logger.debug(f"Creating store with config '{config}'...")
store_class = BaseZenStore.get_store_class(config.type)
store = store_class(
config=config,
skip_default_registrations=skip_default_registrations,
)
return store
@staticmethod
def get_default_store_config(path: str) -> StoreConfiguration:
"""Get the default store configuration.
The default store is a SQLite store that saves the DB contents on the
local filesystem.
Args:
path: The local path where the store DB will be stored.
Returns:
The default store configuration.
"""
from zenml.zen_stores.sql_zen_store import (
SqlZenStore,
SqlZenStoreConfiguration,
)
config = SqlZenStoreConfiguration(
type=StoreType.SQL, url=SqlZenStore.get_local_url(path)
)
return config
def _initialize_database(self) -> None:
"""Initialize the database on first use."""
try:
default_project = self._default_project
except KeyError:
default_project = self._create_default_project()
try:
default_user = self._default_user
except KeyError:
default_user = self._create_default_user()
try:
self._get_default_stack(
project_name_or_id=default_project.id,
user_name_or_id=default_user.id,
)
except KeyError:
self._create_default_stack(
project_name_or_id=default_project.id,
user_name_or_id=default_user.id,
)
@property
def url(self) -> str:
"""The URL of the store.
Returns:
The URL of the store.
"""
return self.config.url
@property
def type(self) -> StoreType:
"""The type of the store.
Returns:
The type of the store.
"""
return self.TYPE
def validate_active_config(
self,
active_project_name_or_id: Optional[Union[str, UUID]] = None,
active_stack_id: Optional[UUID] = None,
config_name: str = "",
) -> Tuple[ProjectModel, StackModel]:
"""Validate the active configuration.
Call this method to validate the supplied active project and active
stack values.
This method is guaranteed to return valid project ID and stack ID
values. If the supplied project and stack are not set or are not valid
(e.g. they do not exist or are not accessible), the default project and
default project stack will be returned in their stead.
Args:
active_project_name_or_id: The name or ID of the active project.
active_stack_id: The ID of the active stack.
config_name: The name of the configuration to validate (used in the
displayed logs/messages).
Returns:
A tuple containing the active project and active stack.
"""
active_project: ProjectModel
# Ensure that the current active project is still valid
if active_project_name_or_id:
try:
active_project = self.get_project(active_project_name_or_id)
except KeyError:
logger.warning(
"The current %s active project is no longer available. "
"Resetting the active project to default.",
config_name,
)
active_project = self._default_project
else:
logger.info(
"Setting the %s active project to default.",
config_name,
)
active_project = self._default_project
active_stack: StackModel
# Create a default stack in the active project for the active user if
# one is not yet created.
try:
default_stack = self._get_default_stack(
project_name_or_id=active_project.id,
user_name_or_id=self.active_user.id,
)
except KeyError:
default_stack = self._create_default_stack(
project_name_or_id=active_project.id,
user_name_or_id=self.active_user.id,
)
# Sanitize the active stack
if active_stack_id:
# Ensure that the active stack is still valid
try:
active_stack = self.get_stack(stack_id=active_stack_id)
except KeyError:
logger.warning(
"The current %s active stack is no longer available. "
"Resetting the active stack to default.",
config_name,
)
active_stack = default_stack
else:
if active_stack.project != active_project.id:
logger.warning(
"The current %s active stack is not part of the active "
"project. Resetting the active stack to default.",
config_name,
)
active_stack = default_stack
elif (
not active_stack.is_shared
and active_stack.user != self.active_user.id
):
logger.warning(
"The current %s active stack is not shared and not "
"owned by the active user. "
"Resetting the active stack to default.",
config_name,
)
active_stack = default_stack
else:
logger.warning(
"Setting the %s active stack to default.",
config_name,
)
active_stack = default_stack
return active_project, active_stack
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
return ServerModel(
id=GlobalConfiguration().user_id,
version=zenml.__version__,
deployment_type=os.environ.get(
ENV_ZENML_SERVER_DEPLOYMENT_TYPE, ServerDeploymentType.OTHER
),
database_type=ServerDatabaseType.OTHER,
)
def is_local_store(self) -> bool:
"""Check if the store is a local store or connected to a locally deployed ZenML server.
Returns:
True if the store is local, False otherwise.
"""
return self.get_store_info().is_local()
# ------
# Stacks
# ------
@track(AnalyticsEvent.REGISTERED_DEFAULT_STACK)
def _create_default_stack(
self,
project_name_or_id: Union[str, UUID],
user_name_or_id: Union[str, UUID],
) -> StackModel:
"""Create the default stack components and stack.
The default stack contains a local orchestrator and a local artifact
store.
Args:
project_name_or_id: Name or ID of the project to which the stack
belongs.
user_name_or_id: The name or ID of the user that owns the stack.
Returns:
The model of the created default stack.
Raises:
StackExistsError: If a default stack already exists for the
user in the supplied project.
"""
project = self.get_project(project_name_or_id=project_name_or_id)
user = self.get_user(user_name_or_id=user_name_or_id)
try:
self._get_default_stack(
project_name_or_id=project_name_or_id,
user_name_or_id=user_name_or_id,
)
except KeyError:
pass
else:
raise StackExistsError(
f"Default stack already exists for user "
f"{user.name} in project {project.name}"
)
logger.info(
f"Creating default stack for user {user.name} in project "
f"{project.name}..."
)
# Register the default orchestrator
orchestrator = self.create_stack_component(
component=ComponentModel(
user=user.id,
project=project.id,
name="default",
type=StackComponentType.ORCHESTRATOR,
flavor="local",
configuration={},
),
)
# Register the default artifact store
artifact_store = self.create_stack_component(
component=ComponentModel(
user=user.id,
project=project.id,
name="default",
type=StackComponentType.ARTIFACT_STORE,
flavor="local",
configuration={},
),
)
components = {c.type: [c.id] for c in [orchestrator, artifact_store]}
# Register the default stack
stack = StackModel(
name="default",
components=components,
is_shared=False,
project=project.id,
user=user.id,
)
return self.create_stack(stack=stack)
def _get_default_stack(
self,
project_name_or_id: Union[str, UUID],
user_name_or_id: Union[str, UUID],
) -> StackModel:
"""Get the default stack for a user in a project.
Args:
project_name_or_id: Name or ID of the project.
user_name_or_id: Name or ID of the user.
Returns:
The default stack in the project owned by the supplied user.
Raises:
KeyError: if the project or default stack doesn't exist.
"""
default_stacks = self.list_stacks(
project_name_or_id=project_name_or_id,
user_name_or_id=user_name_or_id,
name=DEFAULT_STACK_NAME,
)
if len(default_stacks) == 0:
raise KeyError(
f"No default stack found for user {str(user_name_or_id)} in "
f"project {str(project_name_or_id)}"
)
return default_stacks[0]
# -----
# Users
# -----
@property
def active_user(self) -> UserModel:
"""The active user.
Returns:
The active user.
"""
if self._active_user is None:
self._active_user = self.get_user(self.active_user_name)
return self._active_user
@property
def users(self) -> List[UserModel]:
"""All existing users.
Returns:
A list of all existing users.
"""
return self.list_users()
@property
def _default_user_name(self) -> str:
"""Get the default user name.
Returns:
The default user name.
"""
return os.getenv(ENV_ZENML_DEFAULT_USER_NAME, DEFAULT_USERNAME)
@property
def _default_user(self) -> UserModel:
"""Get the default user.
Returns:
The default user.
Raises:
KeyError: If the default user doesn't exist.
"""
user_name = self._default_user_name
try:
return self.get_user(user_name)
except KeyError:
raise KeyError(f"The default user '{user_name}' is not configured")
@track(AnalyticsEvent.CREATED_DEFAULT_USER)
def _create_default_user(self) -> UserModel:
"""Creates a default user.
Returns:
The default user.
"""
user_name = os.getenv(ENV_ZENML_DEFAULT_USER_NAME, DEFAULT_USERNAME)
user_password = os.getenv(
ENV_ZENML_DEFAULT_USER_PASSWORD, DEFAULT_PASSWORD
)
logger.info(f"Creating default user '{user_name}' ...")
return self.create_user(
UserModel(
name=user_name,
active=True,
password=user_password,
)
)
# -----
# Teams
# -----
@property
def teams(self) -> List[TeamModel]:
"""List all teams.
Returns:
A list of all teams.
"""
return self.list_teams()
# -----
# Roles
# -----
@property
def roles(self) -> List[RoleModel]:
"""All existing roles.
Returns:
A list of all existing roles.
"""
return self.list_roles()
@property
def role_assignments(self) -> List[RoleAssignmentModel]:
"""All role assignments.
Returns:
A list of all role assignments.
"""
return self.list_role_assignments(user_name_or_id=self.active_user_name)
# --------
# Projects
# --------
@property
def _default_project(self) -> ProjectModel:
"""Get the default project.
Returns:
The default project.
Raises:
KeyError: if the default project doesn't exist.
"""
project_name = os.getenv(
ENV_ZENML_DEFAULT_PROJECT_NAME, DEFAULT_PROJECT_NAME
)
try:
return self.get_project(project_name)
except KeyError:
raise KeyError(
f"The default project '{project_name}' is not configured"
)
@track(AnalyticsEvent.CREATED_DEFAULT_PROJECT)
def _create_default_project(self) -> ProjectModel:
"""Creates a default project.
Returns:
The default project.
"""
project_name = os.getenv(
ENV_ZENML_DEFAULT_PROJECT_NAME, DEFAULT_PROJECT_NAME
)
logger.info(f"Creating default project '{project_name}' ...")
return self.create_project(ProjectModel(name=project_name))
# ------------
# Repositories
# ------------
# ---------
# Pipelines
# ---------
def get_pipeline_in_project(
self, pipeline_name: str, project_name_or_id: Union[str, UUID]
) -> PipelineModel:
"""Get a pipeline with a given name in a project.
Args:
pipeline_name: Name of the pipeline.
project_name_or_id: ID of the project.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
pipelines = self.list_pipelines(
project_name_or_id=project_name_or_id, name=pipeline_name
)
if len(pipelines) == 0:
raise KeyError(
f"No pipeline found with name {pipeline_name} in project "
f"{project_name_or_id}"
)
return pipelines[0]
# -------------
# Pipeline runs
# -------------
# ------------------
# Pipeline run steps
# ------------------
# ---------
# Analytics
# ---------
def track_event(
self,
event: Union[str, AnalyticsEvent],
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Track an analytics event.
Args:
event: The event to track.
metadata: Additional metadata to track with the event.
"""
if self.track_analytics:
# Server information is always tracked, if available.
track_event(event, metadata)
class Config:
"""Pydantic configuration class."""
# Validate attributes when assigning them. We need to set this in order
# to have a mix of mutable and immutable attributes
validate_assignment = True
# Ignore extra attributes from configs of previous ZenML versions
extra = "ignore"
# all attributes with leading underscore are private and therefore
# are mutable and not included in serialization
underscore_attrs_are_private = True
active_user: UserModel
property
readonly
The active user.
Returns:
| Type | Description |
|---|---|
UserModel |
The active user. |
role_assignments: List[zenml.models.user_management_models.RoleAssignmentModel]
property
readonly
All role assignments.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.RoleAssignmentModel] |
A list of all role assignments. |
roles: List[zenml.models.user_management_models.RoleModel]
property
readonly
All existing roles.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.RoleModel] |
A list of all existing roles. |
teams: List[zenml.models.user_management_models.TeamModel]
property
readonly
List all teams.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.TeamModel] |
A list of all teams. |
type: StoreType
property
readonly
The type of the store.
Returns:
| Type | Description |
|---|---|
StoreType |
The type of the store. |
url: str
property
readonly
The URL of the store.
Returns:
| Type | Description |
|---|---|
str |
The URL of the store. |
users: List[zenml.models.user_management_models.UserModel]
property
readonly
All existing users.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.UserModel] |
A list of all existing users. |
Config
Pydantic configuration class.
Source code in zenml/zen_stores/base_zen_store.py
class Config:
"""Pydantic configuration class."""
# Validate attributes when assigning them. We need to set this in order
# to have a mix of mutable and immutable attributes
validate_assignment = True
# Ignore extra attributes from configs of previous ZenML versions
extra = "ignore"
# all attributes with leading underscore are private and therefore
# are mutable and not included in serialization
underscore_attrs_are_private = True
__init__(self, skip_default_registrations=False, **kwargs)
special
Create and initialize a store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
skip_default_registrations |
bool |
If |
False |
**kwargs |
Any |
Additional keyword arguments to pass to the Pydantic constructor. |
{} |
Exceptions:
| Type | Description |
|---|---|
RuntimeError |
If the store cannot be initialized. |
Source code in zenml/zen_stores/base_zen_store.py
def __init__(
self,
skip_default_registrations: bool = False,
**kwargs: Any,
) -> None:
"""Create and initialize a store.
Args:
skip_default_registrations: If `True`, the creation of the default
stack and user in the store will be skipped.
**kwargs: Additional keyword arguments to pass to the Pydantic
constructor.
Raises:
RuntimeError: If the store cannot be initialized.
"""
super().__init__(**kwargs)
try:
self._initialize()
except Exception as e:
raise RuntimeError(
f"Error initializing {self.type.value} store with URL "
f"'{self.url}': {str(e)}"
) from e
if not skip_default_registrations:
logger.debug("Initializing database")
self._initialize_database()
else:
logger.debug("Skipping database initialization")
create_store(config, skip_default_registrations=False, **kwargs)
staticmethod
Create and initialize a store from a store configuration.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The store configuration to use. |
required |
skip_default_registrations |
bool |
If |
False |
**kwargs |
Any |
Additional keyword arguments to pass to the store class |
{} |
Returns:
| Type | Description |
|---|---|
BaseZenStore |
The initialized store. |
Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def create_store(
config: StoreConfiguration,
skip_default_registrations: bool = False,
**kwargs: Any,
) -> "BaseZenStore":
"""Create and initialize a store from a store configuration.
Args:
config: The store configuration to use.
skip_default_registrations: If `True`, the creation of the default
stack and user in the store will be skipped.
**kwargs: Additional keyword arguments to pass to the store class
Returns:
The initialized store.
"""
logger.debug(f"Creating store with config '{config}'...")
store_class = BaseZenStore.get_store_class(config.type)
store = store_class(
config=config,
skip_default_registrations=skip_default_registrations,
)
return store
get_default_store_config(path)
staticmethod
Get the default store configuration.
The default store is a SQLite store that saves the DB contents on the local filesystem.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The local path where the store DB will be stored. |
required |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
The default store configuration. |
Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_default_store_config(path: str) -> StoreConfiguration:
"""Get the default store configuration.
The default store is a SQLite store that saves the DB contents on the
local filesystem.
Args:
path: The local path where the store DB will be stored.
Returns:
The default store configuration.
"""
from zenml.zen_stores.sql_zen_store import (
SqlZenStore,
SqlZenStoreConfiguration,
)
config = SqlZenStoreConfiguration(
type=StoreType.SQL, url=SqlZenStore.get_local_url(path)
)
return config
get_pipeline_in_project(self, pipeline_name, project_name_or_id)
Get a pipeline with a given name in a project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_name |
str |
Name of the pipeline. |
required |
project_name_or_id |
Union[str, uuid.UUID] |
ID of the project. |
required |
Returns:
| Type | Description |
|---|---|
PipelineModel |
The pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline does not exist. |
Source code in zenml/zen_stores/base_zen_store.py
def get_pipeline_in_project(
self, pipeline_name: str, project_name_or_id: Union[str, UUID]
) -> PipelineModel:
"""Get a pipeline with a given name in a project.
Args:
pipeline_name: Name of the pipeline.
project_name_or_id: ID of the project.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
pipelines = self.list_pipelines(
project_name_or_id=project_name_or_id, name=pipeline_name
)
if len(pipelines) == 0:
raise KeyError(
f"No pipeline found with name {pipeline_name} in project "
f"{project_name_or_id}"
)
return pipelines[0]
get_store_class(store_type)
staticmethod
Returns the class of the given store type.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
store_type |
StoreType |
The type of the store to get the class for. |
required |
Returns:
| Type | Description |
|---|---|
Type[BaseZenStore] |
The class of the given store type or None if the type is unknown. |
Exceptions:
| Type | Description |
|---|---|
TypeError |
If the store type is unsupported. |
Source code in zenml/zen_stores/base_zen_store.py
@staticmethod
def get_store_class(store_type: StoreType) -> Type["BaseZenStore"]:
"""Returns the class of the given store type.
Args:
store_type: The type of the store to get the class for.
Returns:
The class of the given store type or None if the type is unknown.
Raises:
TypeError: If the store type is unsupported.
"""
if store_type == StoreType.SQL:
from zenml.zen_stores.sql_zen_store import SqlZenStore
return SqlZenStore
elif store_type == StoreType.REST:
from zenml.zen_stores.rest_zen_store import RestZenStore
return RestZenStore
else:
raise TypeError(
f"No store implementation found for store type "
f"`{store_type.value}`."
)
get_store_info(self)
Get information about the store.
Returns:
| Type | Description |
|---|---|
ServerModel |
Information about the store. |
Source code in zenml/zen_stores/base_zen_store.py
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
return ServerModel(
id=GlobalConfiguration().user_id,
version=zenml.__version__,
deployment_type=os.environ.get(
ENV_ZENML_SERVER_DEPLOYMENT_TYPE, ServerDeploymentType.OTHER
),
database_type=ServerDatabaseType.OTHER,
)
is_local_store(self)
Check if the store is a local store or connected to a locally deployed ZenML server.
Returns:
| Type | Description |
|---|---|
bool |
True if the store is local, False otherwise. |
Source code in zenml/zen_stores/base_zen_store.py
def is_local_store(self) -> bool:
"""Check if the store is a local store or connected to a locally deployed ZenML server.
Returns:
True if the store is local, False otherwise.
"""
return self.get_store_info().is_local()
track_event(self, event, metadata=None)
Track an analytics event.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
event |
Union[str, zenml.utils.analytics_utils.AnalyticsEvent] |
The event to track. |
required |
metadata |
Optional[Dict[str, Any]] |
Additional metadata to track with the event. |
None |
Source code in zenml/zen_stores/base_zen_store.py
def track_event(
self,
event: Union[str, AnalyticsEvent],
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Track an analytics event.
Args:
event: The event to track.
metadata: Additional metadata to track with the event.
"""
if self.track_analytics:
# Server information is always tracked, if available.
track_event(event, metadata)
validate_active_config(self, active_project_name_or_id=None, active_stack_id=None, config_name='')
Validate the active configuration.
Call this method to validate the supplied active project and active stack values.
This method is guaranteed to return valid project ID and stack ID values. If the supplied project and stack are not set or are not valid (e.g. they do not exist or are not accessible), the default project and default project stack will be returned in their stead.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
active_project_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the active project. |
None |
active_stack_id |
Optional[uuid.UUID] |
The ID of the active stack. |
None |
config_name |
str |
The name of the configuration to validate (used in the displayed logs/messages). |
'' |
Returns:
| Type | Description |
|---|---|
Tuple[zenml.models.project_models.ProjectModel, zenml.models.stack_models.StackModel] |
A tuple containing the active project and active stack. |
Source code in zenml/zen_stores/base_zen_store.py
def validate_active_config(
self,
active_project_name_or_id: Optional[Union[str, UUID]] = None,
active_stack_id: Optional[UUID] = None,
config_name: str = "",
) -> Tuple[ProjectModel, StackModel]:
"""Validate the active configuration.
Call this method to validate the supplied active project and active
stack values.
This method is guaranteed to return valid project ID and stack ID
values. If the supplied project and stack are not set or are not valid
(e.g. they do not exist or are not accessible), the default project and
default project stack will be returned in their stead.
Args:
active_project_name_or_id: The name or ID of the active project.
active_stack_id: The ID of the active stack.
config_name: The name of the configuration to validate (used in the
displayed logs/messages).
Returns:
A tuple containing the active project and active stack.
"""
active_project: ProjectModel
# Ensure that the current active project is still valid
if active_project_name_or_id:
try:
active_project = self.get_project(active_project_name_or_id)
except KeyError:
logger.warning(
"The current %s active project is no longer available. "
"Resetting the active project to default.",
config_name,
)
active_project = self._default_project
else:
logger.info(
"Setting the %s active project to default.",
config_name,
)
active_project = self._default_project
active_stack: StackModel
# Create a default stack in the active project for the active user if
# one is not yet created.
try:
default_stack = self._get_default_stack(
project_name_or_id=active_project.id,
user_name_or_id=self.active_user.id,
)
except KeyError:
default_stack = self._create_default_stack(
project_name_or_id=active_project.id,
user_name_or_id=self.active_user.id,
)
# Sanitize the active stack
if active_stack_id:
# Ensure that the active stack is still valid
try:
active_stack = self.get_stack(stack_id=active_stack_id)
except KeyError:
logger.warning(
"The current %s active stack is no longer available. "
"Resetting the active stack to default.",
config_name,
)
active_stack = default_stack
else:
if active_stack.project != active_project.id:
logger.warning(
"The current %s active stack is not part of the active "
"project. Resetting the active stack to default.",
config_name,
)
active_stack = default_stack
elif (
not active_stack.is_shared
and active_stack.user != self.active_user.id
):
logger.warning(
"The current %s active stack is not shared and not "
"owned by the active user. "
"Resetting the active stack to default.",
config_name,
)
active_stack = default_stack
else:
logger.warning(
"Setting the %s active stack to default.",
config_name,
)
active_stack = default_stack
return active_project, active_stack
metadata_store
Base implementation of a metadata store.
MLMDArtifactModel (BaseModel)
pydantic-model
Class that models an artifact response from the metadata store.
Source code in zenml/zen_stores/metadata_store.py
class MLMDArtifactModel(BaseModel):
"""Class that models an artifact response from the metadata store."""
mlmd_id: int
type: ArtifactType
uri: str
materializer: str
data_type: str
mlmd_parent_step_id: int
mlmd_producer_step_id: int
is_cached: bool
MLMDPipelineRunModel (BaseModel)
pydantic-model
Class that models a pipeline run response from the metadata store.
Source code in zenml/zen_stores/metadata_store.py
class MLMDPipelineRunModel(BaseModel):
"""Class that models a pipeline run response from the metadata store."""
mlmd_id: int
name: str
project: UUID
user: UUID
pipeline_id: Optional[UUID]
stack_id: UUID
pipeline_configuration: Dict[str, Any]
num_steps: int
MLMDStepRunModel (BaseModel)
pydantic-model
Class that models a step run response from the metadata store.
Source code in zenml/zen_stores/metadata_store.py
class MLMDStepRunModel(BaseModel):
"""Class that models a step run response from the metadata store."""
mlmd_id: int
mlmd_parent_step_ids: List[int]
entrypoint_name: str
name: str
parameters: Dict[str, str]
step_configuration: Dict[str, Any]
MetadataStore
ZenML MLMD metadata store.
Source code in zenml/zen_stores/metadata_store.py
class MetadataStore:
"""ZenML MLMD metadata store."""
upgrade_migration_enabled: bool = True
store: metadata_store.MetadataStore
def __init__(self, config: metadata_store_pb2.ConnectionConfig) -> None:
"""Initializes the metadata store.
Args:
config: The connection configuration for the metadata store.
"""
self.store = metadata_store.MetadataStore(
config, enable_upgrade_migration=True
)
@property
def step_type_mapping(self) -> Dict[int, str]:
"""Maps type_ids to step names.
Returns:
Dict[int, str]: a mapping from type_ids to step names.
"""
return {
type_.id: type_.name for type_ in self.store.get_execution_types()
}
def _check_if_executions_belong_to_pipeline(
self,
executions: List[proto.Execution],
pipeline_id: int,
) -> bool:
"""Returns `True` if the executions are associated with the pipeline context.
Args:
executions: List of executions.
pipeline_id: The ID of the pipeline to check.
Returns:
`True` if the executions are associated with the pipeline context.
"""
for execution in executions:
associated_contexts = self.store.get_contexts_by_execution(
execution.id
)
for context in associated_contexts:
if context.id == pipeline_id: # noqa
return True
return False
def _get_zenml_execution_context_properties(
self, execution: proto.Execution
) -> Any:
associated_contexts = self.store.get_contexts_by_execution(execution.id)
for context in associated_contexts:
context_type = self.store.get_context_types_by_id(
[context.type_id]
)[0].name
if context_type == ZENML_CONTEXT_TYPE_NAME:
return context.custom_properties
raise RuntimeError(
f"Could not find 'zenml' context for execution {execution.name}."
)
def _get_step_model_from_execution(
self, execution: proto.Execution
) -> MLMDStepRunModel:
"""Get the original step from an execution.
Args:
execution: proto.Execution object from mlmd store.
Returns:
Model of the original step derived from the proto.Execution.
Raises:
KeyError: If the execution is not associated with a step.
"""
impl_name = self.step_type_mapping[execution.type_id].split(".")[-1]
step_name_property = execution.custom_properties.get(
INTERNAL_EXECUTION_PARAMETER_PREFIX + PARAM_PIPELINE_PARAMETER_NAME,
None,
)
if step_name_property:
step_name = json.loads(step_name_property.string_value)
else:
raise KeyError(
f"Step name missing for execution with ID {execution.id}. "
f"This error probably occurs because you're using ZenML "
f"version 0.5.4 or newer but your metadata store contains "
f"data from previous versions."
)
step_parameters = {}
for k, v in execution.custom_properties.items():
if not k.startswith(INTERNAL_EXECUTION_PARAMETER_PREFIX):
try:
json.loads(v.string_value)
step_parameters[k] = v.string_value
except JSONDecodeError:
# this means there is a property in there that is neither
# an internal one or one created by zenml. Therefore, we can
# ignore it
pass
step_context_properties = self._get_zenml_execution_context_properties(
execution=execution,
)
step_configuration = json.loads(
step_context_properties.get(
MLMD_CONTEXT_STEP_CONFIG_PROPERTY_NAME
).string_value
)
# TODO [ENG-222]: This is a lot of querying to the metadata store. We
# should refactor and make it nicer. Probably it makes more sense
# to first get `executions_ids_for_current_run` and then filter on
# `event.execution_id in execution_ids_for_current_run`.
# Core logic here is that we get the event of this particular execution
# id that gives us the artifacts of this execution. We then go through
# all `input` artifacts of this execution and get all events related to
# that artifact. This in turn gives us other events for which this
# artifact was an `output` artifact. Then we simply need to sort by
# time to get the most recent execution (i.e. step) that produced that
# particular artifact.
events_for_execution = self.store.get_events_by_execution_ids(
[execution.id]
)
parents_step_ids = set()
for current_event in events_for_execution:
if current_event.type == current_event.INPUT:
# this means the artifact is an input artifact
events_for_input_artifact = [
e
for e in self.store.get_events_by_artifact_ids(
[current_event.artifact_id]
)
# should be output type and should NOT be the same id as
# the execution we are querying and it should be BEFORE
# the time of the current event.
if e.type == e.OUTPUT
and e.execution_id != current_event.execution_id
and e.milliseconds_since_epoch
< current_event.milliseconds_since_epoch
]
# sort by time
events_for_input_artifact.sort(
key=lambda x: x.milliseconds_since_epoch # type: ignore[no-any-return] # noqa
)
# take the latest one and add execution to the parents.
parents_step_ids.add(events_for_input_artifact[-1].execution_id)
return MLMDStepRunModel(
mlmd_id=execution.id,
mlmd_parent_step_ids=list(parents_step_ids),
entrypoint_name=impl_name,
name=step_name,
parameters=step_parameters,
step_configuration=step_configuration,
)
def _get_pipeline_run_model_from_context(
self, context: proto.Context
) -> MLMDPipelineRunModel:
context_properties = self._get_zenml_execution_context_properties(
self.store.get_executions_by_context(context_id=context.id)[-1]
)
model_ids = json.loads(
context_properties.get(
MLMD_CONTEXT_MODEL_IDS_PROPERTY_NAME
).string_value
)
pipeline_configuration = json.loads(
context_properties.get(
MLMD_CONTEXT_PIPELINE_CONFIG_PROPERTY_NAME
).string_value
)
num_steps = int(
context_properties.get(
MLMD_CONTEXT_NUM_STEPS_PROPERTY_NAME
).string_value
)
return MLMDPipelineRunModel(
mlmd_id=context.id,
name=context.name,
project=model_ids["project_id"],
user=model_ids["user_id"],
pipeline_id=model_ids["pipeline_id"],
stack_id=model_ids["stack_id"],
pipeline_configuration=pipeline_configuration,
num_steps=num_steps,
)
def get_all_runs(self) -> Dict[str, MLMDPipelineRunModel]:
"""Gets a mapping run name -> ID for all runs registered in MLMD.
Returns:
A mapping run name -> ID for all runs registered in MLMD.
"""
all_pipeline_runs = self.store.get_contexts_by_type(
PIPELINE_RUN_CONTEXT_TYPE_NAME
)
return {
run.name: self._get_pipeline_run_model_from_context(run)
for run in all_pipeline_runs
}
def get_pipeline_run_steps(
self, run_id: int
) -> Dict[str, MLMDStepRunModel]:
"""Gets all steps for the given pipeline run.
Args:
run_id: The ID of the pipeline run to get the steps for.
Returns:
A dictionary of step names to step views.
"""
steps: Dict[str, MLMDStepRunModel] = OrderedDict()
# reverse the executions as they get returned in reverse chronological
# order from the metadata store
executions = self.store.get_executions_by_context(run_id)
for execution in reversed(executions): # noqa
step = self._get_step_model_from_execution(execution)
steps[step.name] = step
logger.debug(f"Fetched {len(steps)} steps for pipeline run '{run_id}'.")
return steps
def get_step_by_id(self, step_id: int) -> MLMDStepRunModel:
"""Gets a step by its ID.
Args:
step_id: The ID of the step to get.
Returns:
A model of the step with the given ID.
"""
execution = self.store.get_executions_by_id([step_id])[0]
return self._get_step_model_from_execution(execution)
def get_step_status(self, step_id: int) -> ExecutionStatus:
"""Gets the execution status of a single step.
Args:
step_id: The ID of the step to get the status for.
Returns:
ExecutionStatus: The status of the step.
"""
proto = self.store.get_executions_by_id([step_id])[0] # noqa
state = proto.last_known_state
if state == proto.COMPLETE:
return ExecutionStatus.COMPLETED
elif state == proto.RUNNING:
return ExecutionStatus.RUNNING
elif state == proto.CACHED:
return ExecutionStatus.CACHED
else:
return ExecutionStatus.FAILED
def get_step_artifacts(
self, step_id: int, step_parent_step_ids: List[int], step_name: str
) -> Tuple[Dict[str, MLMDArtifactModel], Dict[str, MLMDArtifactModel]]:
"""Returns input and output artifacts for the given step.
Args:
step_id: The ID of the step to get the artifacts for.
step_parent_step_ids: The IDs of the parent steps of the given step.
step_name: The name of the step.
Returns:
A tuple (inputs, outputs) where inputs and outputs are both Dicts mapping artifact names to the input and output artifacts respectively.
"""
# maps artifact types to their string representation
artifact_type_mapping = {
type_.id: type_.name for type_ in self.store.get_artifact_types()
}
events = self.store.get_events_by_execution_ids([step_id]) # noqa
artifacts = self.store.get_artifacts_by_id(
[event.artifact_id for event in events]
)
inputs: Dict[str, MLMDArtifactModel] = {}
outputs: Dict[str, MLMDArtifactModel] = {}
# sort them according to artifact_id's so that the zip works.
events.sort(key=lambda x: x.artifact_id)
artifacts.sort(key=lambda x: x.id)
for event_proto, artifact_proto in zip(events, artifacts):
artifact_type = artifact_type_mapping[artifact_proto.type_id]
artifact_name = event_proto.path.steps[0].key
materializer = artifact_proto.properties[
MATERIALIZER_PROPERTY_KEY
].string_value
data_type = artifact_proto.properties[
DATATYPE_PROPERTY_KEY
].string_value
parent_step_id = step_id
if event_proto.type == event_proto.INPUT:
# In the case that this is an input event, we actually need
# to resolve it via its parents outputs.
for parent_id in step_parent_step_ids:
parent_step = self.get_step_by_id(parent_id)
parent_outputs = self.get_step_artifacts(
step_id=parent_id,
step_parent_step_ids=parent_step.mlmd_parent_step_ids,
step_name=parent_step.entrypoint_name,
)[1]
for parent_output in parent_outputs.values():
if artifact_proto.id == parent_output.mlmd_id:
parent_step_id = parent_id
artifact_id = event_proto.artifact_id
producer_step = self.get_producer_step_from_artifact(artifact_id)
producer_step_id = producer_step.mlmd_id
artifact = MLMDArtifactModel(
mlmd_id=artifact_id,
type=artifact_type,
uri=artifact_proto.uri,
materializer=materializer,
data_type=data_type,
mlmd_parent_step_id=parent_step_id,
mlmd_producer_step_id=producer_step_id,
is_cached=parent_step_id != producer_step_id,
)
if event_proto.type == event_proto.INPUT:
inputs[artifact_name] = artifact
elif event_proto.type == event_proto.OUTPUT:
outputs[artifact_name] = artifact
logger.debug(
"Fetched %d inputs and %d outputs for step '%s'.",
len(inputs),
len(outputs),
step_name,
)
return inputs, outputs
def get_producer_step_from_artifact(
self, artifact_id: int
) -> MLMDStepRunModel:
"""Find the original step that created an artifact.
Args:
artifact_id: ID of the artifact for which to get the producer step.
Returns:
Original step that produced the artifact.
"""
executions_ids = set(
event.execution_id
for event in self.store.get_events_by_artifact_ids([artifact_id])
if event.type == event.OUTPUT
)
execution = self.store.get_executions_by_id(executions_ids)[0]
return self._get_step_model_from_execution(execution)
class Config:
"""Pydantic configuration class."""
# public attributes are immutable
allow_mutation = False
# all attributes with leading underscore are private and therefore
# are mutable and not included in serialization
underscore_attrs_are_private = True
# prevent extra attributes during model initialization
extra = Extra.forbid
step_type_mapping: Dict[int, str]
property
readonly
Maps type_ids to step names.
Returns:
| Type | Description |
|---|---|
Dict[int, str] |
a mapping from type_ids to step names. |
Config
Pydantic configuration class.
Source code in zenml/zen_stores/metadata_store.py
class Config:
"""Pydantic configuration class."""
# public attributes are immutable
allow_mutation = False
# all attributes with leading underscore are private and therefore
# are mutable and not included in serialization
underscore_attrs_are_private = True
# prevent extra attributes during model initialization
extra = Extra.forbid
__init__(self, config)
special
Initializes the metadata store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
ConnectionConfig |
The connection configuration for the metadata store. |
required |
Source code in zenml/zen_stores/metadata_store.py
def __init__(self, config: metadata_store_pb2.ConnectionConfig) -> None:
"""Initializes the metadata store.
Args:
config: The connection configuration for the metadata store.
"""
self.store = metadata_store.MetadataStore(
config, enable_upgrade_migration=True
)
get_all_runs(self)
Gets a mapping run name -> ID for all runs registered in MLMD.
Returns:
| Type | Description |
|---|---|
Dict[str, zenml.zen_stores.metadata_store.MLMDPipelineRunModel] |
A mapping run name -> ID for all runs registered in MLMD. |
Source code in zenml/zen_stores/metadata_store.py
def get_all_runs(self) -> Dict[str, MLMDPipelineRunModel]:
"""Gets a mapping run name -> ID for all runs registered in MLMD.
Returns:
A mapping run name -> ID for all runs registered in MLMD.
"""
all_pipeline_runs = self.store.get_contexts_by_type(
PIPELINE_RUN_CONTEXT_TYPE_NAME
)
return {
run.name: self._get_pipeline_run_model_from_context(run)
for run in all_pipeline_runs
}
get_pipeline_run_steps(self, run_id)
Gets all steps for the given pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
int |
The ID of the pipeline run to get the steps for. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, zenml.zen_stores.metadata_store.MLMDStepRunModel] |
A dictionary of step names to step views. |
Source code in zenml/zen_stores/metadata_store.py
def get_pipeline_run_steps(
self, run_id: int
) -> Dict[str, MLMDStepRunModel]:
"""Gets all steps for the given pipeline run.
Args:
run_id: The ID of the pipeline run to get the steps for.
Returns:
A dictionary of step names to step views.
"""
steps: Dict[str, MLMDStepRunModel] = OrderedDict()
# reverse the executions as they get returned in reverse chronological
# order from the metadata store
executions = self.store.get_executions_by_context(run_id)
for execution in reversed(executions): # noqa
step = self._get_step_model_from_execution(execution)
steps[step.name] = step
logger.debug(f"Fetched {len(steps)} steps for pipeline run '{run_id}'.")
return steps
get_producer_step_from_artifact(self, artifact_id)
Find the original step that created an artifact.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_id |
int |
ID of the artifact for which to get the producer step. |
required |
Returns:
| Type | Description |
|---|---|
MLMDStepRunModel |
Original step that produced the artifact. |
Source code in zenml/zen_stores/metadata_store.py
def get_producer_step_from_artifact(
self, artifact_id: int
) -> MLMDStepRunModel:
"""Find the original step that created an artifact.
Args:
artifact_id: ID of the artifact for which to get the producer step.
Returns:
Original step that produced the artifact.
"""
executions_ids = set(
event.execution_id
for event in self.store.get_events_by_artifact_ids([artifact_id])
if event.type == event.OUTPUT
)
execution = self.store.get_executions_by_id(executions_ids)[0]
return self._get_step_model_from_execution(execution)
get_step_artifacts(self, step_id, step_parent_step_ids, step_name)
Returns input and output artifacts for the given step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
int |
The ID of the step to get the artifacts for. |
required |
step_parent_step_ids |
List[int] |
The IDs of the parent steps of the given step. |
required |
step_name |
str |
The name of the step. |
required |
Returns:
| Type | Description |
|---|---|
Tuple[Dict[str, zenml.zen_stores.metadata_store.MLMDArtifactModel], Dict[str, zenml.zen_stores.metadata_store.MLMDArtifactModel]] |
A tuple (inputs, outputs) where inputs and outputs are both Dicts mapping artifact names to the input and output artifacts respectively. |
Source code in zenml/zen_stores/metadata_store.py
def get_step_artifacts(
self, step_id: int, step_parent_step_ids: List[int], step_name: str
) -> Tuple[Dict[str, MLMDArtifactModel], Dict[str, MLMDArtifactModel]]:
"""Returns input and output artifacts for the given step.
Args:
step_id: The ID of the step to get the artifacts for.
step_parent_step_ids: The IDs of the parent steps of the given step.
step_name: The name of the step.
Returns:
A tuple (inputs, outputs) where inputs and outputs are both Dicts mapping artifact names to the input and output artifacts respectively.
"""
# maps artifact types to their string representation
artifact_type_mapping = {
type_.id: type_.name for type_ in self.store.get_artifact_types()
}
events = self.store.get_events_by_execution_ids([step_id]) # noqa
artifacts = self.store.get_artifacts_by_id(
[event.artifact_id for event in events]
)
inputs: Dict[str, MLMDArtifactModel] = {}
outputs: Dict[str, MLMDArtifactModel] = {}
# sort them according to artifact_id's so that the zip works.
events.sort(key=lambda x: x.artifact_id)
artifacts.sort(key=lambda x: x.id)
for event_proto, artifact_proto in zip(events, artifacts):
artifact_type = artifact_type_mapping[artifact_proto.type_id]
artifact_name = event_proto.path.steps[0].key
materializer = artifact_proto.properties[
MATERIALIZER_PROPERTY_KEY
].string_value
data_type = artifact_proto.properties[
DATATYPE_PROPERTY_KEY
].string_value
parent_step_id = step_id
if event_proto.type == event_proto.INPUT:
# In the case that this is an input event, we actually need
# to resolve it via its parents outputs.
for parent_id in step_parent_step_ids:
parent_step = self.get_step_by_id(parent_id)
parent_outputs = self.get_step_artifacts(
step_id=parent_id,
step_parent_step_ids=parent_step.mlmd_parent_step_ids,
step_name=parent_step.entrypoint_name,
)[1]
for parent_output in parent_outputs.values():
if artifact_proto.id == parent_output.mlmd_id:
parent_step_id = parent_id
artifact_id = event_proto.artifact_id
producer_step = self.get_producer_step_from_artifact(artifact_id)
producer_step_id = producer_step.mlmd_id
artifact = MLMDArtifactModel(
mlmd_id=artifact_id,
type=artifact_type,
uri=artifact_proto.uri,
materializer=materializer,
data_type=data_type,
mlmd_parent_step_id=parent_step_id,
mlmd_producer_step_id=producer_step_id,
is_cached=parent_step_id != producer_step_id,
)
if event_proto.type == event_proto.INPUT:
inputs[artifact_name] = artifact
elif event_proto.type == event_proto.OUTPUT:
outputs[artifact_name] = artifact
logger.debug(
"Fetched %d inputs and %d outputs for step '%s'.",
len(inputs),
len(outputs),
step_name,
)
return inputs, outputs
get_step_by_id(self, step_id)
Gets a step by its ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
int |
The ID of the step to get. |
required |
Returns:
| Type | Description |
|---|---|
MLMDStepRunModel |
A model of the step with the given ID. |
Source code in zenml/zen_stores/metadata_store.py
def get_step_by_id(self, step_id: int) -> MLMDStepRunModel:
"""Gets a step by its ID.
Args:
step_id: The ID of the step to get.
Returns:
A model of the step with the given ID.
"""
execution = self.store.get_executions_by_id([step_id])[0]
return self._get_step_model_from_execution(execution)
get_step_status(self, step_id)
Gets the execution status of a single step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
int |
The ID of the step to get the status for. |
required |
Returns:
| Type | Description |
|---|---|
ExecutionStatus |
The status of the step. |
Source code in zenml/zen_stores/metadata_store.py
def get_step_status(self, step_id: int) -> ExecutionStatus:
"""Gets the execution status of a single step.
Args:
step_id: The ID of the step to get the status for.
Returns:
ExecutionStatus: The status of the step.
"""
proto = self.store.get_executions_by_id([step_id])[0] # noqa
state = proto.last_known_state
if state == proto.COMPLETE:
return ExecutionStatus.COMPLETED
elif state == proto.RUNNING:
return ExecutionStatus.RUNNING
elif state == proto.CACHED:
return ExecutionStatus.CACHED
else:
return ExecutionStatus.FAILED
rest_zen_store
REST Zen Store implementation.
RestZenStore (BaseZenStore)
pydantic-model
Store implementation for accessing data from a REST API.
Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStore(BaseZenStore):
"""Store implementation for accessing data from a REST API."""
config: RestZenStoreConfiguration
TYPE: ClassVar[StoreType] = StoreType.REST
CONFIG_TYPE: ClassVar[Type[StoreConfiguration]] = RestZenStoreConfiguration
_api_token: Optional[str] = None
_session: Optional[requests.Session] = None
def _initialize_database(self) -> None:
"""Initialize the database."""
# don't do anything for a REST store
# ====================================
# ZenML Store interface implementation
# ====================================
# --------------------------------
# Initialization and configuration
# --------------------------------
def _initialize(self) -> None:
"""Initialize the REST store."""
# try to connect to the server to validate the configuration
self.active_user
@staticmethod
def get_local_url(path: str) -> str:
"""Get a local URL for a given local path.
Args:
path: the path string to build a URL out of.
Raises:
NotImplementedError: always
"""
raise NotImplementedError("Cannot build a REST url from a path.")
@classmethod
def copy_local_store(
cls,
config: StoreConfiguration,
path: str,
load_config_path: Optional[PurePath] = None,
) -> StoreConfiguration:
"""Copy a local store to a new location.
Use this method to create a copy of a store database to a new location
and return a new store configuration pointing to the database copy. This
only applies to stores that use the local filesystem to store their
data. Calling this method for remote stores simply returns the input
store configuration unaltered.
Args:
config: The configuration of the store to copy.
path: The new local path where the store DB will be copied.
load_config_path: path that will be used to load the copied store
database. This can be set to a value different from `path`
if the local database copy will be loaded from a different
environment, e.g. when the database is copied to a container
image and loaded using a different absolute path. This will be
reflected in the paths and URLs encoded in the copied store
configuration.
Returns:
The store configuration of the copied store.
"""
assert isinstance(config, RestZenStoreConfiguration)
if isinstance(config.verify_ssl, str) and os.path.isfile(
config.verify_ssl
):
config = config.copy(deep=True)
with open(config.verify_ssl, "r") as f:
config.verify_ssl = f.read()
return config
def get_store_info(self) -> ServerModel:
"""Get information about the server.
Returns:
Information about the server.
"""
body = self.get(INFO)
return ServerModel.parse_obj(body)
# ------------
# TFX Metadata
# ------------
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the TFX metadata config of this ZenStore.
Args:
expand_certs: Whether to expand the certificate paths in the
connection config to their value.
Raises:
ValueError: if the server response is invalid.
Returns:
The TFX metadata config of this ZenStore.
"""
from google.protobuf.json_format import Parse
from ml_metadata.proto.metadata_store_pb2 import ConnectionConfig
body = self.get(f"{METADATA_CONFIG}")
if not isinstance(body, str):
raise ValueError(
f"Invalid response from server: {body}. Expected string."
)
metadata_config_pb = Parse(body, ConnectionConfig())
if not expand_certs:
if metadata_config_pb.HasField(
"mysql"
) and metadata_config_pb.mysql.HasField("ssl_options"):
# Save the certificates in a secure location on disk
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
if not metadata_config_pb.mysql.ssl_options.HasField(
key.lstrip("ssl_")
):
continue
content = getattr(
metadata_config_pb.mysql.ssl_options, key.lstrip("ssl_")
)
if content and not os.path.isfile(content):
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, f"{key}.pem")
with open(file_path, "w") as f:
f.write(content)
file_path.chmod(0o600)
setattr(
metadata_config_pb.mysql.ssl_options,
key.lstrip("ssl_"),
str(file_path),
)
return metadata_config_pb
# ------
# Stacks
# ------
@track(AnalyticsEvent.REGISTERED_STACK)
def create_stack(
self,
stack: StackModel,
) -> StackModel:
"""Register a new stack.
Args:
stack: The stack to register.
Returns:
The registered stack.
"""
return self._create_project_scoped_resource(
resource=stack,
route=STACKS,
request_model=CreateStackRequest,
)
def get_stack(self, stack_id: UUID) -> StackModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
"""
return self._get_resource(
resource_id=stack_id,
route=STACKS,
resource_model=StackModel,
)
def list_stacks(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_id: Optional[UUID] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[StackModel]:
"""List all stacks matching the given filter criteria.
Args:
project_name_or_id: Id or name of the Project containing the stack
user_name_or_id: Optionally filter stacks by their owner
component_id: Optionally filter for stacks that contain the
component
name: Optionally filter stacks by their name
is_shared: Optionally filter out stacks by whether they are shared
or not
Returns:
A list of all stacks matching the filter criteria.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=STACKS,
resource_model=StackModel,
**filters,
)
@track(AnalyticsEvent.UPDATED_STACK)
def update_stack(
self,
stack: StackModel,
) -> StackModel:
"""Update a stack.
Args:
stack: The stack to use for the update.
Returns:
The updated stack.
"""
return self._update_resource(
resource=stack,
route=STACKS,
request_model=UpdateStackRequest,
)
@track(AnalyticsEvent.DELETED_STACK)
def delete_stack(self, stack_id: UUID) -> None:
"""Delete a stack.
Args:
stack_id: The ID of the stack to delete.
"""
self._delete_resource(
resource_id=stack_id,
route=STACKS,
)
# ----------------
# Stack components
# ----------------
@track(AnalyticsEvent.REGISTERED_STACK_COMPONENT)
def create_stack_component(
self,
component: ComponentModel,
) -> ComponentModel:
"""Create a stack component.
Args:
component: The stack component to create.
Returns:
The created stack component.
"""
return self._create_project_scoped_resource(
resource=component,
route=STACK_COMPONENTS,
# TODO[Stefan]: for when the request model is ready
# request_model=CreateStackComponentRequest,
)
def get_stack_component(self, component_id: UUID) -> ComponentModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
"""
return self._get_resource(
resource_id=component_id,
route=STACK_COMPONENTS,
resource_model=ComponentModel,
)
def list_stack_components(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
type: Optional[str] = None,
flavor_name: Optional[str] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[ComponentModel]:
"""List all stack components matching the given filter criteria.
Args:
project_name_or_id: The ID or name of the Project to which the stack
components belong
type: Optionally filter by type of stack component
flavor_name: Optionally filter by flavor
user_name_or_id: Optionally filter stack components by the owner
name: Optionally filter stack component by name
is_shared: Optionally filter out stack component by whether they are
shared or not
Returns:
A list of all stack components matching the filter criteria.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=STACK_COMPONENTS,
resource_model=ComponentModel,
**filters,
)
@track(AnalyticsEvent.UPDATED_STACK_COMPONENT)
def update_stack_component(
self,
component: ComponentModel,
) -> ComponentModel:
"""Update an existing stack component.
Args:
component: The stack component to use for the update.
Returns:
The updated stack component.
"""
return self._update_resource(
resource=component,
route=STACK_COMPONENTS,
# TODO[Stefan]: for when the request model is ready
# request_model=UpdateComponentRequest,
)
@track(AnalyticsEvent.DELETED_STACK_COMPONENT)
def delete_stack_component(self, component_id: UUID) -> None:
"""Delete a stack component.
Args:
component_id: The ID of the stack component to delete.
"""
self._delete_resource(
resource_id=component_id,
route=STACK_COMPONENTS,
)
def get_stack_component_side_effects(
self,
component_id: UUID,
run_id: UUID,
pipeline_id: UUID,
stack_id: UUID,
) -> Dict[Any, Any]:
"""Get the side effects of a stack component.
Args:
component_id: The ID of the stack component to get side effects for.
run_id: The ID of the run to get side effects for.
pipeline_id: The ID of the pipeline to get side effects for.
stack_id: The ID of the stack to get side effects for.
"""
# -----------------------
# Stack component flavors
# -----------------------
@track(AnalyticsEvent.CREATED_FLAVOR)
def create_flavor(
self,
flavor: FlavorModel,
) -> FlavorModel:
"""Creates a new stack component flavor.
Args:
flavor: The stack component flavor to create.
Returns:
The newly created flavor.
"""
return self._create_project_scoped_resource(
resource=flavor,
route=FLAVORS,
# TODO[Stefan]: for when the request model is ready
# request_model=CreateFlavorRequest,
)
def get_flavor(self, flavor_id: UUID) -> FlavorModel:
"""Get a stack component flavor by ID.
Args:
flavor_id: The ID of the stack component flavor to get.
Returns:
The stack component flavor.
"""
return self._get_resource(
resource_id=flavor_id,
route=FLAVORS,
resource_model=FlavorModel,
)
def list_flavors(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_type: Optional[StackComponentType] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[FlavorModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
project_name_or_id: Optionally filter by the Project to which the
component flavors belong
user_name_or_id: Optionally filter by the owner
component_type: Optionally filter by type of stack component
name: Optionally filter flavors by name
is_shared: Optionally filter out flavors by whether they are
shared or not
Returns:
List of all the stack component flavors matching the given criteria.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=FLAVORS,
resource_model=FlavorModel,
**filters,
)
@track(AnalyticsEvent.UPDATED_FLAVOR)
def update_flavor(self, flavor: FlavorModel) -> FlavorModel:
"""Update an existing stack component flavor.
Args:
flavor: The stack component flavor to use for the update.
Returns:
The updated stack component flavor.
"""
return self._update_resource(
resource=flavor,
route=FLAVORS,
# TODO[Stefan]: for when the request model is ready
# request_model=UpdateFlavorRequest,
)
@track(AnalyticsEvent.DELETED_FLAVOR)
def delete_flavor(self, flavor_id: UUID) -> None:
"""Delete a stack component flavor.
Args:
flavor_id: The ID of the stack component flavor to delete.
"""
self._delete_resource(
resource_id=flavor_id,
route=FLAVORS,
)
# -----
# Users
# -----
@property
def active_user_name(self) -> str:
"""Gets the active username.
Returns:
The active username.
"""
return self.config.username
@track(AnalyticsEvent.CREATED_USER)
def create_user(self, user: UserModel) -> UserModel:
"""Creates a new user.
Args:
user: User to be created.
Returns:
The newly created user.
"""
return self._create_resource(
resource=user,
route=USERS,
request_model=CreateUserRequest,
response_model=CreateUserResponse,
)
def get_user(self, user_name_or_id: Union[str, UUID]) -> UserModel:
"""Gets a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
"""
return self._get_resource(
resource_id=user_name_or_id,
route=USERS,
resource_model=UserModel,
)
# TODO: [ALEX] add filtering param(s)
def list_users(self) -> List[UserModel]:
"""List all users.
Returns:
A list of all users.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=USERS,
resource_model=UserModel,
**filters,
)
@track(AnalyticsEvent.UPDATED_USER)
def update_user(self, user: UserModel) -> UserModel:
"""Updates an existing user.
Args:
user: The user model to use for the update.
Returns:
The updated user.
"""
return self._update_resource(
resource=user,
route=USERS,
request_model=UpdateUserRequest,
)
@track(AnalyticsEvent.DELETED_USER)
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
"""Deletes a user.
Args:
user_name_or_id: The name or ID of the user to delete.
"""
self._delete_resource(
resource_id=user_name_or_id,
route=USERS,
)
def user_email_opt_in(
self,
user_name_or_id: Union[str, UUID],
user_opt_in_response: bool,
email: Optional[str] = None,
) -> UserModel:
"""Persist user response to the email prompt.
Args:
user_name_or_id: The name or the ID of the user.
user_opt_in_response: Whether this email should be associated
with the user id in the telemetry
email: The users email
Returns:
The updated user.
"""
request = EmailOptInModel(
email=email, email_opted_in=user_opt_in_response
)
route = f"{USERS}/{str(user_name_or_id)}{EMAIL_ANALYTICS}"
response_body = self.put(route, body=request)
user = UserModel.parse_obj(response_body)
return user
# -----
# Teams
# -----
@track(AnalyticsEvent.CREATED_TEAM)
def create_team(self, team: TeamModel) -> TeamModel:
"""Creates a new team.
Args:
team: The team model to create.
Returns:
The newly created team.
"""
return self._create_resource(
resource=team,
route=TEAMS,
request_model=CreateTeamRequest,
)
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
"""
return self._get_resource(
resource_id=team_name_or_id,
route=TEAMS,
resource_model=TeamModel,
)
def list_teams(self) -> List[TeamModel]:
"""List all teams.
Returns:
A list of all teams.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=TEAMS,
resource_model=TeamModel,
**filters,
)
@track(AnalyticsEvent.UPDATED_TEAM)
def update_team(self, team: TeamModel) -> TeamModel:
"""Update an existing team.
Args:
team: The team to use for the update.
Returns:
The updated team.
"""
return self._update_resource(
resource=team,
route=TEAMS,
request_model=UpdateTeamRequest,
)
@track(AnalyticsEvent.DELETED_TEAM)
def delete_team(self, team_name_or_id: Union[str, UUID]) -> None:
"""Deletes a team.
Args:
team_name_or_id: Name or ID of the team to delete.
"""
self._delete_resource(
resource_id=team_name_or_id,
route=TEAMS,
)
# ---------------
# Team membership
# ---------------
def get_users_for_team(
self, team_name_or_id: Union[str, UUID]
) -> List[UserModel]:
"""Fetches all users of a team.
Args:
team_name_or_id: The name or ID of the team for which to get users.
"""
def get_teams_for_user(
self, user_name_or_id: Union[str, UUID]
) -> List[TeamModel]:
"""Fetches all teams for a user.
Args:
user_name_or_id: The name or ID of the user for which to get all
teams.
"""
def add_user_to_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Adds a user to a team.
Args:
user_name_or_id: Name or ID of the user to add to the team.
team_name_or_id: Name or ID of the team to which to add the user to.
"""
def remove_user_from_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Removes a user from a team.
Args:
user_name_or_id: Name or ID of the user to remove from the team.
team_name_or_id: Name or ID of the team from which to remove the user.
"""
# -----
# Roles
# -----
@track(AnalyticsEvent.CREATED_ROLE)
def create_role(self, role: RoleModel) -> RoleModel:
"""Creates a new role.
Args:
role: The role model to create.
Returns:
The newly created role.
"""
return self._create_resource(
resource=role,
route=ROLES,
request_model=CreateRoleRequest,
)
# TODO: consider using team_id instead
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
"""
return self._get_resource(
resource_id=role_name_or_id,
route=ROLES,
resource_model=RoleModel,
)
# TODO: [ALEX] add filtering param(s)
def list_roles(self) -> List[RoleModel]:
"""List all roles.
Returns:
A list of all roles.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=ROLES,
resource_model=RoleModel,
**filters,
)
@track(AnalyticsEvent.UPDATED_ROLE)
def update_role(self, role: RoleModel) -> RoleModel:
"""Update an existing role.
Args:
role: The role to use for the update.
Returns:
The updated role.
"""
return self._update_resource(
resource=role,
route=ROLES,
request_model=UpdateRoleRequest,
)
@track(AnalyticsEvent.DELETED_ROLE)
def delete_role(self, role_name_or_id: Union[str, UUID]) -> None:
"""Deletes a role.
Args:
role_name_or_id: Name or ID of the role to delete.
"""
self._delete_resource(
resource_id=role_name_or_id,
route=ROLES,
)
# ----------------
# Role assignments
# ----------------
def list_role_assignments(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
team_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
) -> List[RoleAssignmentModel]:
"""List all role assignments.
Args:
project_name_or_id: If provided, only list assignments for the given
project
team_name_or_id: If provided, only list assignments for the given
team
user_name_or_id: If provided, only list assignments for the given
user
Returns:
A list of all role assignments.
"""
roles: List[RoleAssignmentModel] = []
if user_name_or_id:
roles.extend(
self._list_resources(
route=f"{USERS}/{user_name_or_id}{ROLES}",
resource_model=RoleAssignmentModel,
project_name_or_id=project_name_or_id,
)
)
if team_name_or_id:
roles.extend(
self._list_resources(
route=f"{TEAMS}/{team_name_or_id}{ROLES}",
resource_model=RoleAssignmentModel,
project_name_or_id=project_name_or_id,
)
)
return roles
def assign_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
project_name_or_id: Optional[Union[str, UUID]] = None,
is_user: bool = True,
) -> None:
"""Assigns a role to a user or team, scoped to a specific project.
Args:
role_name_or_id: Name or ID of the role to assign.
user_or_team_name_or_id: Name or ID of the user or team to which to
assign the role.
is_user: Whether `user_or_team_id` refers to a user or a team.
project_name_or_id: Optional Name or ID of a project in which to
assign the role. If this is not provided, the role will be
assigned globally.
"""
def revoke_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
is_user: bool = True,
project_name_or_id: Optional[Union[str, UUID]] = None,
) -> None:
"""Revokes a role from a user or team for a given project.
Args:
role_name_or_id: ID of the role to revoke.
user_or_team_name_or_id: Name or ID of the user or team from which
to revoke the role.
is_user: Whether `user_or_team_id` refers to a user or a team.
project_name_or_id: Optional ID of a project in which to revoke
the role. If this is not provided, the role will be revoked
globally.
"""
# --------
# Projects
# --------
@track(AnalyticsEvent.CREATED_PROJECT)
def create_project(self, project: ProjectModel) -> ProjectModel:
"""Creates a new project.
Args:
project: The project to create.
Returns:
The newly created project.
"""
return self._create_resource(
resource=project,
route=PROJECTS,
request_model=CreateProjectRequest,
)
def get_project(self, project_name_or_id: Union[UUID, str]) -> ProjectModel:
"""Get an existing project by name or ID.
Args:
project_name_or_id: Name or ID of the project to get.
Returns:
The requested project.
"""
return self._get_resource(
resource_id=project_name_or_id,
route=PROJECTS,
resource_model=ProjectModel,
)
# TODO: [ALEX] add filtering param(s)
def list_projects(self) -> List[ProjectModel]:
"""List all projects.
Returns:
A list of all projects.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=PROJECTS,
resource_model=ProjectModel,
**filters,
)
@track(AnalyticsEvent.UPDATED_PROJECT)
def update_project(self, project: ProjectModel) -> ProjectModel:
"""Update an existing project.
Args:
project: The project to use for the update.
Returns:
The updated project.
"""
return self._update_resource(
resource=project,
route=PROJECTS,
request_model=UpdateProjectRequest,
)
@track(AnalyticsEvent.DELETED_PROJECT)
def delete_project(self, project_name_or_id: Union[str, UUID]) -> None:
"""Deletes a project.
Args:
project_name_or_id: Name or ID of the project to delete.
"""
self._delete_resource(
resource_id=project_name_or_id,
route=PROJECTS,
)
# ---------
# Pipelines
# ---------
@track(AnalyticsEvent.CREATE_PIPELINE)
def create_pipeline(self, pipeline: PipelineModel) -> PipelineModel:
"""Creates a new pipeline in a project.
Args:
pipeline: The pipeline to create.
Returns:
The newly created pipeline.
"""
return self._create_project_scoped_resource(
resource=pipeline,
route=PIPELINES,
request_model=CreatePipelineRequest,
)
def get_pipeline(self, pipeline_id: UUID) -> PipelineModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
"""
return self._get_resource(
resource_id=pipeline_id,
route=PIPELINES,
resource_model=PipelineModel,
)
def list_pipelines(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
name: Optional[str] = None,
) -> List[PipelineModel]:
"""List all pipelines in the project.
Args:
project_name_or_id: If provided, only list pipelines in this project.
user_name_or_id: If provided, only list pipelines from this user.
name: If provided, only list pipelines with this name.
Returns:
A list of pipelines.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=PIPELINES,
resource_model=PipelineModel,
**filters,
)
@track(AnalyticsEvent.UPDATE_PIPELINE)
def update_pipeline(self, pipeline: PipelineModel) -> PipelineModel:
"""Updates a pipeline.
Args:
pipeline: The pipeline to use for the update.
Returns:
The updated pipeline.
"""
return self._update_resource(
resource=pipeline,
route=PIPELINES,
request_model=UpdatePipelineRequest,
)
@track(AnalyticsEvent.DELETE_PIPELINE)
def delete_pipeline(self, pipeline_id: UUID) -> None:
"""Deletes a pipeline.
Args:
pipeline_id: The ID of the pipeline to delete.
"""
self._delete_resource(
resource_id=pipeline_id,
route=PIPELINES,
)
# --------------
# Pipeline steps
# --------------
# TODO: Note that this doesn't have a corresponding API endpoint (consider adding?)
# TODO: Discuss whether we even need this, given that the endpoint is on
# pipeline runs
# TODO: [ALEX] add filtering param(s)
def list_steps(self, pipeline_id: UUID) -> List[StepRunModel]:
"""List all steps.
Args:
pipeline_id: The ID of the pipeline to list steps for.
"""
# --------------
# Pipeline runs
# --------------
def get_run(self, run_id: UUID) -> PipelineRunModel:
"""Gets a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
Returns:
The pipeline run.
"""
return self._get_resource(
resource_id=run_id,
route=RUNS,
resource_model=PipelineRunModel,
)
# TODO: Figure out what exactly gets returned from this
def get_run_component_side_effects(
self,
run_id: UUID,
component_id: Optional[UUID] = None,
) -> Dict[str, Any]:
"""Gets the side effects for a component in a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
component_id: The ID of the component to get.
"""
def list_runs(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
stack_id: Optional[UUID] = None,
component_id: Optional[UUID] = None,
run_name: Optional[str] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
pipeline_id: Optional[UUID] = None,
unlisted: bool = False,
) -> List[PipelineRunModel]:
"""Gets all pipeline runs.
Args:
project_name_or_id: If provided, only return runs for this project.
stack_id: If provided, only return runs for this stack.
component_id: Optionally filter for runs that used the
component
run_name: Run name if provided
user_name_or_id: If provided, only return runs for this user.
pipeline_id: If provided, only return runs for this pipeline.
unlisted: If True, only return unlisted runs that are not
associated with any pipeline (filter by `pipeline_id==None`).
Returns:
A list of all pipeline runs.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=RUNS,
resource_model=PipelineRunModel,
**filters,
)
def get_run_status(self, run_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a pipeline run.
Args:
run_id: The ID of the pipeline run to get the status for.
Returns:
The execution status of the pipeline run.
"""
body = self.get(f"{RUNS}/{str(run_id)}{STATUS}")
return ExecutionStatus(body)
# ------------------
# Pipeline run steps
# ------------------
def get_run_step(self, step_id: UUID) -> StepRunModel:
"""Get a step by ID.
Args:
step_id: The ID of the step to get.
Returns:
The step.
"""
return self._get_resource(
resource_id=step_id,
route=STEPS,
resource_model=StepRunModel,
)
def get_run_step_outputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get a list of outputs for a specific step.
Args:
step_id: The id of the step to get outputs for.
Returns:
A dict mapping artifact names to the output artifacts for the step.
Raises:
ValueError: if the response from the API is not a dict.
"""
body = self.get(f"{STEPS}/{str(step_id)}{OUTPUTS}")
if not isinstance(body, dict):
raise ValueError(
f"Bad API Response. Expected dict, got {type(body)}"
)
return {
name: ArtifactModel.parse_obj(entry) for name, entry in body.items()
}
def get_run_step_inputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get a list of inputs for a specific step.
Args:
step_id: The id of the step to get inputs for.
Returns:
A dict mapping artifact names to the input artifacts for the step.
Raises:
ValueError: if the response from the API is not a dict.
"""
body = self.get(f"{STEPS}/{str(step_id)}{INPUTS}")
if not isinstance(body, dict):
raise ValueError(
f"Bad API Response. Expected dict, got {type(body)}"
)
return {
name: ArtifactModel.parse_obj(entry) for name, entry in body.items()
}
def get_run_step_status(self, step_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a single step.
Args:
step_id: The ID of the step to get the status for.
Returns:
The execution status of the step.
"""
body = self.get(f"{STEPS}/{str(step_id)}{STATUS}")
return ExecutionStatus(body)
def list_run_steps(self, run_id: UUID) -> List[StepRunModel]:
"""Gets all steps in a pipeline run.
Args:
run_id: The ID of the pipeline run for which to list runs.
Returns:
A mapping from step names to step models for all steps in the run.
"""
return self._list_resources(
route=f"{RUNS}/{str(run_id)}{STEPS}",
resource_model=StepRunModel,
)
def list_artifacts(
self, artifact_uri: Optional[str] = None
) -> List[ArtifactModel]:
"""Lists all artifacts.
Args:
artifact_uri: If specified, only artifacts with the given URI will
be returned.
Returns:
A list of all artifacts.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=ARTIFACTS,
resource_model=ArtifactModel,
**filters,
)
# =======================
# Internal helper methods
# =======================
def _get_auth_token(self) -> str:
"""Get the authentication token for the REST store.
Returns:
The authentication token.
Raises:
ValueError: if the response from the server isn't in the right format.
"""
if self._api_token is None:
response = self._handle_response(
requests.post(
self.url + API + VERSION_1 + LOGIN,
data={
"username": self.config.username,
"password": self.config.password,
},
verify=self.config.verify_ssl,
timeout=self.config.http_timeout,
)
)
if not isinstance(response, dict) or "access_token" not in response:
raise ValueError(
f"Bad API Response. Expected access token dict, got "
f"{type(response)}"
)
self._api_token = response["access_token"]
return self._api_token
@property
def session(self) -> requests.Session:
"""Authenticate to the ZenML server.
Returns:
A requests session with the authentication token.
"""
if self._session is None:
if self.config.verify_ssl is False:
urllib3.disable_warnings(
urllib3.exceptions.InsecureRequestWarning
)
self._session = requests.Session()
self._session.verify = self.config.verify_ssl
token = self._get_auth_token()
self._session.headers.update({"Authorization": "Bearer " + token})
logger.debug("Authenticated to ZenML server.")
return self._session
def _handle_response(self, response: requests.Response) -> Json:
"""Handle API response, translating http status codes to Exception.
Args:
response: The response to handle.
Returns:
The parsed response.
Raises:
DoesNotExistException: If the response indicates that the
requested entity does not exist.
EntityExistsError: If the response indicates that the requested
entity already exists.
AuthorizationException: If the response indicates that the request
is not authorized.
KeyError: If the response indicates that the requested entity
does not exist.
RuntimeError: If the response indicates that the requested entity
does not exist.
StackComponentExistsError: If the response indicates that the
requested entity already exists.
StackExistsError: If the response indicates that the requested
entity already exists.
ValueError: If the response indicates that the requested entity
does not exist.
"""
if response.status_code >= 200 and response.status_code < 300:
try:
payload: Json = response.json()
return payload
except requests.exceptions.JSONDecodeError:
raise ValueError(
"Bad response from API. Expected json, got\n"
f"{response.text}"
)
elif response.status_code == 401:
raise AuthorizationException(
f"{response.status_code} Client Error: Unauthorized request to "
f"URL {response.url}: {response.json().get('detail')}"
)
elif response.status_code == 404:
if "KeyError" in response.text:
raise KeyError(
response.json().get("detail", (response.text,))[1]
)
elif "DoesNotExistException" in response.text:
message = ": ".join(
response.json().get("detail", (response.text,))
)
raise DoesNotExistException(message)
raise DoesNotExistException("Endpoint does not exist.")
elif response.status_code == 409:
if "StackComponentExistsError" in response.text:
raise StackComponentExistsError(
message=": ".join(
response.json().get("detail", (response.text,))
)
)
elif "StackExistsError" in response.text:
raise StackExistsError(
message=": ".join(
response.json().get("detail", (response.text,))
)
)
elif "EntityExistsError" in response.text:
raise EntityExistsError(
message=": ".join(
response.json().get("detail", (response.text,))
)
)
else:
raise ValueError(
": ".join(response.json().get("detail", (response.text,)))
)
elif response.status_code == 422:
raise RuntimeError(
": ".join(response.json().get("detail", (response.text,)))
)
elif response.status_code == 500:
raise RuntimeError(response.text)
else:
raise RuntimeError(
"Error retrieving from API. Got response "
f"{response.status_code} with body:\n{response.text}"
)
def _request(
self,
method: str,
url: str,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a request to the REST API.
Args:
method: The HTTP method to use.
url: The URL to request.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The parsed response.
"""
params = {k: str(v) for k, v in params.items()} if params else {}
try:
return self._handle_response(
self.session.request(
method,
url,
params=params,
verify=self.config.verify_ssl,
timeout=self.config.http_timeout,
**kwargs,
)
)
except AuthorizationException:
# The authentication token could have expired; refresh it and try
# again
self._session = None
return self._handle_response(
self.session.request(
method,
url,
params=params,
verify=self.config.verify_ssl,
timeout=self.config.http_timeout,
**kwargs,
)
)
def get(
self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
"""Make a GET request to the given endpoint path.
Args:
path: The path to the endpoint.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending GET request to {path}...")
return self._request(
"GET", self.url + API + VERSION_1 + path, params=params, **kwargs
)
def delete(
self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
"""Make a DELETE request to the given endpoint path.
Args:
path: The path to the endpoint.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending DELETE request to {path}...")
return self._request(
"DELETE", self.url + API + VERSION_1 + path, params=params, **kwargs
)
def post(
self,
path: str,
body: BaseModel,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a POST request to the given endpoint path.
Args:
path: The path to the endpoint.
body: The body to send.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending POST request to {path}...")
return self._request(
"POST",
self.url + API + VERSION_1 + path,
data=body.json(),
params=params,
**kwargs,
)
def put(
self,
path: str,
body: BaseModel,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a PUT request to the given endpoint path.
Args:
path: The path to the endpoint.
body: The body to send.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending PUT request to {path}...")
return self._request(
"PUT",
self.url + API + VERSION_1 + path,
data=body.json(),
params=params,
**kwargs,
)
def _create_resource(
self,
resource: AnyModel,
route: str,
request_model: Optional[Type[CreateRequest[AnyModel]]] = None,
response_model: Optional[Type[CreateResponse[AnyModel]]] = None,
) -> AnyModel:
"""Create a new resource.
Args:
resource: The resource to create.
route: The resource REST API route to use.
request_model: Optional model to use to serialize the request body.
If not provided, the resource object itself will be used.
response_model: Optional model to use to deserialize the response
body. If not provided, the resource class itself will be used.
Returns:
The created resource.
"""
request: BaseModel = resource
if request_model is not None:
request = request_model.from_model(resource)
response_body = self.post(f"{route}", body=request)
if response_model is not None:
response = response_model.parse_obj(response_body)
created_resource = response.to_model()
else:
created_resource = resource.parse_obj(response_body)
return created_resource
def _create_project_scoped_resource(
self,
resource: AnyProjectScopedModel,
route: str,
request_model: Optional[
Type[CreateRequest[AnyProjectScopedModel]]
] = None,
response_model: Optional[
Type[CreateResponse[AnyProjectScopedModel]]
] = None,
) -> AnyProjectScopedModel:
"""Create a new project scoped resource.
Args:
resource: The resource to create.
route: The resource REST API route to use.
request_model: Optional model to use to serialize the request body.
If not provided, the resource object itself will be used.
response_model: Optional model to use to deserialize the response
body. If not provided, the resource class itself will be used.
Returns:
The created resource.
"""
return self._create_resource(
resource=resource,
route=f"{PROJECTS}/{str(resource.project)}{route}",
request_model=request_model,
response_model=response_model,
)
def _get_resource(
self,
resource_id: Union[str, UUID],
route: str,
resource_model: Type[AnyModel],
) -> AnyModel:
"""Retrieve a single resource.
Args:
resource_id: The ID of the resource to retrieve.
route: The resource REST API route to use.
resource_model: Model to use to serialize the response body.
Returns:
The retrieved resource.
"""
body = self.get(f"{route}/{str(resource_id)}")
return resource_model.parse_obj(body)
def _list_resources(
self,
route: str,
resource_model: Type[AnyModel],
**filters: Any,
) -> List[AnyModel]:
"""Retrieve a list of resources filtered by some criteria.
Args:
route: The resource REST API route to use.
resource_model: Model to use to serialize the response body.
filters: Filter parameters to use in the query.
Returns:
List of retrieved resources matching the filter criteria.
Raises:
ValueError: If the value returned by the server is not a list.
"""
# leave out filter params that are not supplied
params = dict(filter(lambda x: x[1] is not None, filters.items()))
body = self.get(f"{route}", params=params)
if not isinstance(body, list):
raise ValueError(
f"Bad API Response. Expected list, got {type(body)}"
)
return [resource_model.parse_obj(entry) for entry in body]
def _update_resource(
self,
resource: AnyModel,
route: str,
request_model: Optional[Type[UpdateRequest[AnyModel]]] = None,
response_model: Optional[Type[UpdateResponse[AnyModel]]] = None,
) -> AnyModel:
"""Update an existing resource.
Args:
resource: The resource to update.
route: The resource REST API route to use.
request_model: Optional model to use to serialize the request body.
If not provided, the resource object itself will be used.
response_model: Optional model to use to deserialize the response
body. If not provided, the resource class itself will be used.
Returns:
The updated resource.
"""
request: BaseModel = resource
if request_model is not None:
request = request_model.from_model(resource)
response_body = self.put(f"{route}/{str(resource.id)}", body=request)
if response_model is not None:
response = response_model.parse_obj(response_body)
updated_resource = response.to_model()
else:
updated_resource = resource.parse_obj(response_body)
return updated_resource
def _delete_resource(
self, resource_id: Union[str, UUID], route: str
) -> None:
"""Delete a resource.
Args:
resource_id: The ID of the resource to delete.
route: The resource REST API route to use.
"""
self.delete(f"{route}/{str(resource_id)}")
def _sync_runs(self) -> None:
"""Syncs runs from MLMD.
Raises:
NotImplementedError: This internal method may not be called on a
`RestZenStore`.
"""
raise NotImplementedError
active_user_name: str
property
readonly
Gets the active username.
Returns:
| Type | Description |
|---|---|
str |
The active username. |
session: Session
property
readonly
Authenticate to the ZenML server.
Returns:
| Type | Description |
|---|---|
Session |
A requests session with the authentication token. |
CONFIG_TYPE (StoreConfiguration)
pydantic-model
REST ZenML store configuration.
Attributes:
| Name | Type | Description |
|---|---|---|
username |
str |
The username to use to connect to the Zen server. |
password |
str |
The password to use to connect to the Zen server. |
verify_ssl |
Union[bool, str] |
Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use or the CA bundle value itself. |
http_timeout |
int |
The timeout to use for all requests. |
Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStoreConfiguration(StoreConfiguration):
"""REST ZenML store configuration.
Attributes:
username: The username to use to connect to the Zen server.
password: The password to use to connect to the Zen server.
verify_ssl: Either a boolean, in which case it controls whether we
verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use or the CA bundle value itself.
http_timeout: The timeout to use for all requests.
"""
type: StoreType = StoreType.REST
username: str
password: str = ""
verify_ssl: Union[bool, str] = True
http_timeout: int = DEFAULT_HTTP_TIMEOUT
@validator("url")
def validate_url(cls, url: str) -> str:
"""Validates that the URL is a well formed REST store URL.
Args:
url: The URL to be validated.
Returns:
The validated URL without trailing slashes.
Raises:
ValueError: If the URL is not a well formed REST store URL.
"""
url = url.rstrip("/")
scheme = re.search("^([a-z0-9]+://)", url)
if scheme is None or scheme.group() not in ("https://", "http://"):
raise ValueError(
"Invalid URL for REST store: {url}. Should be in the form "
"https://hostname[:port] or http://hostname[:port]."
)
return url
@validator("verify_ssl")
def validate_verify_ssl(
cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
"""Validates that the verify_ssl field either points to a file or is a bool.
Args:
verify_ssl: The verify_ssl value to be validated.
Returns:
The validated verify_ssl value.
"""
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
if isinstance(verify_ssl, bool) or verify_ssl.startswith(
str(secret_folder)
):
return verify_ssl
if os.path.isfile(verify_ssl):
with open(verify_ssl, "r") as f:
verify_ssl = f.read()
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, "ca_bundle.pem")
with open(file_path, "w") as f:
f.write(verify_ssl)
file_path.chmod(0o600)
verify_ssl = str(file_path)
return verify_ssl
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the `verify_ssl` attribute can be expanded to the contents
# of the certificate file.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
Config
Pydantic configuration class.
Source code in zenml/zen_stores/rest_zen_store.py
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the `verify_ssl` attribute can be expanded to the contents
# of the certificate file.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
validate_url(url)
classmethod
Validates that the URL is a well formed REST store URL.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The URL to be validated. |
required |
Returns:
| Type | Description |
|---|---|
str |
The validated URL without trailing slashes. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the URL is not a well formed REST store URL. |
Source code in zenml/zen_stores/rest_zen_store.py
@validator("url")
def validate_url(cls, url: str) -> str:
"""Validates that the URL is a well formed REST store URL.
Args:
url: The URL to be validated.
Returns:
The validated URL without trailing slashes.
Raises:
ValueError: If the URL is not a well formed REST store URL.
"""
url = url.rstrip("/")
scheme = re.search("^([a-z0-9]+://)", url)
if scheme is None or scheme.group() not in ("https://", "http://"):
raise ValueError(
"Invalid URL for REST store: {url}. Should be in the form "
"https://hostname[:port] or http://hostname[:port]."
)
return url
validate_verify_ssl(verify_ssl)
classmethod
Validates that the verify_ssl field either points to a file or is a bool.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
verify_ssl |
Union[bool, str] |
The verify_ssl value to be validated. |
required |
Returns:
| Type | Description |
|---|---|
Union[bool, str] |
The validated verify_ssl value. |
Source code in zenml/zen_stores/rest_zen_store.py
@validator("verify_ssl")
def validate_verify_ssl(
cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
"""Validates that the verify_ssl field either points to a file or is a bool.
Args:
verify_ssl: The verify_ssl value to be validated.
Returns:
The validated verify_ssl value.
"""
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
if isinstance(verify_ssl, bool) or verify_ssl.startswith(
str(secret_folder)
):
return verify_ssl
if os.path.isfile(verify_ssl):
with open(verify_ssl, "r") as f:
verify_ssl = f.read()
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, "ca_bundle.pem")
with open(file_path, "w") as f:
f.write(verify_ssl)
file_path.chmod(0o600)
verify_ssl = str(file_path)
return verify_ssl
add_user_to_team(self, user_name_or_id, team_name_or_id)
Adds a user to a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user to add to the team. |
required |
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to which to add the user to. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def add_user_to_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Adds a user to a team.
Args:
user_name_or_id: Name or ID of the user to add to the team.
team_name_or_id: Name or ID of the team to which to add the user to.
"""
assign_role(self, role_name_or_id, user_or_team_name_or_id, project_name_or_id=None, is_user=True)
Assigns a role to a user or team, scoped to a specific project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to assign. |
required |
user_or_team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user or team to which to assign the role. |
required |
is_user |
bool |
Whether |
True |
project_name_or_id |
Union[str, uuid.UUID] |
Optional Name or ID of a project in which to assign the role. If this is not provided, the role will be assigned globally. |
None |
Source code in zenml/zen_stores/rest_zen_store.py
def assign_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
project_name_or_id: Optional[Union[str, UUID]] = None,
is_user: bool = True,
) -> None:
"""Assigns a role to a user or team, scoped to a specific project.
Args:
role_name_or_id: Name or ID of the role to assign.
user_or_team_name_or_id: Name or ID of the user or team to which to
assign the role.
is_user: Whether `user_or_team_id` refers to a user or a team.
project_name_or_id: Optional Name or ID of a project in which to
assign the role. If this is not provided, the role will be
assigned globally.
"""
copy_local_store(config, path, load_config_path=None)
classmethod
Copy a local store to a new location.
Use this method to create a copy of a store database to a new location and return a new store configuration pointing to the database copy. This only applies to stores that use the local filesystem to store their data. Calling this method for remote stores simply returns the input store configuration unaltered.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The configuration of the store to copy. |
required |
path |
str |
The new local path where the store DB will be copied. |
required |
load_config_path |
Optional[pathlib.PurePath] |
path that will be used to load the copied store
database. This can be set to a value different from |
None |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
The store configuration of the copied store. |
Source code in zenml/zen_stores/rest_zen_store.py
@classmethod
def copy_local_store(
cls,
config: StoreConfiguration,
path: str,
load_config_path: Optional[PurePath] = None,
) -> StoreConfiguration:
"""Copy a local store to a new location.
Use this method to create a copy of a store database to a new location
and return a new store configuration pointing to the database copy. This
only applies to stores that use the local filesystem to store their
data. Calling this method for remote stores simply returns the input
store configuration unaltered.
Args:
config: The configuration of the store to copy.
path: The new local path where the store DB will be copied.
load_config_path: path that will be used to load the copied store
database. This can be set to a value different from `path`
if the local database copy will be loaded from a different
environment, e.g. when the database is copied to a container
image and loaded using a different absolute path. This will be
reflected in the paths and URLs encoded in the copied store
configuration.
Returns:
The store configuration of the copied store.
"""
assert isinstance(config, RestZenStoreConfiguration)
if isinstance(config.verify_ssl, str) and os.path.isfile(
config.verify_ssl
):
config = config.copy(deep=True)
with open(config.verify_ssl, "r") as f:
config.verify_ssl = f.read()
return config
create_flavor(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_pipeline(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_project(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_role(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_stack(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_stack_component(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_team(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_user(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete(self, path, params=None, **kwargs)
Make a DELETE request to the given endpoint path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the endpoint. |
required |
params |
Optional[Dict[str, Any]] |
The query parameters to pass to the endpoint. |
None |
kwargs |
Any |
Additional keyword arguments to pass to the request. |
{} |
Returns:
| Type | Description |
|---|---|
Union[Dict[str, Any], List[Any], str, int, float, bool] |
The response body. |
Source code in zenml/zen_stores/rest_zen_store.py
def delete(
self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
"""Make a DELETE request to the given endpoint path.
Args:
path: The path to the endpoint.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending DELETE request to {path}...")
return self._request(
"DELETE", self.url + API + VERSION_1 + path, params=params, **kwargs
)
delete_flavor(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_pipeline(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_project(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_role(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_stack(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_stack_component(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_team(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_user(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
get(self, path, params=None, **kwargs)
Make a GET request to the given endpoint path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the endpoint. |
required |
params |
Optional[Dict[str, Any]] |
The query parameters to pass to the endpoint. |
None |
kwargs |
Any |
Additional keyword arguments to pass to the request. |
{} |
Returns:
| Type | Description |
|---|---|
Union[Dict[str, Any], List[Any], str, int, float, bool] |
The response body. |
Source code in zenml/zen_stores/rest_zen_store.py
def get(
self, path: str, params: Optional[Dict[str, Any]] = None, **kwargs: Any
) -> Json:
"""Make a GET request to the given endpoint path.
Args:
path: The path to the endpoint.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending GET request to {path}...")
return self._request(
"GET", self.url + API + VERSION_1 + path, params=params, **kwargs
)
get_flavor(self, flavor_id)
Get a stack component flavor by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
UUID |
The ID of the stack component flavor to get. |
required |
Returns:
| Type | Description |
|---|---|
FlavorModel |
The stack component flavor. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_flavor(self, flavor_id: UUID) -> FlavorModel:
"""Get a stack component flavor by ID.
Args:
flavor_id: The ID of the stack component flavor to get.
Returns:
The stack component flavor.
"""
return self._get_resource(
resource_id=flavor_id,
route=FLAVORS,
resource_model=FlavorModel,
)
get_local_url(path)
staticmethod
Get a local URL for a given local path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
the path string to build a URL out of. |
required |
Exceptions:
| Type | Description |
|---|---|
NotImplementedError |
always |
Source code in zenml/zen_stores/rest_zen_store.py
@staticmethod
def get_local_url(path: str) -> str:
"""Get a local URL for a given local path.
Args:
path: the path string to build a URL out of.
Raises:
NotImplementedError: always
"""
raise NotImplementedError("Cannot build a REST url from a path.")
get_metadata_config(self, expand_certs=False)
Get the TFX metadata config of this ZenStore.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
expand_certs |
bool |
Whether to expand the certificate paths in the connection config to their value. |
False |
Exceptions:
| Type | Description |
|---|---|
ValueError |
if the server response is invalid. |
Returns:
| Type | Description |
|---|---|
ConnectionConfig |
The TFX metadata config of this ZenStore. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the TFX metadata config of this ZenStore.
Args:
expand_certs: Whether to expand the certificate paths in the
connection config to their value.
Raises:
ValueError: if the server response is invalid.
Returns:
The TFX metadata config of this ZenStore.
"""
from google.protobuf.json_format import Parse
from ml_metadata.proto.metadata_store_pb2 import ConnectionConfig
body = self.get(f"{METADATA_CONFIG}")
if not isinstance(body, str):
raise ValueError(
f"Invalid response from server: {body}. Expected string."
)
metadata_config_pb = Parse(body, ConnectionConfig())
if not expand_certs:
if metadata_config_pb.HasField(
"mysql"
) and metadata_config_pb.mysql.HasField("ssl_options"):
# Save the certificates in a secure location on disk
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
if not metadata_config_pb.mysql.ssl_options.HasField(
key.lstrip("ssl_")
):
continue
content = getattr(
metadata_config_pb.mysql.ssl_options, key.lstrip("ssl_")
)
if content and not os.path.isfile(content):
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, f"{key}.pem")
with open(file_path, "w") as f:
f.write(content)
file_path.chmod(0o600)
setattr(
metadata_config_pb.mysql.ssl_options,
key.lstrip("ssl_"),
str(file_path),
)
return metadata_config_pb
get_pipeline(self, pipeline_id)
Get a pipeline with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
ID of the pipeline. |
required |
Returns:
| Type | Description |
|---|---|
PipelineModel |
The pipeline. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_pipeline(self, pipeline_id: UUID) -> PipelineModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
"""
return self._get_resource(
resource_id=pipeline_id,
route=PIPELINES,
resource_model=PipelineModel,
)
get_project(self, project_name_or_id)
Get an existing project by name or ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[uuid.UUID, str] |
Name or ID of the project to get. |
required |
Returns:
| Type | Description |
|---|---|
ProjectModel |
The requested project. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_project(self, project_name_or_id: Union[UUID, str]) -> ProjectModel:
"""Get an existing project by name or ID.
Args:
project_name_or_id: Name or ID of the project to get.
Returns:
The requested project.
"""
return self._get_resource(
resource_id=project_name_or_id,
route=PROJECTS,
resource_model=ProjectModel,
)
get_role(self, role_name_or_id)
Gets a specific role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to get. |
required |
Returns:
| Type | Description |
|---|---|
RoleModel |
The requested role. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
"""
return self._get_resource(
resource_id=role_name_or_id,
route=ROLES,
resource_model=RoleModel,
)
get_run(self, run_id)
Gets a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunModel |
The pipeline run. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run(self, run_id: UUID) -> PipelineRunModel:
"""Gets a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
Returns:
The pipeline run.
"""
return self._get_resource(
resource_id=run_id,
route=RUNS,
resource_model=PipelineRunModel,
)
get_run_component_side_effects(self, run_id, component_id=None)
Gets the side effects for a component in a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get. |
required |
component_id |
Optional[uuid.UUID] |
The ID of the component to get. |
None |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run_component_side_effects(
self,
run_id: UUID,
component_id: Optional[UUID] = None,
) -> Dict[str, Any]:
"""Gets the side effects for a component in a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
component_id: The ID of the component to get.
"""
get_run_status(self, run_id)
Gets the execution status of a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get the status for. |
required |
Returns:
| Type | Description |
|---|---|
ExecutionStatus |
The execution status of the pipeline run. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run_status(self, run_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a pipeline run.
Args:
run_id: The ID of the pipeline run to get the status for.
Returns:
The execution status of the pipeline run.
"""
body = self.get(f"{RUNS}/{str(run_id)}{STATUS}")
return ExecutionStatus(body)
get_run_step(self, step_id)
Get a step by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The ID of the step to get. |
required |
Returns:
| Type | Description |
|---|---|
StepRunModel |
The step. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run_step(self, step_id: UUID) -> StepRunModel:
"""Get a step by ID.
Args:
step_id: The ID of the step to get.
Returns:
The step.
"""
return self._get_resource(
resource_id=step_id,
route=STEPS,
resource_model=StepRunModel,
)
get_run_step_inputs(self, step_id)
Get a list of inputs for a specific step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The id of the step to get inputs for. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, zenml.models.pipeline_models.ArtifactModel] |
A dict mapping artifact names to the input artifacts for the step. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
if the response from the API is not a dict. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run_step_inputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get a list of inputs for a specific step.
Args:
step_id: The id of the step to get inputs for.
Returns:
A dict mapping artifact names to the input artifacts for the step.
Raises:
ValueError: if the response from the API is not a dict.
"""
body = self.get(f"{STEPS}/{str(step_id)}{INPUTS}")
if not isinstance(body, dict):
raise ValueError(
f"Bad API Response. Expected dict, got {type(body)}"
)
return {
name: ArtifactModel.parse_obj(entry) for name, entry in body.items()
}
get_run_step_outputs(self, step_id)
Get a list of outputs for a specific step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The id of the step to get outputs for. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, zenml.models.pipeline_models.ArtifactModel] |
A dict mapping artifact names to the output artifacts for the step. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
if the response from the API is not a dict. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run_step_outputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get a list of outputs for a specific step.
Args:
step_id: The id of the step to get outputs for.
Returns:
A dict mapping artifact names to the output artifacts for the step.
Raises:
ValueError: if the response from the API is not a dict.
"""
body = self.get(f"{STEPS}/{str(step_id)}{OUTPUTS}")
if not isinstance(body, dict):
raise ValueError(
f"Bad API Response. Expected dict, got {type(body)}"
)
return {
name: ArtifactModel.parse_obj(entry) for name, entry in body.items()
}
get_run_step_status(self, step_id)
Gets the execution status of a single step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The ID of the step to get the status for. |
required |
Returns:
| Type | Description |
|---|---|
ExecutionStatus |
The execution status of the step. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_run_step_status(self, step_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a single step.
Args:
step_id: The ID of the step to get the status for.
Returns:
The execution status of the step.
"""
body = self.get(f"{STEPS}/{str(step_id)}{STATUS}")
return ExecutionStatus(body)
get_stack(self, stack_id)
Get a stack by its unique ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack to get. |
required |
Returns:
| Type | Description |
|---|---|
StackModel |
The stack with the given ID. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_stack(self, stack_id: UUID) -> StackModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
"""
return self._get_resource(
resource_id=stack_id,
route=STACKS,
resource_model=StackModel,
)
get_stack_component(self, component_id)
Get a stack component by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to get. |
required |
Returns:
| Type | Description |
|---|---|
ComponentModel |
The stack component. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_stack_component(self, component_id: UUID) -> ComponentModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
"""
return self._get_resource(
resource_id=component_id,
route=STACK_COMPONENTS,
resource_model=ComponentModel,
)
get_stack_component_side_effects(self, component_id, run_id, pipeline_id, stack_id)
Get the side effects of a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to get side effects for. |
required |
run_id |
UUID |
The ID of the run to get side effects for. |
required |
pipeline_id |
UUID |
The ID of the pipeline to get side effects for. |
required |
stack_id |
UUID |
The ID of the stack to get side effects for. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def get_stack_component_side_effects(
self,
component_id: UUID,
run_id: UUID,
pipeline_id: UUID,
stack_id: UUID,
) -> Dict[Any, Any]:
"""Get the side effects of a stack component.
Args:
component_id: The ID of the stack component to get side effects for.
run_id: The ID of the run to get side effects for.
pipeline_id: The ID of the pipeline to get side effects for.
stack_id: The ID of the stack to get side effects for.
"""
get_store_info(self)
Get information about the server.
Returns:
| Type | Description |
|---|---|
ServerModel |
Information about the server. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_store_info(self) -> ServerModel:
"""Get information about the server.
Returns:
Information about the server.
"""
body = self.get(INFO)
return ServerModel.parse_obj(body)
get_team(self, team_name_or_id)
Gets a specific team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamModel |
The requested team. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
"""
return self._get_resource(
resource_id=team_name_or_id,
route=TEAMS,
resource_model=TeamModel,
)
get_teams_for_user(self, user_name_or_id)
Fetches all teams for a user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user for which to get all teams. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def get_teams_for_user(
self, user_name_or_id: Union[str, UUID]
) -> List[TeamModel]:
"""Fetches all teams for a user.
Args:
user_name_or_id: The name or ID of the user for which to get all
teams.
"""
get_user(self, user_name_or_id)
Gets a specific user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
required |
Returns:
| Type | Description |
|---|---|
UserModel |
The requested user, if it was found. |
Source code in zenml/zen_stores/rest_zen_store.py
def get_user(self, user_name_or_id: Union[str, UUID]) -> UserModel:
"""Gets a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
"""
return self._get_resource(
resource_id=user_name_or_id,
route=USERS,
resource_model=UserModel,
)
get_users_for_team(self, team_name_or_id)
Fetches all users of a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the team for which to get users. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def get_users_for_team(
self, team_name_or_id: Union[str, UUID]
) -> List[UserModel]:
"""Fetches all users of a team.
Args:
team_name_or_id: The name or ID of the team for which to get users.
"""
list_artifacts(self, artifact_uri=None)
Lists all artifacts.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_uri |
Optional[str] |
If specified, only artifacts with the given URI will be returned. |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.ArtifactModel] |
A list of all artifacts. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_artifacts(
self, artifact_uri: Optional[str] = None
) -> List[ArtifactModel]:
"""Lists all artifacts.
Args:
artifact_uri: If specified, only artifacts with the given URI will
be returned.
Returns:
A list of all artifacts.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=ARTIFACTS,
resource_model=ArtifactModel,
**filters,
)
list_flavors(self, project_name_or_id=None, user_name_or_id=None, component_type=None, name=None, is_shared=None)
List all stack component flavors matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Optionally filter by the Project to which the component flavors belong |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter by the owner |
None |
component_type |
Optional[zenml.enums.StackComponentType] |
Optionally filter by type of stack component |
None |
name |
Optional[str] |
Optionally filter flavors by name |
None |
is_shared |
Optional[bool] |
Optionally filter out flavors by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.flavor_models.FlavorModel] |
List of all the stack component flavors matching the given criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_flavors(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_type: Optional[StackComponentType] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[FlavorModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
project_name_or_id: Optionally filter by the Project to which the
component flavors belong
user_name_or_id: Optionally filter by the owner
component_type: Optionally filter by type of stack component
name: Optionally filter flavors by name
is_shared: Optionally filter out flavors by whether they are
shared or not
Returns:
List of all the stack component flavors matching the given criteria.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=FLAVORS,
resource_model=FlavorModel,
**filters,
)
list_pipelines(self, project_name_or_id=None, user_name_or_id=None, name=None)
List all pipelines in the project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only list pipelines in this project. |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only list pipelines from this user. |
None |
name |
Optional[str] |
If provided, only list pipelines with this name. |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.PipelineModel] |
A list of pipelines. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_pipelines(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
name: Optional[str] = None,
) -> List[PipelineModel]:
"""List all pipelines in the project.
Args:
project_name_or_id: If provided, only list pipelines in this project.
user_name_or_id: If provided, only list pipelines from this user.
name: If provided, only list pipelines with this name.
Returns:
A list of pipelines.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=PIPELINES,
resource_model=PipelineModel,
**filters,
)
list_projects(self)
List all projects.
Returns:
| Type | Description |
|---|---|
List[zenml.models.project_models.ProjectModel] |
A list of all projects. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_projects(self) -> List[ProjectModel]:
"""List all projects.
Returns:
A list of all projects.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=PROJECTS,
resource_model=ProjectModel,
**filters,
)
list_role_assignments(self, project_name_or_id=None, team_name_or_id=None, user_name_or_id=None)
List all role assignments.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only list assignments for the given project |
None |
team_name_or_id |
Union[str, uuid.UUID] |
If provided, only list assignments for the given team |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only list assignments for the given user |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.RoleAssignmentModel] |
A list of all role assignments. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_role_assignments(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
team_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
) -> List[RoleAssignmentModel]:
"""List all role assignments.
Args:
project_name_or_id: If provided, only list assignments for the given
project
team_name_or_id: If provided, only list assignments for the given
team
user_name_or_id: If provided, only list assignments for the given
user
Returns:
A list of all role assignments.
"""
roles: List[RoleAssignmentModel] = []
if user_name_or_id:
roles.extend(
self._list_resources(
route=f"{USERS}/{user_name_or_id}{ROLES}",
resource_model=RoleAssignmentModel,
project_name_or_id=project_name_or_id,
)
)
if team_name_or_id:
roles.extend(
self._list_resources(
route=f"{TEAMS}/{team_name_or_id}{ROLES}",
resource_model=RoleAssignmentModel,
project_name_or_id=project_name_or_id,
)
)
return roles
list_roles(self)
List all roles.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.RoleModel] |
A list of all roles. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_roles(self) -> List[RoleModel]:
"""List all roles.
Returns:
A list of all roles.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=ROLES,
resource_model=RoleModel,
**filters,
)
list_run_steps(self, run_id)
Gets all steps in a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run for which to list runs. |
required |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.StepRunModel] |
A mapping from step names to step models for all steps in the run. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_run_steps(self, run_id: UUID) -> List[StepRunModel]:
"""Gets all steps in a pipeline run.
Args:
run_id: The ID of the pipeline run for which to list runs.
Returns:
A mapping from step names to step models for all steps in the run.
"""
return self._list_resources(
route=f"{RUNS}/{str(run_id)}{STEPS}",
resource_model=StepRunModel,
)
list_runs(self, project_name_or_id=None, stack_id=None, component_id=None, run_name=None, user_name_or_id=None, pipeline_id=None, unlisted=False)
Gets all pipeline runs.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only return runs for this project. |
None |
stack_id |
Optional[uuid.UUID] |
If provided, only return runs for this stack. |
None |
component_id |
Optional[uuid.UUID] |
Optionally filter for runs that used the component |
None |
run_name |
Optional[str] |
Run name if provided |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only return runs for this user. |
None |
pipeline_id |
Optional[uuid.UUID] |
If provided, only return runs for this pipeline. |
None |
unlisted |
bool |
If True, only return unlisted runs that are not
associated with any pipeline (filter by |
False |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.PipelineRunModel] |
A list of all pipeline runs. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_runs(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
stack_id: Optional[UUID] = None,
component_id: Optional[UUID] = None,
run_name: Optional[str] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
pipeline_id: Optional[UUID] = None,
unlisted: bool = False,
) -> List[PipelineRunModel]:
"""Gets all pipeline runs.
Args:
project_name_or_id: If provided, only return runs for this project.
stack_id: If provided, only return runs for this stack.
component_id: Optionally filter for runs that used the
component
run_name: Run name if provided
user_name_or_id: If provided, only return runs for this user.
pipeline_id: If provided, only return runs for this pipeline.
unlisted: If True, only return unlisted runs that are not
associated with any pipeline (filter by `pipeline_id==None`).
Returns:
A list of all pipeline runs.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=RUNS,
resource_model=PipelineRunModel,
**filters,
)
list_stack_components(self, project_name_or_id=None, user_name_or_id=None, type=None, flavor_name=None, name=None, is_shared=None)
List all stack components matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
The ID or name of the Project to which the stack components belong |
None |
type |
Optional[str] |
Optionally filter by type of stack component |
None |
flavor_name |
Optional[str] |
Optionally filter by flavor |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter stack components by the owner |
None |
name |
Optional[str] |
Optionally filter stack component by name |
None |
is_shared |
Optional[bool] |
Optionally filter out stack component by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.component_model.ComponentModel] |
A list of all stack components matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_stack_components(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
type: Optional[str] = None,
flavor_name: Optional[str] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[ComponentModel]:
"""List all stack components matching the given filter criteria.
Args:
project_name_or_id: The ID or name of the Project to which the stack
components belong
type: Optionally filter by type of stack component
flavor_name: Optionally filter by flavor
user_name_or_id: Optionally filter stack components by the owner
name: Optionally filter stack component by name
is_shared: Optionally filter out stack component by whether they are
shared or not
Returns:
A list of all stack components matching the filter criteria.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=STACK_COMPONENTS,
resource_model=ComponentModel,
**filters,
)
list_stacks(self, project_name_or_id=None, user_name_or_id=None, component_id=None, name=None, is_shared=None)
List all stacks matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Id or name of the Project containing the stack |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter stacks by their owner |
None |
component_id |
Optional[uuid.UUID] |
Optionally filter for stacks that contain the component |
None |
name |
Optional[str] |
Optionally filter stacks by their name |
None |
is_shared |
Optional[bool] |
Optionally filter out stacks by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.stack_models.StackModel] |
A list of all stacks matching the filter criteria. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_stacks(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_id: Optional[UUID] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[StackModel]:
"""List all stacks matching the given filter criteria.
Args:
project_name_or_id: Id or name of the Project containing the stack
user_name_or_id: Optionally filter stacks by their owner
component_id: Optionally filter for stacks that contain the
component
name: Optionally filter stacks by their name
is_shared: Optionally filter out stacks by whether they are shared
or not
Returns:
A list of all stacks matching the filter criteria.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=STACKS,
resource_model=StackModel,
**filters,
)
list_steps(self, pipeline_id)
List all steps.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
The ID of the pipeline to list steps for. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def list_steps(self, pipeline_id: UUID) -> List[StepRunModel]:
"""List all steps.
Args:
pipeline_id: The ID of the pipeline to list steps for.
"""
list_teams(self)
List all teams.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.TeamModel] |
A list of all teams. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_teams(self) -> List[TeamModel]:
"""List all teams.
Returns:
A list of all teams.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=TEAMS,
resource_model=TeamModel,
**filters,
)
list_users(self)
List all users.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.UserModel] |
A list of all users. |
Source code in zenml/zen_stores/rest_zen_store.py
def list_users(self) -> List[UserModel]:
"""List all users.
Returns:
A list of all users.
"""
filters = locals()
filters.pop("self")
return self._list_resources(
route=USERS,
resource_model=UserModel,
**filters,
)
post(self, path, body, params=None, **kwargs)
Make a POST request to the given endpoint path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the endpoint. |
required |
body |
BaseModel |
The body to send. |
required |
params |
Optional[Dict[str, Any]] |
The query parameters to pass to the endpoint. |
None |
kwargs |
Any |
Additional keyword arguments to pass to the request. |
{} |
Returns:
| Type | Description |
|---|---|
Union[Dict[str, Any], List[Any], str, int, float, bool] |
The response body. |
Source code in zenml/zen_stores/rest_zen_store.py
def post(
self,
path: str,
body: BaseModel,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a POST request to the given endpoint path.
Args:
path: The path to the endpoint.
body: The body to send.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending POST request to {path}...")
return self._request(
"POST",
self.url + API + VERSION_1 + path,
data=body.json(),
params=params,
**kwargs,
)
put(self, path, body, params=None, **kwargs)
Make a PUT request to the given endpoint path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the endpoint. |
required |
body |
BaseModel |
The body to send. |
required |
params |
Optional[Dict[str, Any]] |
The query parameters to pass to the endpoint. |
None |
kwargs |
Any |
Additional keyword arguments to pass to the request. |
{} |
Returns:
| Type | Description |
|---|---|
Union[Dict[str, Any], List[Any], str, int, float, bool] |
The response body. |
Source code in zenml/zen_stores/rest_zen_store.py
def put(
self,
path: str,
body: BaseModel,
params: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Json:
"""Make a PUT request to the given endpoint path.
Args:
path: The path to the endpoint.
body: The body to send.
params: The query parameters to pass to the endpoint.
kwargs: Additional keyword arguments to pass to the request.
Returns:
The response body.
"""
logger.debug(f"Sending PUT request to {path}...")
return self._request(
"PUT",
self.url + API + VERSION_1 + path,
data=body.json(),
params=params,
**kwargs,
)
remove_user_from_team(self, user_name_or_id, team_name_or_id)
Removes a user from a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user to remove from the team. |
required |
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team from which to remove the user. |
required |
Source code in zenml/zen_stores/rest_zen_store.py
def remove_user_from_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Removes a user from a team.
Args:
user_name_or_id: Name or ID of the user to remove from the team.
team_name_or_id: Name or ID of the team from which to remove the user.
"""
revoke_role(self, role_name_or_id, user_or_team_name_or_id, is_user=True, project_name_or_id=None)
Revokes a role from a user or team for a given project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
ID of the role to revoke. |
required |
user_or_team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user or team from which to revoke the role. |
required |
is_user |
bool |
Whether |
True |
project_name_or_id |
Union[str, uuid.UUID] |
Optional ID of a project in which to revoke the role. If this is not provided, the role will be revoked globally. |
None |
Source code in zenml/zen_stores/rest_zen_store.py
def revoke_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
is_user: bool = True,
project_name_or_id: Optional[Union[str, UUID]] = None,
) -> None:
"""Revokes a role from a user or team for a given project.
Args:
role_name_or_id: ID of the role to revoke.
user_or_team_name_or_id: Name or ID of the user or team from which
to revoke the role.
is_user: Whether `user_or_team_id` refers to a user or a team.
project_name_or_id: Optional ID of a project in which to revoke
the role. If this is not provided, the role will be revoked
globally.
"""
update_flavor(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_pipeline(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_project(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_role(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_stack(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_stack_component(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_team(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_user(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/rest_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
user_email_opt_in(self, user_name_or_id, user_opt_in_response, email=None)
Persist user response to the email prompt.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or the ID of the user. |
required |
user_opt_in_response |
bool |
Whether this email should be associated with the user id in the telemetry |
required |
email |
Optional[str] |
The users email |
None |
Returns:
| Type | Description |
|---|---|
UserModel |
The updated user. |
Source code in zenml/zen_stores/rest_zen_store.py
def user_email_opt_in(
self,
user_name_or_id: Union[str, UUID],
user_opt_in_response: bool,
email: Optional[str] = None,
) -> UserModel:
"""Persist user response to the email prompt.
Args:
user_name_or_id: The name or the ID of the user.
user_opt_in_response: Whether this email should be associated
with the user id in the telemetry
email: The users email
Returns:
The updated user.
"""
request = EmailOptInModel(
email=email, email_opted_in=user_opt_in_response
)
route = f"{USERS}/{str(user_name_or_id)}{EMAIL_ANALYTICS}"
response_body = self.put(route, body=request)
user = UserModel.parse_obj(response_body)
return user
RestZenStoreConfiguration (StoreConfiguration)
pydantic-model
REST ZenML store configuration.
Attributes:
| Name | Type | Description |
|---|---|---|
username |
str |
The username to use to connect to the Zen server. |
password |
str |
The password to use to connect to the Zen server. |
verify_ssl |
Union[bool, str] |
Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use or the CA bundle value itself. |
http_timeout |
int |
The timeout to use for all requests. |
Source code in zenml/zen_stores/rest_zen_store.py
class RestZenStoreConfiguration(StoreConfiguration):
"""REST ZenML store configuration.
Attributes:
username: The username to use to connect to the Zen server.
password: The password to use to connect to the Zen server.
verify_ssl: Either a boolean, in which case it controls whether we
verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use or the CA bundle value itself.
http_timeout: The timeout to use for all requests.
"""
type: StoreType = StoreType.REST
username: str
password: str = ""
verify_ssl: Union[bool, str] = True
http_timeout: int = DEFAULT_HTTP_TIMEOUT
@validator("url")
def validate_url(cls, url: str) -> str:
"""Validates that the URL is a well formed REST store URL.
Args:
url: The URL to be validated.
Returns:
The validated URL without trailing slashes.
Raises:
ValueError: If the URL is not a well formed REST store URL.
"""
url = url.rstrip("/")
scheme = re.search("^([a-z0-9]+://)", url)
if scheme is None or scheme.group() not in ("https://", "http://"):
raise ValueError(
"Invalid URL for REST store: {url}. Should be in the form "
"https://hostname[:port] or http://hostname[:port]."
)
return url
@validator("verify_ssl")
def validate_verify_ssl(
cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
"""Validates that the verify_ssl field either points to a file or is a bool.
Args:
verify_ssl: The verify_ssl value to be validated.
Returns:
The validated verify_ssl value.
"""
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
if isinstance(verify_ssl, bool) or verify_ssl.startswith(
str(secret_folder)
):
return verify_ssl
if os.path.isfile(verify_ssl):
with open(verify_ssl, "r") as f:
verify_ssl = f.read()
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, "ca_bundle.pem")
with open(file_path, "w") as f:
f.write(verify_ssl)
file_path.chmod(0o600)
verify_ssl = str(file_path)
return verify_ssl
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the `verify_ssl` attribute can be expanded to the contents
# of the certificate file.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
Config
Pydantic configuration class.
Source code in zenml/zen_stores/rest_zen_store.py
class Config:
"""Pydantic configuration class."""
# Don't validate attributes when assigning them. This is necessary
# because the `verify_ssl` attribute can be expanded to the contents
# of the certificate file.
validate_assignment = False
# Forbid extra attributes set in the class.
extra = "forbid"
validate_url(url)
classmethod
Validates that the URL is a well formed REST store URL.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
url |
str |
The URL to be validated. |
required |
Returns:
| Type | Description |
|---|---|
str |
The validated URL without trailing slashes. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the URL is not a well formed REST store URL. |
Source code in zenml/zen_stores/rest_zen_store.py
@validator("url")
def validate_url(cls, url: str) -> str:
"""Validates that the URL is a well formed REST store URL.
Args:
url: The URL to be validated.
Returns:
The validated URL without trailing slashes.
Raises:
ValueError: If the URL is not a well formed REST store URL.
"""
url = url.rstrip("/")
scheme = re.search("^([a-z0-9]+://)", url)
if scheme is None or scheme.group() not in ("https://", "http://"):
raise ValueError(
"Invalid URL for REST store: {url}. Should be in the form "
"https://hostname[:port] or http://hostname[:port]."
)
return url
validate_verify_ssl(verify_ssl)
classmethod
Validates that the verify_ssl field either points to a file or is a bool.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
verify_ssl |
Union[bool, str] |
The verify_ssl value to be validated. |
required |
Returns:
| Type | Description |
|---|---|
Union[bool, str] |
The validated verify_ssl value. |
Source code in zenml/zen_stores/rest_zen_store.py
@validator("verify_ssl")
def validate_verify_ssl(
cls, verify_ssl: Union[bool, str]
) -> Union[bool, str]:
"""Validates that the verify_ssl field either points to a file or is a bool.
Args:
verify_ssl: The verify_ssl value to be validated.
Returns:
The validated verify_ssl value.
"""
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
if isinstance(verify_ssl, bool) or verify_ssl.startswith(
str(secret_folder)
):
return verify_ssl
if os.path.isfile(verify_ssl):
with open(verify_ssl, "r") as f:
verify_ssl = f.read()
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, "ca_bundle.pem")
with open(file_path, "w") as f:
f.write(verify_ssl)
file_path.chmod(0o600)
verify_ssl = str(file_path)
return verify_ssl
schemas
special
SQL Model Implementations.
component_schemas
SQL Model Implementations for Stack Components.
StackComponentSchema (SQLModel)
pydantic-model
SQL Model for stack components.
Source code in zenml/zen_stores/schemas/component_schemas.py
class StackComponentSchema(SQLModel, table=True):
"""SQL Model for stack components."""
id: UUID = Field(primary_key=True)
name: str
is_shared: bool
type: StackComponentType
flavor: str
project_id: UUID = Field(
sa_column=Column(ForeignKey("projectschema.id", ondelete="CASCADE"))
)
project: "ProjectSchema" = Relationship(back_populates="components")
user_id: UUID = Field(
sa_column=Column(ForeignKey("userschema.id", ondelete="SET NULL"))
)
user: "UserSchema" = Relationship(back_populates="components")
configuration: bytes
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
stacks: List["StackSchema"] = Relationship(
back_populates="components", link_model=StackCompositionSchema
)
@classmethod
def from_create_model(
cls, component: ComponentModel
) -> "StackComponentSchema":
"""Create a `StackComponentSchema`.
Args:
component: The component model from which to create the schema.
Returns:
The created `StackComponentSchema`.
"""
return cls(
id=component.id,
name=component.name,
project_id=component.project,
user_id=component.user,
is_shared=component.is_shared,
type=component.type,
flavor=component.flavor,
configuration=base64.b64encode(
json.dumps(component.configuration).encode("utf-8")
),
created=component.created,
updated=component.updated,
)
def from_update_model(
self,
component: ComponentModel,
) -> "StackComponentSchema":
"""Update the updatable fields on an existing `StackSchema`.
Args:
component: The component model from which to update the schema.
Returns:
A `StackSchema`
"""
self.name = component.name
self.is_shared = component.is_shared
self.configuration = base64.b64encode(
json.dumps(component.configuration).encode("utf-8")
)
return self
def to_model(self) -> "ComponentModel":
"""Creates a `ComponentModel` from an instance of a `StackSchema`.
Returns:
A `ComponentModel`
"""
return ComponentModel(
id=self.id,
name=self.name,
type=self.type,
flavor=self.flavor,
user=self.user_id,
project=self.project_id,
is_shared=self.is_shared,
configuration=json.loads(
base64.b64decode(self.configuration).decode()
),
created=self.created,
updated=self.updated,
)
from_create_model(component)
classmethod
Create a StackComponentSchema.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
ComponentModel |
The component model from which to create the schema. |
required |
Returns:
| Type | Description |
|---|---|
StackComponentSchema |
The created |
Source code in zenml/zen_stores/schemas/component_schemas.py
@classmethod
def from_create_model(
cls, component: ComponentModel
) -> "StackComponentSchema":
"""Create a `StackComponentSchema`.
Args:
component: The component model from which to create the schema.
Returns:
The created `StackComponentSchema`.
"""
return cls(
id=component.id,
name=component.name,
project_id=component.project,
user_id=component.user,
is_shared=component.is_shared,
type=component.type,
flavor=component.flavor,
configuration=base64.b64encode(
json.dumps(component.configuration).encode("utf-8")
),
created=component.created,
updated=component.updated,
)
from_update_model(self, component)
Update the updatable fields on an existing StackSchema.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
ComponentModel |
The component model from which to update the schema. |
required |
Returns:
| Type | Description |
|---|---|
StackComponentSchema |
A |
Source code in zenml/zen_stores/schemas/component_schemas.py
def from_update_model(
self,
component: ComponentModel,
) -> "StackComponentSchema":
"""Update the updatable fields on an existing `StackSchema`.
Args:
component: The component model from which to update the schema.
Returns:
A `StackSchema`
"""
self.name = component.name
self.is_shared = component.is_shared
self.configuration = base64.b64encode(
json.dumps(component.configuration).encode("utf-8")
)
return self
to_model(self)
Creates a ComponentModel from an instance of a StackSchema.
Returns:
| Type | Description |
|---|---|
ComponentModel |
A |
Source code in zenml/zen_stores/schemas/component_schemas.py
def to_model(self) -> "ComponentModel":
"""Creates a `ComponentModel` from an instance of a `StackSchema`.
Returns:
A `ComponentModel`
"""
return ComponentModel(
id=self.id,
name=self.name,
type=self.type,
flavor=self.flavor,
user=self.user_id,
project=self.project_id,
is_shared=self.is_shared,
configuration=json.loads(
base64.b64decode(self.configuration).decode()
),
created=self.created,
updated=self.updated,
)
flavor_schemas
SQL Model Implementations for Flavors.
FlavorSchema (SQLModel)
pydantic-model
SQL Model for flavors.
Attributes:
| Name | Type | Description |
|---|---|---|
id |
The unique id of the flavor. |
|
name |
The name of the flavor. |
|
type |
The type of the flavor. |
|
source |
The source of the flavor. |
|
config_schema |
The config schema of the flavor. |
|
integration |
The integration associated with the flavor. |
|
user_id |
The user associated with the flavor. |
|
project_id |
The project associated with the flavor. |
|
created |
The creation time of the flavor. |
|
updated |
The last update time of the flavor. |
Source code in zenml/zen_stores/schemas/flavor_schemas.py
class FlavorSchema(SQLModel, table=True):
"""SQL Model for flavors.
Attributes:
id: The unique id of the flavor.
name: The name of the flavor.
type: The type of the flavor.
source: The source of the flavor.
config_schema: The config schema of the flavor.
integration: The integration associated with the flavor.
user_id: The user associated with the flavor.
project_id: The project associated with the flavor.
created: The creation time of the flavor.
updated: The last update time of the flavor.
"""
id: UUID = Field(primary_key=True)
type: StackComponentType
source: str
name: str
integration: Optional[str] = Field(default="")
config_schema: str
project_id: UUID = Field(
sa_column=Column(ForeignKey("projectschema.id", ondelete="CASCADE"))
)
project: "ProjectSchema" = Relationship(back_populates="flavors")
user_id: UUID = Field(
sa_column=Column(ForeignKey("userschema.id", ondelete="SET NULL"))
)
user: "UserSchema" = Relationship(back_populates="flavors")
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
@classmethod
def from_create_model(cls, flavor: FlavorModel) -> "FlavorSchema":
"""Returns a flavor schema from a flavor model.
Args:
flavor: The flavor model.
Returns:
The flavor schema.
"""
return cls(
id=flavor.id,
name=flavor.name,
type=flavor.type,
source=flavor.source,
config_schema=flavor.config_schema,
integration=flavor.integration,
user_id=flavor.user,
project_id=flavor.project,
)
def from_update_model(
self,
flavor: FlavorModel,
) -> "FlavorSchema":
"""Returns a flavor schema from a flavor model.
Args:
flavor: The flavor model.
Returns:
The flavor schema.
"""
return self
def to_model(self) -> FlavorModel:
"""Converts a flavor schema to a flavor model.
Returns:
The flavor model.
"""
return FlavorModel(
id=self.id,
name=self.name,
type=self.type,
source=self.source,
config_schema=self.config_schema,
integration=self.integration,
user=self.user_id,
project=self.project_id,
created=self.created,
updated=self.updated,
)
from_create_model(flavor)
classmethod
Returns a flavor schema from a flavor model.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor |
FlavorModel |
The flavor model. |
required |
Returns:
| Type | Description |
|---|---|
FlavorSchema |
The flavor schema. |
Source code in zenml/zen_stores/schemas/flavor_schemas.py
@classmethod
def from_create_model(cls, flavor: FlavorModel) -> "FlavorSchema":
"""Returns a flavor schema from a flavor model.
Args:
flavor: The flavor model.
Returns:
The flavor schema.
"""
return cls(
id=flavor.id,
name=flavor.name,
type=flavor.type,
source=flavor.source,
config_schema=flavor.config_schema,
integration=flavor.integration,
user_id=flavor.user,
project_id=flavor.project,
)
from_update_model(self, flavor)
Returns a flavor schema from a flavor model.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor |
FlavorModel |
The flavor model. |
required |
Returns:
| Type | Description |
|---|---|
FlavorSchema |
The flavor schema. |
Source code in zenml/zen_stores/schemas/flavor_schemas.py
def from_update_model(
self,
flavor: FlavorModel,
) -> "FlavorSchema":
"""Returns a flavor schema from a flavor model.
Args:
flavor: The flavor model.
Returns:
The flavor schema.
"""
return self
to_model(self)
Converts a flavor schema to a flavor model.
Returns:
| Type | Description |
|---|---|
FlavorModel |
The flavor model. |
Source code in zenml/zen_stores/schemas/flavor_schemas.py
def to_model(self) -> FlavorModel:
"""Converts a flavor schema to a flavor model.
Returns:
The flavor model.
"""
return FlavorModel(
id=self.id,
name=self.name,
type=self.type,
source=self.source,
config_schema=self.config_schema,
integration=self.integration,
user=self.user_id,
project=self.project_id,
created=self.created,
updated=self.updated,
)
pipeline_schemas
SQL Model Implementations for Pipelines and Pipeline Runs.
ArtifactSchema (SQLModel)
pydantic-model
SQL Model for artifacts of steps.
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
class ArtifactSchema(SQLModel, table=True):
"""SQL Model for artifacts of steps."""
id: UUID = Field(primary_key=True)
name: str # Name of the output in the parent step
parent_step_id: UUID = Field(foreign_key="steprunschema.id")
producer_step_id: UUID = Field(foreign_key="steprunschema.id")
type: ArtifactType
uri: str
materializer: str
data_type: str
is_cached: bool
mlmd_id: int = Field(default=None, nullable=True)
mlmd_parent_step_id: int = Field(default=None, nullable=True)
mlmd_producer_step_id: int = Field(default=None, nullable=True)
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
@classmethod
def from_create_model(cls, model: ArtifactModel) -> "ArtifactSchema":
"""Create an `ArtifactSchema` from an `ArtifactModel`.
Args:
model: The `ArtifactModel` to create the schema from.
Returns:
The created `ArtifactSchema`.
"""
return cls(
id=model.id,
name=model.name,
parent_step_id=model.parent_step_id,
producer_step_id=model.producer_step_id,
type=model.type,
uri=model.uri,
materializer=model.materializer,
data_type=model.data_type,
is_cached=model.is_cached,
mlmd_id=model.mlmd_id,
mlmd_parent_step_id=model.mlmd_parent_step_id,
mlmd_producer_step_id=model.mlmd_producer_step_id,
)
def to_model(self) -> ArtifactModel:
"""Convert an `ArtifactSchema` to an `ArtifactModel`.
Returns:
The created `ArtifactModel`.
"""
return ArtifactModel(
id=self.id,
name=self.name,
parent_step_id=self.parent_step_id,
producer_step_id=self.producer_step_id,
type=self.type,
uri=self.uri,
materializer=self.materializer,
data_type=self.data_type,
is_cached=self.is_cached,
mlmd_id=self.mlmd_id,
mlmd_parent_step_id=self.mlmd_parent_step_id,
mlmd_producer_step_id=self.mlmd_producer_step_id,
created=self.created,
updated=self.updated,
)
from_create_model(model)
classmethod
Create an ArtifactSchema from an ArtifactModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
ArtifactModel |
The |
required |
Returns:
| Type | Description |
|---|---|
ArtifactSchema |
The created |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
@classmethod
def from_create_model(cls, model: ArtifactModel) -> "ArtifactSchema":
"""Create an `ArtifactSchema` from an `ArtifactModel`.
Args:
model: The `ArtifactModel` to create the schema from.
Returns:
The created `ArtifactSchema`.
"""
return cls(
id=model.id,
name=model.name,
parent_step_id=model.parent_step_id,
producer_step_id=model.producer_step_id,
type=model.type,
uri=model.uri,
materializer=model.materializer,
data_type=model.data_type,
is_cached=model.is_cached,
mlmd_id=model.mlmd_id,
mlmd_parent_step_id=model.mlmd_parent_step_id,
mlmd_producer_step_id=model.mlmd_producer_step_id,
)
to_model(self)
Convert an ArtifactSchema to an ArtifactModel.
Returns:
| Type | Description |
|---|---|
ArtifactModel |
The created |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def to_model(self) -> ArtifactModel:
"""Convert an `ArtifactSchema` to an `ArtifactModel`.
Returns:
The created `ArtifactModel`.
"""
return ArtifactModel(
id=self.id,
name=self.name,
parent_step_id=self.parent_step_id,
producer_step_id=self.producer_step_id,
type=self.type,
uri=self.uri,
materializer=self.materializer,
data_type=self.data_type,
is_cached=self.is_cached,
mlmd_id=self.mlmd_id,
mlmd_parent_step_id=self.mlmd_parent_step_id,
mlmd_producer_step_id=self.mlmd_producer_step_id,
created=self.created,
updated=self.updated,
)
PipelineRunSchema (SQLModel)
pydantic-model
SQL Model for pipeline runs.
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
class PipelineRunSchema(SQLModel, table=True):
"""SQL Model for pipeline runs."""
id: UUID = Field(primary_key=True)
name: str
project_id: UUID = Field(
sa_column=Column(ForeignKey("projectschema.id", ondelete="CASCADE"))
)
project: "ProjectSchema" = Relationship(back_populates="runs")
user_id: UUID = Field(
nullable=False,
sa_column=Column(ForeignKey("userschema.id", ondelete="CASCADE")),
)
user: "UserSchema" = Relationship(back_populates="runs")
stack_id: Optional[UUID] = Field(
nullable=True,
sa_column=Column(ForeignKey("stackschema.id", ondelete="SET NULL")),
)
stack: "StackSchema" = Relationship(back_populates="runs")
pipeline_id: Optional[UUID] = Field(
nullable=True,
sa_column=Column(ForeignKey("pipelineschema.id", ondelete="SET NULL")),
)
pipeline: PipelineSchema = Relationship(back_populates="runs")
pipeline_configuration: str = Field(max_length=4096)
num_steps: int
zenml_version: str
git_sha: Optional[str] = Field(nullable=True)
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
mlmd_id: int = Field(default=None, nullable=True)
@classmethod
def from_create_model(
cls,
run: PipelineRunModel,
pipeline: Optional[PipelineSchema] = None,
) -> "PipelineRunSchema":
"""Create a `PipelineRunSchema` from a `PipelineRunModel`.
Args:
run: The `PipelineRunModel` to create the schema from.
pipeline: The `PipelineSchema` to link to the run.
Returns:
The created `PipelineRunSchema`.
"""
return cls(
id=run.id,
name=run.name,
stack_id=run.stack_id,
project_id=run.project,
user_id=run.user,
pipeline_id=run.pipeline_id,
pipeline_configuration=json.dumps(run.pipeline_configuration),
num_steps=run.num_steps,
git_sha=run.git_sha,
zenml_version=run.zenml_version,
pipeline=pipeline,
mlmd_id=run.mlmd_id,
)
def from_update_model(self, model: PipelineRunModel) -> "PipelineRunSchema":
"""Update a `PipelineRunSchema` from a `PipelineRunModel`.
Args:
model: The `PipelineRunModel` to update the schema from.
Returns:
The updated `PipelineRunSchema`.
"""
self.name = model.name
self.git_sha = model.git_sha
if model.zenml_version is not None:
self.zenml_version = model.zenml_version
if model.mlmd_id is not None:
self.mlmd_id = model.mlmd_id
self.updated = datetime.now()
return self
def to_model(self) -> PipelineRunModel:
"""Convert a `PipelineRunSchema` to a `PipelineRunModel`.
Returns:
The created `PipelineRunModel`.
"""
return PipelineRunModel(
id=self.id,
name=self.name,
stack_id=self.stack_id,
project=self.project_id,
user=self.user_id,
pipeline_id=self.pipeline_id,
pipeline_configuration=json.loads(self.pipeline_configuration),
num_steps=self.num_steps,
git_sha=self.git_sha,
zenml_version=self.zenml_version,
mlmd_id=self.mlmd_id,
created=self.created,
updated=self.updated,
)
from_create_model(run, pipeline=None)
classmethod
Create a PipelineRunSchema from a PipelineRunModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run |
PipelineRunModel |
The |
required |
pipeline |
Optional[zenml.zen_stores.schemas.pipeline_schemas.PipelineSchema] |
The |
None |
Returns:
| Type | Description |
|---|---|
PipelineRunSchema |
The created |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
@classmethod
def from_create_model(
cls,
run: PipelineRunModel,
pipeline: Optional[PipelineSchema] = None,
) -> "PipelineRunSchema":
"""Create a `PipelineRunSchema` from a `PipelineRunModel`.
Args:
run: The `PipelineRunModel` to create the schema from.
pipeline: The `PipelineSchema` to link to the run.
Returns:
The created `PipelineRunSchema`.
"""
return cls(
id=run.id,
name=run.name,
stack_id=run.stack_id,
project_id=run.project,
user_id=run.user,
pipeline_id=run.pipeline_id,
pipeline_configuration=json.dumps(run.pipeline_configuration),
num_steps=run.num_steps,
git_sha=run.git_sha,
zenml_version=run.zenml_version,
pipeline=pipeline,
mlmd_id=run.mlmd_id,
)
from_update_model(self, model)
Update a PipelineRunSchema from a PipelineRunModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
PipelineRunModel |
The |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunSchema |
The updated |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def from_update_model(self, model: PipelineRunModel) -> "PipelineRunSchema":
"""Update a `PipelineRunSchema` from a `PipelineRunModel`.
Args:
model: The `PipelineRunModel` to update the schema from.
Returns:
The updated `PipelineRunSchema`.
"""
self.name = model.name
self.git_sha = model.git_sha
if model.zenml_version is not None:
self.zenml_version = model.zenml_version
if model.mlmd_id is not None:
self.mlmd_id = model.mlmd_id
self.updated = datetime.now()
return self
to_model(self)
Convert a PipelineRunSchema to a PipelineRunModel.
Returns:
| Type | Description |
|---|---|
PipelineRunModel |
The created |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def to_model(self) -> PipelineRunModel:
"""Convert a `PipelineRunSchema` to a `PipelineRunModel`.
Returns:
The created `PipelineRunModel`.
"""
return PipelineRunModel(
id=self.id,
name=self.name,
stack_id=self.stack_id,
project=self.project_id,
user=self.user_id,
pipeline_id=self.pipeline_id,
pipeline_configuration=json.loads(self.pipeline_configuration),
num_steps=self.num_steps,
git_sha=self.git_sha,
zenml_version=self.zenml_version,
mlmd_id=self.mlmd_id,
created=self.created,
updated=self.updated,
)
PipelineSchema (SQLModel)
pydantic-model
SQL Model for pipelines.
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
class PipelineSchema(SQLModel, table=True):
"""SQL Model for pipelines."""
id: UUID = Field(primary_key=True)
name: str
project_id: UUID = Field(
sa_column=Column(ForeignKey("projectschema.id", ondelete="CASCADE"))
)
project: "ProjectSchema" = Relationship(back_populates="pipelines")
user_id: UUID = Field(
sa_column=Column(ForeignKey("userschema.id", ondelete="SET NULL"))
)
user: "UserSchema" = Relationship(back_populates="pipelines")
docstring: Optional[str] = Field(max_length=4096, nullable=True)
spec: str = Field(max_length=4096)
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
runs: List["PipelineRunSchema"] = Relationship(
back_populates="pipeline",
)
@classmethod
def from_create_model(cls, pipeline: PipelineModel) -> "PipelineSchema":
"""Create a `PipelineSchema` from a `PipelineModel`.
Args:
pipeline: The `PipelineModel` to create the schema from.
Returns:
The created `PipelineSchema`.
"""
return cls(
id=pipeline.id,
name=pipeline.name,
project_id=pipeline.project,
user_id=pipeline.user,
docstring=pipeline.docstring,
spec=pipeline.spec.json(sort_keys=True),
)
def from_update_model(self, model: PipelineModel) -> "PipelineSchema":
"""Update a `PipelineSchema` from a PipelineModel.
Args:
model: The `PipelineModel` to update the schema from.
Returns:
The updated `PipelineSchema`.
"""
self.name = model.name
self.updated = datetime.now()
self.docstring = model.docstring
self.spec = model.spec.json(sort_keys=True)
return self
def to_model(self) -> "PipelineModel":
"""Convert a `PipelineSchema` to a `PipelineModel`.
Returns:
The created PipelineModel.
"""
return PipelineModel(
id=self.id,
name=self.name,
project=self.project_id,
user=self.user_id,
docstring=self.docstring,
spec=PipelineSpec.parse_raw(self.spec),
created=self.created,
updated=self.updated,
)
from_create_model(pipeline)
classmethod
Create a PipelineSchema from a PipelineModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline |
PipelineModel |
The |
required |
Returns:
| Type | Description |
|---|---|
PipelineSchema |
The created |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
@classmethod
def from_create_model(cls, pipeline: PipelineModel) -> "PipelineSchema":
"""Create a `PipelineSchema` from a `PipelineModel`.
Args:
pipeline: The `PipelineModel` to create the schema from.
Returns:
The created `PipelineSchema`.
"""
return cls(
id=pipeline.id,
name=pipeline.name,
project_id=pipeline.project,
user_id=pipeline.user,
docstring=pipeline.docstring,
spec=pipeline.spec.json(sort_keys=True),
)
from_update_model(self, model)
Update a PipelineSchema from a PipelineModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
PipelineModel |
The |
required |
Returns:
| Type | Description |
|---|---|
PipelineSchema |
The updated |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def from_update_model(self, model: PipelineModel) -> "PipelineSchema":
"""Update a `PipelineSchema` from a PipelineModel.
Args:
model: The `PipelineModel` to update the schema from.
Returns:
The updated `PipelineSchema`.
"""
self.name = model.name
self.updated = datetime.now()
self.docstring = model.docstring
self.spec = model.spec.json(sort_keys=True)
return self
to_model(self)
Convert a PipelineSchema to a PipelineModel.
Returns:
| Type | Description |
|---|---|
PipelineModel |
The created PipelineModel. |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def to_model(self) -> "PipelineModel":
"""Convert a `PipelineSchema` to a `PipelineModel`.
Returns:
The created PipelineModel.
"""
return PipelineModel(
id=self.id,
name=self.name,
project=self.project_id,
user=self.user_id,
docstring=self.docstring,
spec=PipelineSpec.parse_raw(self.spec),
created=self.created,
updated=self.updated,
)
StepInputArtifactSchema (SQLModel)
pydantic-model
SQL Model that defines which artifacts are inputs to which step.
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
class StepInputArtifactSchema(SQLModel, table=True):
"""SQL Model that defines which artifacts are inputs to which step."""
step_id: UUID = Field(foreign_key="steprunschema.id", primary_key=True)
artifact_id: UUID = Field(foreign_key="artifactschema.id", primary_key=True)
name: str # Name of the input in the step
StepRunOrderSchema (SQLModel)
pydantic-model
SQL Model that defines the order of steps.
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
class StepRunOrderSchema(SQLModel, table=True):
"""SQL Model that defines the order of steps."""
parent_id: UUID = Field(foreign_key="steprunschema.id", primary_key=True)
child_id: UUID = Field(foreign_key="steprunschema.id", primary_key=True)
StepRunSchema (SQLModel)
pydantic-model
SQL Model for steps of pipeline runs.
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
class StepRunSchema(SQLModel, table=True):
"""SQL Model for steps of pipeline runs."""
id: UUID = Field(primary_key=True)
name: str
pipeline_run_id: UUID = Field(foreign_key="pipelinerunschema.id")
entrypoint_name: str
parameters: str = Field(max_length=4096)
step_configuration: str = Field(max_length=4096)
docstring: Optional[str] = Field(max_length=4096, nullable=True)
mlmd_id: int = Field(default=None, nullable=True)
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
@classmethod
def from_create_model(cls, model: StepRunModel) -> "StepRunSchema":
"""Create a `StepRunSchema` from a `StepRunModel`.
Args:
model: The `StepRunModel` to create the schema from.
Returns:
The created `StepRunSchema`.
"""
return cls(
id=model.id,
name=model.name,
pipeline_run_id=model.pipeline_run_id,
entrypoint_name=model.entrypoint_name,
parameters=json.dumps(model.parameters),
step_configuration=json.dumps(model.step_configuration),
docstring=model.docstring,
mlmd_id=model.mlmd_id,
)
def to_model(
self, parent_step_ids: List[UUID], mlmd_parent_step_ids: List[int]
) -> StepRunModel:
"""Convert a `StepRunSchema` to a `StepRunModel`.
Args:
parent_step_ids: The parent step ids to link to the step.
mlmd_parent_step_ids: The parent step ids in MLMD.
Returns:
The created StepRunModel.
"""
return StepRunModel(
id=self.id,
name=self.name,
pipeline_run_id=self.pipeline_run_id,
parent_step_ids=parent_step_ids,
entrypoint_name=self.entrypoint_name,
parameters=json.loads(self.parameters),
step_configuration=json.loads(self.step_configuration),
docstring=self.docstring,
mlmd_id=self.mlmd_id,
mlmd_parent_step_ids=mlmd_parent_step_ids,
created=self.created,
updated=self.updated,
)
from_create_model(model)
classmethod
Create a StepRunSchema from a StepRunModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
StepRunModel |
The |
required |
Returns:
| Type | Description |
|---|---|
StepRunSchema |
The created |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
@classmethod
def from_create_model(cls, model: StepRunModel) -> "StepRunSchema":
"""Create a `StepRunSchema` from a `StepRunModel`.
Args:
model: The `StepRunModel` to create the schema from.
Returns:
The created `StepRunSchema`.
"""
return cls(
id=model.id,
name=model.name,
pipeline_run_id=model.pipeline_run_id,
entrypoint_name=model.entrypoint_name,
parameters=json.dumps(model.parameters),
step_configuration=json.dumps(model.step_configuration),
docstring=model.docstring,
mlmd_id=model.mlmd_id,
)
to_model(self, parent_step_ids, mlmd_parent_step_ids)
Convert a StepRunSchema to a StepRunModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
parent_step_ids |
List[uuid.UUID] |
The parent step ids to link to the step. |
required |
mlmd_parent_step_ids |
List[int] |
The parent step ids in MLMD. |
required |
Returns:
| Type | Description |
|---|---|
StepRunModel |
The created StepRunModel. |
Source code in zenml/zen_stores/schemas/pipeline_schemas.py
def to_model(
self, parent_step_ids: List[UUID], mlmd_parent_step_ids: List[int]
) -> StepRunModel:
"""Convert a `StepRunSchema` to a `StepRunModel`.
Args:
parent_step_ids: The parent step ids to link to the step.
mlmd_parent_step_ids: The parent step ids in MLMD.
Returns:
The created StepRunModel.
"""
return StepRunModel(
id=self.id,
name=self.name,
pipeline_run_id=self.pipeline_run_id,
parent_step_ids=parent_step_ids,
entrypoint_name=self.entrypoint_name,
parameters=json.loads(self.parameters),
step_configuration=json.loads(self.step_configuration),
docstring=self.docstring,
mlmd_id=self.mlmd_id,
mlmd_parent_step_ids=mlmd_parent_step_ids,
created=self.created,
updated=self.updated,
)
project_schemas
SQL Model Implementations for Projects.
ProjectSchema (SQLModel)
pydantic-model
SQL Model for projects.
Source code in zenml/zen_stores/schemas/project_schemas.py
class ProjectSchema(SQLModel, table=True):
"""SQL Model for projects."""
id: UUID = Field(primary_key=True)
name: str
description: str
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
user_role_assignments: List["UserRoleAssignmentSchema"] = Relationship(
back_populates="project", sa_relationship_kwargs={"cascade": "delete"}
)
team_role_assignments: List["TeamRoleAssignmentSchema"] = Relationship(
back_populates="project",
sa_relationship_kwargs={"cascade": "all, delete"},
)
stacks: List["StackSchema"] = Relationship(
back_populates="project", sa_relationship_kwargs={"cascade": "delete"}
)
components: List["StackComponentSchema"] = Relationship(
back_populates="project", sa_relationship_kwargs={"cascade": "delete"}
)
flavors: List["FlavorSchema"] = Relationship(
back_populates="project", sa_relationship_kwargs={"cascade": "delete"}
)
pipelines: List["PipelineSchema"] = Relationship(
back_populates="project", sa_relationship_kwargs={"cascade": "delete"}
)
runs: List["PipelineRunSchema"] = Relationship(
back_populates="project", sa_relationship_kwargs={"cascade": "delete"}
)
@classmethod
def from_create_model(cls, project: ProjectModel) -> "ProjectSchema":
"""Create a `ProjectSchema` from a `ProjectModel`.
Args:
project: The `ProjectModel` from which to create the schema.
Returns:
The created `ProjectSchema`.
"""
return cls(
id=project.id, name=project.name, description=project.description
)
def from_update_model(self, model: ProjectModel) -> "ProjectSchema":
"""Update a `ProjectSchema` from a `ProjectModel`.
Args:
model: The `ProjectModel` from which to update the schema.
Returns:
The updated `ProjectSchema`.
"""
self.name = model.name
self.description = model.description
self.updated = datetime.now()
return self
def to_model(self) -> ProjectModel:
"""Convert a `ProjectSchema` to a `ProjectModel`.
Returns:
The converted `ProjectModel`.
"""
return ProjectModel.parse_obj(self)
from_create_model(project)
classmethod
Create a ProjectSchema from a ProjectModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project |
ProjectModel |
The |
required |
Returns:
| Type | Description |
|---|---|
ProjectSchema |
The created |
Source code in zenml/zen_stores/schemas/project_schemas.py
@classmethod
def from_create_model(cls, project: ProjectModel) -> "ProjectSchema":
"""Create a `ProjectSchema` from a `ProjectModel`.
Args:
project: The `ProjectModel` from which to create the schema.
Returns:
The created `ProjectSchema`.
"""
return cls(
id=project.id, name=project.name, description=project.description
)
from_update_model(self, model)
Update a ProjectSchema from a ProjectModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
ProjectModel |
The |
required |
Returns:
| Type | Description |
|---|---|
ProjectSchema |
The updated |
Source code in zenml/zen_stores/schemas/project_schemas.py
def from_update_model(self, model: ProjectModel) -> "ProjectSchema":
"""Update a `ProjectSchema` from a `ProjectModel`.
Args:
model: The `ProjectModel` from which to update the schema.
Returns:
The updated `ProjectSchema`.
"""
self.name = model.name
self.description = model.description
self.updated = datetime.now()
return self
to_model(self)
Convert a ProjectSchema to a ProjectModel.
Returns:
| Type | Description |
|---|---|
ProjectModel |
The converted |
Source code in zenml/zen_stores/schemas/project_schemas.py
def to_model(self) -> ProjectModel:
"""Convert a `ProjectSchema` to a `ProjectModel`.
Returns:
The converted `ProjectModel`.
"""
return ProjectModel.parse_obj(self)
stack_schemas
SQL Model Implementations for Stacks.
StackCompositionSchema (SQLModel)
pydantic-model
SQL Model for stack definitions.
Join table between Stacks and StackComponents.
Source code in zenml/zen_stores/schemas/stack_schemas.py
class StackCompositionSchema(SQLModel, table=True):
"""SQL Model for stack definitions.
Join table between Stacks and StackComponents.
"""
stack_id: UUID = Field(primary_key=True, foreign_key="stackschema.id")
component_id: UUID = Field(
primary_key=True, foreign_key="stackcomponentschema.id"
)
StackSchema (SQLModel)
pydantic-model
SQL Model for stacks.
Source code in zenml/zen_stores/schemas/stack_schemas.py
class StackSchema(SQLModel, table=True):
"""SQL Model for stacks."""
id: UUID = Field(primary_key=True)
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
name: str
is_shared: bool
project_id: UUID = Field(
sa_column=Column(ForeignKey("projectschema.id", ondelete="CASCADE"))
)
project: "ProjectSchema" = Relationship(back_populates="stacks")
user_id: UUID = Field(
sa_column=Column(ForeignKey("userschema.id", ondelete="SET NULL"))
)
user: "UserSchema" = Relationship(back_populates="stacks")
components: List["StackComponentSchema"] = Relationship(
back_populates="stacks", link_model=StackCompositionSchema
)
runs: List["PipelineRunSchema"] = Relationship(
back_populates="stack",
)
@classmethod
def from_create_model(
cls,
defined_components: List["StackComponentSchema"],
stack: StackModel,
) -> "StackSchema":
"""Create a StackSchema.
Args:
defined_components: The components that are part of the stack.
stack: The stack model to create the schema from.
Returns:
A StackSchema
"""
return cls(
id=stack.id,
name=stack.name,
project_id=stack.project,
user_id=stack.user,
is_shared=stack.is_shared,
components=defined_components,
)
def from_update_model(
self,
defined_components: List["StackComponentSchema"],
stack: StackModel,
) -> "StackSchema":
"""Update the updatable fields on an existing `StackSchema`.
Args:
defined_components: The components that are part of the stack.
stack: The stack model to create the schema from.
Returns:
A `StackSchema`
"""
self.name = stack.name
self.is_shared = stack.is_shared
self.components = defined_components
self.updated = datetime.now()
return self
def to_model(self) -> "StackModel":
"""Creates a `ComponentModel` from an instance of a `StackSchema`.
Returns:
a `FullStackModel`
"""
# This needs to be updated once multiple stack components per type are
# supported
return StackModel(
id=self.id,
name=self.name,
user=self.user_id,
project=self.project_id,
is_shared=self.is_shared,
components={c.type: [c.id] for c in self.components},
created=self.created,
updated=self.updated,
)
from_create_model(defined_components, stack)
classmethod
Create a StackSchema.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
defined_components |
List[StackComponentSchema] |
The components that are part of the stack. |
required |
stack |
StackModel |
The stack model to create the schema from. |
required |
Returns:
| Type | Description |
|---|---|
StackSchema |
A StackSchema |
Source code in zenml/zen_stores/schemas/stack_schemas.py
@classmethod
def from_create_model(
cls,
defined_components: List["StackComponentSchema"],
stack: StackModel,
) -> "StackSchema":
"""Create a StackSchema.
Args:
defined_components: The components that are part of the stack.
stack: The stack model to create the schema from.
Returns:
A StackSchema
"""
return cls(
id=stack.id,
name=stack.name,
project_id=stack.project,
user_id=stack.user,
is_shared=stack.is_shared,
components=defined_components,
)
from_update_model(self, defined_components, stack)
Update the updatable fields on an existing StackSchema.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
defined_components |
List[StackComponentSchema] |
The components that are part of the stack. |
required |
stack |
StackModel |
The stack model to create the schema from. |
required |
Returns:
| Type | Description |
|---|---|
StackSchema |
A |
Source code in zenml/zen_stores/schemas/stack_schemas.py
def from_update_model(
self,
defined_components: List["StackComponentSchema"],
stack: StackModel,
) -> "StackSchema":
"""Update the updatable fields on an existing `StackSchema`.
Args:
defined_components: The components that are part of the stack.
stack: The stack model to create the schema from.
Returns:
A `StackSchema`
"""
self.name = stack.name
self.is_shared = stack.is_shared
self.components = defined_components
self.updated = datetime.now()
return self
to_model(self)
Creates a ComponentModel from an instance of a StackSchema.
Returns:
| Type | Description |
|---|---|
StackModel |
a |
Source code in zenml/zen_stores/schemas/stack_schemas.py
def to_model(self) -> "StackModel":
"""Creates a `ComponentModel` from an instance of a `StackSchema`.
Returns:
a `FullStackModel`
"""
# This needs to be updated once multiple stack components per type are
# supported
return StackModel(
id=self.id,
name=self.name,
user=self.user_id,
project=self.project_id,
is_shared=self.is_shared,
components={c.type: [c.id] for c in self.components},
created=self.created,
updated=self.updated,
)
user_management_schemas
SQL Model Implementations for Users, Teams, Roles.
RoleSchema (SQLModel)
pydantic-model
SQL Model for roles.
Source code in zenml/zen_stores/schemas/user_management_schemas.py
class RoleSchema(SQLModel, table=True):
"""SQL Model for roles."""
id: UUID = Field(primary_key=True)
name: str
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
user_role_assignments: List["UserRoleAssignmentSchema"] = Relationship(
back_populates="role", sa_relationship_kwargs={"cascade": "delete"}
)
team_role_assignments: List["TeamRoleAssignmentSchema"] = Relationship(
back_populates="role", sa_relationship_kwargs={"cascade": "delete"}
)
@classmethod
def from_create_model(cls, model: RoleModel) -> "RoleSchema":
"""Create a `RoleSchema` from a `RoleModel`.
Args:
model: The `RoleModel` from which to create the schema.
Returns:
The created `RoleSchema`.
"""
return cls(id=model.id, name=model.name)
def from_update_model(self, model: RoleModel) -> "RoleSchema":
"""Update a `RoleSchema` from a `RoleModel`.
Args:
model: The `RoleModel` from which to update the schema.
Returns:
The updated `RoleSchema`.
"""
self.name = model.name
self.updated = datetime.now()
return self
def to_model(self) -> RoleModel:
"""Convert a `RoleSchema` to a `RoleModel`.
Returns:
The converted `RoleModel`.
"""
return RoleModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
)
from_create_model(model)
classmethod
Create a RoleSchema from a RoleModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
RoleModel |
The |
required |
Returns:
| Type | Description |
|---|---|
RoleSchema |
The created |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
@classmethod
def from_create_model(cls, model: RoleModel) -> "RoleSchema":
"""Create a `RoleSchema` from a `RoleModel`.
Args:
model: The `RoleModel` from which to create the schema.
Returns:
The created `RoleSchema`.
"""
return cls(id=model.id, name=model.name)
from_update_model(self, model)
Update a RoleSchema from a RoleModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
RoleModel |
The |
required |
Returns:
| Type | Description |
|---|---|
RoleSchema |
The updated |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
def from_update_model(self, model: RoleModel) -> "RoleSchema":
"""Update a `RoleSchema` from a `RoleModel`.
Args:
model: The `RoleModel` from which to update the schema.
Returns:
The updated `RoleSchema`.
"""
self.name = model.name
self.updated = datetime.now()
return self
to_model(self)
Convert a RoleSchema to a RoleModel.
Returns:
| Type | Description |
|---|---|
RoleModel |
The converted |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
def to_model(self) -> RoleModel:
"""Convert a `RoleSchema` to a `RoleModel`.
Returns:
The converted `RoleModel`.
"""
return RoleModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
)
TeamAssignmentSchema (SQLModel)
pydantic-model
SQL Model for team assignments.
Source code in zenml/zen_stores/schemas/user_management_schemas.py
class TeamAssignmentSchema(SQLModel, table=True):
"""SQL Model for team assignments."""
user_id: UUID = Field(primary_key=True, foreign_key="userschema.id")
team_id: UUID = Field(primary_key=True, foreign_key="teamschema.id")
TeamRoleAssignmentSchema (SQLModel)
pydantic-model
SQL Model for assigning roles to teams for a given project.
Source code in zenml/zen_stores/schemas/user_management_schemas.py
class TeamRoleAssignmentSchema(SQLModel, table=True):
"""SQL Model for assigning roles to teams for a given project."""
id: UUID = Field(primary_key=True, default_factory=uuid4)
role_id: UUID = Field(foreign_key="roleschema.id")
team_id: UUID = Field(foreign_key="teamschema.id")
project_id: Optional[UUID] = Field(
foreign_key="projectschema.id", nullable=True
)
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
role: RoleSchema = Relationship(back_populates="team_role_assignments")
team: TeamSchema = Relationship(back_populates="assigned_roles")
project: Optional["ProjectSchema"] = Relationship(
back_populates="team_role_assignments"
)
def to_model(self) -> RoleAssignmentModel:
"""Convert a `TeamRoleAssignmentSchema` to a `RoleAssignmentModel`.
Returns:
The converted `RoleAssignmentModel`.
"""
return RoleAssignmentModel(
id=self.id,
role=self.role_id,
team=self.team_id,
project=self.project_id,
created=self.created,
updated=self.updated,
)
to_model(self)
Convert a TeamRoleAssignmentSchema to a RoleAssignmentModel.
Returns:
| Type | Description |
|---|---|
RoleAssignmentModel |
The converted |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
def to_model(self) -> RoleAssignmentModel:
"""Convert a `TeamRoleAssignmentSchema` to a `RoleAssignmentModel`.
Returns:
The converted `RoleAssignmentModel`.
"""
return RoleAssignmentModel(
id=self.id,
role=self.role_id,
team=self.team_id,
project=self.project_id,
created=self.created,
updated=self.updated,
)
TeamSchema (SQLModel)
pydantic-model
SQL Model for teams.
Source code in zenml/zen_stores/schemas/user_management_schemas.py
class TeamSchema(SQLModel, table=True):
"""SQL Model for teams."""
id: UUID = Field(primary_key=True)
name: str
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
users: List["UserSchema"] = Relationship(
back_populates="teams", link_model=TeamAssignmentSchema
)
assigned_roles: List["TeamRoleAssignmentSchema"] = Relationship(
back_populates="team", sa_relationship_kwargs={"cascade": "delete"}
)
@classmethod
def from_create_model(cls, model: TeamModel) -> "TeamSchema":
"""Create a `TeamSchema` from a `TeamModel`.
Args:
model: The `TeamModel` from which to create the schema.
Returns:
The created `TeamSchema`.
"""
return cls(id=model.id, name=model.name)
def from_update_model(self, model: TeamModel) -> "TeamSchema":
"""Update a `TeamSchema` from a `TeamModel`.
Args:
model: The `TeamModel` from which to update the schema.
Returns:
The updated `TeamSchema`.
"""
self.name = model.name
self.updated = datetime.now()
return self
def to_model(self) -> TeamModel:
"""Convert a `TeamSchema` to a `TeamModel`.
Returns:
The converted `TeamModel`.
"""
return TeamModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
)
from_create_model(model)
classmethod
Create a TeamSchema from a TeamModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
TeamModel |
The |
required |
Returns:
| Type | Description |
|---|---|
TeamSchema |
The created |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
@classmethod
def from_create_model(cls, model: TeamModel) -> "TeamSchema":
"""Create a `TeamSchema` from a `TeamModel`.
Args:
model: The `TeamModel` from which to create the schema.
Returns:
The created `TeamSchema`.
"""
return cls(id=model.id, name=model.name)
from_update_model(self, model)
Update a TeamSchema from a TeamModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
TeamModel |
The |
required |
Returns:
| Type | Description |
|---|---|
TeamSchema |
The updated |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
def from_update_model(self, model: TeamModel) -> "TeamSchema":
"""Update a `TeamSchema` from a `TeamModel`.
Args:
model: The `TeamModel` from which to update the schema.
Returns:
The updated `TeamSchema`.
"""
self.name = model.name
self.updated = datetime.now()
return self
to_model(self)
Convert a TeamSchema to a TeamModel.
Returns:
| Type | Description |
|---|---|
TeamModel |
The converted |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
def to_model(self) -> TeamModel:
"""Convert a `TeamSchema` to a `TeamModel`.
Returns:
The converted `TeamModel`.
"""
return TeamModel(
id=self.id,
name=self.name,
created=self.created,
updated=self.updated,
)
UserRoleAssignmentSchema (SQLModel)
pydantic-model
SQL Model for assigning roles to users for a given project.
Source code in zenml/zen_stores/schemas/user_management_schemas.py
class UserRoleAssignmentSchema(SQLModel, table=True):
"""SQL Model for assigning roles to users for a given project."""
id: UUID = Field(primary_key=True, default_factory=uuid4)
role_id: UUID = Field(foreign_key="roleschema.id")
user_id: UUID = Field(foreign_key="userschema.id")
project_id: Optional[UUID] = Field(
foreign_key="projectschema.id", nullable=True
)
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
role: RoleSchema = Relationship(back_populates="user_role_assignments")
user: UserSchema = Relationship(back_populates="assigned_roles")
project: Optional["ProjectSchema"] = Relationship(
back_populates="user_role_assignments"
)
def to_model(self) -> RoleAssignmentModel:
"""Convert a `UserRoleAssignmentSchema` to a `RoleAssignmentModel`.
Returns:
The converted `RoleAssignmentModel`.
"""
return RoleAssignmentModel(
id=self.id,
role=self.role_id,
user=self.user_id,
project=self.project_id,
created=self.created,
updated=self.updated,
)
to_model(self)
Convert a UserRoleAssignmentSchema to a RoleAssignmentModel.
Returns:
| Type | Description |
|---|---|
RoleAssignmentModel |
The converted |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
def to_model(self) -> RoleAssignmentModel:
"""Convert a `UserRoleAssignmentSchema` to a `RoleAssignmentModel`.
Returns:
The converted `RoleAssignmentModel`.
"""
return RoleAssignmentModel(
id=self.id,
role=self.role_id,
user=self.user_id,
project=self.project_id,
created=self.created,
updated=self.updated,
)
UserSchema (SQLModel)
pydantic-model
SQL Model for users.
Source code in zenml/zen_stores/schemas/user_management_schemas.py
class UserSchema(SQLModel, table=True):
"""SQL Model for users."""
id: UUID = Field(primary_key=True)
name: str
full_name: str
email: Optional[str] = Field(nullable=True)
active: bool
password: Optional[str] = Field(nullable=True)
activation_token: Optional[str] = Field(nullable=True)
created: datetime = Field(default_factory=datetime.now)
updated: datetime = Field(default_factory=datetime.now)
email_opted_in: Optional[bool] = Field(nullable=True)
teams: List["TeamSchema"] = Relationship(
back_populates="users", link_model=TeamAssignmentSchema
)
assigned_roles: List["UserRoleAssignmentSchema"] = Relationship(
back_populates="user", sa_relationship_kwargs={"cascade": "delete"}
)
stacks: List["StackSchema"] = Relationship(
back_populates="user",
)
components: List["StackComponentSchema"] = Relationship(
back_populates="user",
)
flavors: List["FlavorSchema"] = Relationship(
back_populates="user",
)
pipelines: List["PipelineSchema"] = Relationship(
back_populates="user",
)
runs: List["PipelineRunSchema"] = Relationship(
back_populates="user",
)
@classmethod
def from_create_model(cls, model: UserModel) -> "UserSchema":
"""Create a `UserSchema` from a `UserModel`.
Args:
model: The `UserModel` from which to create the schema.
Returns:
The created `UserSchema`.
"""
return cls(
id=model.id,
name=model.name,
full_name=model.full_name,
active=model.active,
password=model.get_hashed_password(),
activation_token=model.get_hashed_activation_token(),
)
def from_update_model(self, model: UserModel) -> "UserSchema":
"""Update a `UserSchema` from a `UserModel`.
Args:
model: The `UserModel` from which to update the schema.
Returns:
The updated `UserSchema`.
"""
self.name = model.name
self.full_name = model.full_name
self.active = model.active
self.password = model.get_hashed_password()
self.activation_token = model.get_hashed_activation_token()
self.updated = datetime.now()
return self
def to_model(self) -> UserModel:
"""Convert a `UserSchema` to a `UserModel`.
Returns:
The converted `UserModel`.
"""
return UserModel(
id=self.id,
name=self.name,
full_name=self.full_name,
email=self.email,
email_opted_in=self.email_opted_in,
active=self.active,
password=self.password,
activation_token=self.activation_token,
created=self.created,
updated=self.updated,
)
from_create_model(model)
classmethod
Create a UserSchema from a UserModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
UserModel |
The |
required |
Returns:
| Type | Description |
|---|---|
UserSchema |
The created |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
@classmethod
def from_create_model(cls, model: UserModel) -> "UserSchema":
"""Create a `UserSchema` from a `UserModel`.
Args:
model: The `UserModel` from which to create the schema.
Returns:
The created `UserSchema`.
"""
return cls(
id=model.id,
name=model.name,
full_name=model.full_name,
active=model.active,
password=model.get_hashed_password(),
activation_token=model.get_hashed_activation_token(),
)
from_update_model(self, model)
Update a UserSchema from a UserModel.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
model |
UserModel |
The |
required |
Returns:
| Type | Description |
|---|---|
UserSchema |
The updated |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
def from_update_model(self, model: UserModel) -> "UserSchema":
"""Update a `UserSchema` from a `UserModel`.
Args:
model: The `UserModel` from which to update the schema.
Returns:
The updated `UserSchema`.
"""
self.name = model.name
self.full_name = model.full_name
self.active = model.active
self.password = model.get_hashed_password()
self.activation_token = model.get_hashed_activation_token()
self.updated = datetime.now()
return self
to_model(self)
Convert a UserSchema to a UserModel.
Returns:
| Type | Description |
|---|---|
UserModel |
The converted |
Source code in zenml/zen_stores/schemas/user_management_schemas.py
def to_model(self) -> UserModel:
"""Convert a `UserSchema` to a `UserModel`.
Returns:
The converted `UserModel`.
"""
return UserModel(
id=self.id,
name=self.name,
full_name=self.full_name,
email=self.email,
email_opted_in=self.email_opted_in,
active=self.active,
password=self.password,
activation_token=self.activation_token,
created=self.created,
updated=self.updated,
)
sql_zen_store
SQL Zen Store implementation.
SQLDatabaseDriver (StrEnum)
SQL database drivers supported by the SQL ZenML store.
Source code in zenml/zen_stores/sql_zen_store.py
class SQLDatabaseDriver(StrEnum):
"""SQL database drivers supported by the SQL ZenML store."""
MYSQL = "mysql"
SQLITE = "sqlite"
SqlZenStore (BaseZenStore)
pydantic-model
Store Implementation that uses SQL database backend.
Attributes:
| Name | Type | Description |
|---|---|---|
config |
RestZenStoreConfiguration |
The configuration of the SQL ZenML store. |
TYPE |
ClassVar[zenml.enums.StoreType] |
The type of the store. |
CONFIG_TYPE |
ClassVar[Type[zenml.config.store_config.StoreConfiguration]] |
The type of the store configuration. |
_engine |
The SQLAlchemy engine. |
|
_metadata_store |
The metadata store. |
Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStore(BaseZenStore):
"""Store Implementation that uses SQL database backend.
Attributes:
config: The configuration of the SQL ZenML store.
TYPE: The type of the store.
CONFIG_TYPE: The type of the store configuration.
_engine: The SQLAlchemy engine.
_metadata_store: The metadata store.
"""
config: SqlZenStoreConfiguration
TYPE: ClassVar[StoreType] = StoreType.SQL
CONFIG_TYPE: ClassVar[Type[StoreConfiguration]] = SqlZenStoreConfiguration
_engine: Optional[Engine] = None
_metadata_store: Optional["MetadataStore"] = None
@property
def engine(self) -> Engine:
"""The SQLAlchemy engine.
Returns:
The SQLAlchemy engine.
Raises:
ValueError: If the store is not initialized.
"""
if not self._engine:
raise ValueError("Store not initialized")
return self._engine
@property
def metadata_store(self) -> "MetadataStore":
"""The metadata store.
Returns:
The metadata store.
Raises:
ValueError: If the store is not initialized.
"""
if not self._metadata_store:
raise ValueError("Store not initialized")
return self._metadata_store
@property
def runs_inside_server(self) -> bool:
"""Whether the store is running inside a server.
Returns:
Whether the store is running inside a server.
"""
if ENV_ZENML_SERVER_DEPLOYMENT_TYPE in os.environ:
return True
return False
# ====================================
# ZenML Store interface implementation
# ====================================
# --------------------------------
# Initialization and configuration
# --------------------------------
def _initialize(self) -> None:
"""Initialize the SQL store."""
from zenml.zen_stores.metadata_store import MetadataStore
logger.debug("Initializing SqlZenStore at %s", self.config.url)
metadata_config = self.config.get_metadata_config()
self._metadata_store = MetadataStore(config=metadata_config)
url, connect_args, engine_args = self.config.get_sqlmodel_config()
self._engine = create_engine(
url=url, connect_args=connect_args, **engine_args
)
SQLModel.metadata.create_all(self._engine)
@staticmethod
def get_local_url(path: str) -> str:
"""Get a local SQL url for a given local path.
Args:
path: The path to the local sqlite file.
Returns:
The local SQL url for the given path.
"""
return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"
@classmethod
def copy_local_store(
cls,
config: StoreConfiguration,
path: str,
load_config_path: Optional[PurePath] = None,
) -> StoreConfiguration:
"""Copy a local store to a new location.
Use this method to create a copy of a store database to a new location
and return a new store configuration pointing to the database copy. This
only applies to stores that use the local filesystem to store their
data. Calling this method for remote stores simply returns the input
store configuration unaltered.
Args:
config: The configuration of the store to copy.
path: The new local path where the store DB will be copied.
load_config_path: path that will be used to load the copied store
database. This can be set to a value different from `path`
if the local database copy will be loaded from a different
environment, e.g. when the database is copied to a container
image and loaded using a different absolute path. This will be
reflected in the paths and URLs encoded in the copied store
configuration.
Returns:
The store configuration of the copied store.
"""
config_copy = config.copy()
sql_url = make_url(config.url)
if sql_url.drivername != SQLDatabaseDriver.SQLITE:
# this is not a configuration backed by a local filesystem
return config_copy
assert sql_url.database
local_path = Path(sql_url.database)
io_utils.create_dir_recursive_if_not_exists(path)
fileio.copy(
str(local_path), str(Path(path) / local_path.name), overwrite=True
)
if load_config_path:
config_copy.url = cls.get_local_url(str(load_config_path))
else:
config_copy.url = cls.get_local_url(path)
return config_copy
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
model = super().get_store_info()
sql_url = make_url(self.config.url)
model.database_type = ServerDatabaseType(sql_url.drivername)
return model
# ------------
# TFX Metadata
# ------------
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the TFX metadata config of this ZenStore.
Args:
expand_certs: Whether to expand the certificate paths in the
connection config to their value.
Returns:
The TFX metadata config of this ZenStore.
"""
return self.config.get_metadata_config(expand_certs=expand_certs)
# ------
# Stacks
# ------
@track(AnalyticsEvent.REGISTERED_STACK)
def create_stack(
self,
stack: StackModel,
) -> StackModel:
"""Register a new stack.
Args:
stack: The stack to register.
Returns:
The registered stack.
Raises:
KeyError: If one or more of the stack's components are not
registered in the store.
"""
with Session(self.engine) as session:
project = self._get_project_schema(stack.project, session=session)
user = self._get_user_schema(stack.user, session=session)
self.fail_if_stack_with_id_already_exists(
stack=stack, session=session
)
self.fail_if_stack_with_name_exists_for_user(
stack=stack, project=project, user=user, session=session
)
if stack.is_shared:
self.fail_if_stack_with_name_already_shared(
stack=stack, project=project, session=session
)
# Get the Schemas of all components mentioned
component_ids = [
component_id
for list_of_component_ids in stack.components.values()
for component_id in list_of_component_ids
]
filters = [
(StackComponentSchema.id == component_id)
for component_id in component_ids
]
defined_components = session.exec(
select(StackComponentSchema).where(or_(*filters))
).all()
defined_component_ids = [c.id for c in defined_components]
# check if all component IDs are valid
if len(component_ids) > 0 and len(defined_component_ids) != len(
component_ids
):
raise KeyError(
f"Some components referenced in the stack were not found: "
f"{set(component_ids) - set(defined_component_ids)}"
)
# Create the stack
stack_in_db = StackSchema.from_create_model(
defined_components=defined_components,
stack=stack,
)
session.add(stack_in_db)
session.commit()
session.refresh(stack_in_db)
return stack_in_db.to_model()
def get_stack(self, stack_id: UUID) -> StackModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
Raises:
KeyError: if the stack doesn't exist.
"""
with Session(self.engine) as session:
stack = session.exec(
select(StackSchema).where(StackSchema.id == stack_id)
).first()
if stack is None:
raise KeyError(f"Stack with ID {stack_id} not found.")
return stack.to_model()
def list_stacks(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_id: Optional[UUID] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[StackModel]:
"""List all stacks matching the given filter criteria.
Args:
project_name_or_id: Id or name of the Project containing the stack
user_name_or_id: Optionally filter stacks by their owner
component_id: Optionally filter for stacks that contain the
component
name: Optionally filter stacks by their name
is_shared: Optionally filter out stacks by whether they are shared
or not
Returns:
A list of all stacks matching the filter criteria.
"""
with Session(self.engine) as session:
# Get a list of all stacks
query = select(StackSchema)
# TODO: prettify
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(StackSchema.project_id == project.id)
if user_name_or_id:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(StackSchema.user_id == user.id)
if component_id:
query = query.where(
StackCompositionSchema.stack_id == StackSchema.id
).where(StackCompositionSchema.component_id == component_id)
if name:
query = query.where(StackSchema.name == name)
if is_shared is not None:
query = query.where(StackSchema.is_shared == is_shared)
stacks = session.exec(query.order_by(StackSchema.name)).all()
return [stack.to_model() for stack in stacks]
@track(AnalyticsEvent.UPDATED_STACK)
def update_stack(self, stack: StackModel) -> StackModel:
"""Update a stack.
Args:
stack: The stack to use for the update.
Returns:
The updated stack.
Raises:
KeyError: if the stack doesn't exist.
"""
with Session(self.engine) as session:
# Check if stack with the domain key (name, project, owner) already
# exists
existing_stack = session.exec(
select(StackSchema).where(StackSchema.id == stack.id)
).first()
if existing_stack is None:
raise KeyError(
f"Unable to update stack with id "
f"'{stack.id}': Found no"
f"existing stack with this id."
)
# In case of a renaming update, make sure no stack already exists
# with that name
if existing_stack.name != stack.name:
project = self._get_project_schema(
project_name_or_id=stack.project, session=session
)
user = self._get_user_schema(
user_name_or_id=stack.user, session=session
)
self.fail_if_stack_with_name_exists_for_user(
stack=stack, project=project, user=user, session=session
)
# Check if stack update makes the stack a shared stack,
# In that case check if a stack with the same name is
# already shared within the project
if not existing_stack.is_shared and stack.is_shared:
project = self._get_project_schema(
project_name_or_id=stack.project, session=session
)
self.fail_if_stack_with_name_already_shared(
stack=stack, project=project, session=session
)
# Get the Schemas of all components mentioned
filters = [
(StackComponentSchema.id == component_id)
for list_of_component_ids in stack.components.values()
for component_id in list_of_component_ids
]
defined_components = session.exec(
select(StackComponentSchema).where(or_(*filters))
).all()
existing_stack.from_update_model(
stack=stack, defined_components=defined_components
)
session.add(existing_stack)
session.commit()
return existing_stack.to_model()
@track(AnalyticsEvent.DELETED_STACK)
def delete_stack(self, stack_id: UUID) -> None:
"""Delete a stack.
Args:
stack_id: The ID of the stack to delete.
Raises:
KeyError: if the stack doesn't exist.
"""
with Session(self.engine) as session:
try:
stack = session.exec(
select(StackSchema).where(StackSchema.id == stack_id)
).one()
session.delete(stack)
except NoResultFound as error:
raise KeyError from error
session.commit()
@staticmethod
def fail_if_stack_with_id_already_exists(
stack: StackModel, session: Session
) -> None:
"""Raise an exception if a Stack with the same id already exists.
Args:
stack: The Stack
session: The Session
Raises:
StackExistsError: If a stack with the same id already
exists
"""
existing_id_stack = session.exec(
select(StackSchema).where(StackSchema.id == stack.id)
).first()
if existing_id_stack is not None:
raise StackExistsError(
f"Unable to register stack with name "
f"'{stack.name}' and id '{stack.id}': "
f" Found an existing component with the same id."
)
@staticmethod
def fail_if_stack_with_name_exists_for_user(
stack: StackModel,
project: ProjectSchema,
session: Session,
user: UserSchema,
) -> None:
"""Raise an exception if a Component with same name exists for user.
Args:
stack: The Stack
project: The project scope within which to check
user: The user that owns the Stack
session: The Session
Returns:
None
Raises:
StackExistsError: If a Stack with the given name is already
owned by the user
"""
existing_domain_stack = session.exec(
select(StackSchema)
.where(StackSchema.name == stack.name)
.where(StackSchema.project_id == stack.project)
.where(StackSchema.user_id == stack.user)
).first()
if existing_domain_stack is not None:
raise StackExistsError(
f"Unable to register stack with name "
f"'{stack.name}': Found an existing stack with the same "
f"name in the active project, '{project.name}', owned by the "
f"same user, '{user.name}'."
)
return None
def fail_if_stack_with_name_already_shared(
self, stack: StackModel, project: ProjectSchema, session: Session
) -> None:
"""Raise an exception if a Stack with same name is already shared.
Args:
stack: The Stack
project: The project scope within which to check
session: The Session
Raises:
StackExistsError: If a stack with the given name is already shared
by a user.
"""
# Check if component with the same name, type is already shared
# within the project
existing_shared_stack = session.exec(
select(StackSchema)
.where(StackSchema.name == stack.name)
.where(StackSchema.project_id == stack.project)
.where(StackSchema.is_shared == stack.is_shared)
).first()
if existing_shared_stack is not None:
owner_of_shared = self._get_user_schema(
existing_shared_stack.user_id, session=session
)
raise StackExistsError(
f"Unable to share stack with name '{stack.name}': Found an "
f"existing stack with the same name in project "
f"'{project.name}' shared by '{owner_of_shared.name}'."
)
# ----------------
# Stack components
# ----------------
@track(AnalyticsEvent.REGISTERED_STACK_COMPONENT)
def create_stack_component(
self,
component: ComponentModel,
) -> ComponentModel:
"""Create a stack component.
Args:
component: The stack component to create.
Returns:
The created stack component.
"""
with Session(self.engine) as session:
project = self._get_project_schema(
project_name_or_id=component.project, session=session
)
user = self._get_user_schema(
user_name_or_id=component.user, session=session
)
self.fail_if_component_with_id_already_exists(
component=component, session=session
)
self.fail_if_component_with_name_type_exists_for_user(
component=component, project=project, user=user, session=session
)
if component.is_shared:
self.fail_if_component_with_name_type_already_shared(
component=component, project=project, session=session
)
# Create the component
component_in_db = StackComponentSchema.from_create_model(
component=component
)
session.add(component_in_db)
session.commit()
session.refresh(component_in_db)
return component_in_db.to_model()
def get_stack_component(self, component_id: UUID) -> ComponentModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
with Session(self.engine) as session:
stack_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component_id
)
).first()
if stack_component is None:
raise KeyError(
f"Stack component with ID {component_id} not found."
)
return stack_component.to_model()
def list_stack_components(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
type: Optional[str] = None,
flavor_name: Optional[str] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[ComponentModel]:
"""List all stack components matching the given filter criteria.
Args:
project_name_or_id: The ID or name of the Project to which the stack
components belong
user_name_or_id: Optionally filter stack components by the owner
type: Optionally filter by type of stack component
flavor_name: Optionally filter by flavor
name: Optionally filter stack component by name
is_shared: Optionally filter out stack component by whether they are
shared or not
Returns:
A list of all stack components matching the filter criteria.
"""
with Session(self.engine) as session:
# Get a list of all stacks
query = select(StackComponentSchema)
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(
StackComponentSchema.project_id == project.id
)
if user_name_or_id:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(StackComponentSchema.user_id == user.id)
if type:
query = query.where(StackComponentSchema.type == type)
if flavor_name:
query = query.where(StackComponentSchema.flavor == flavor_name)
if name:
query = query.where(StackComponentSchema.name == name)
if is_shared is not None:
query = query.where(StackComponentSchema.is_shared == is_shared)
list_of_stack_components_in_db = session.exec(query).all()
return [comp.to_model() for comp in list_of_stack_components_in_db]
@track(AnalyticsEvent.UPDATED_STACK_COMPONENT)
def update_stack_component(
self, component: ComponentModel
) -> ComponentModel:
"""Update an existing stack component.
Args:
component: The stack component model to use for the update.
Returns:
The updated stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
with Session(self.engine) as session:
existing_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component.id
)
).first()
if existing_component is None:
raise KeyError(
f"Unable to update component with id "
f"'{component.id}': Found no"
f"existing component with this id."
)
# In case of a renaming update, make sure no component of the same
# type already exists with that name
if existing_component.name != component.name:
project = self._get_project_schema(
project_name_or_id=component.project, session=session
)
user = self._get_user_schema(
user_name_or_id=component.user, session=session
)
self.fail_if_component_with_name_type_exists_for_user(
component=component,
project=project,
user=user,
session=session,
)
# Check if component update makes the component a shared component,
# In that case check if a component with the same name, type are
# already shared within the project
if not existing_component.is_shared and component.is_shared:
project = self._get_project_schema(
project_name_or_id=component.project, session=session
)
self.fail_if_component_with_name_type_already_shared(
component=component, project=project, session=session
)
existing_component.from_update_model(component=component)
session.add(existing_component)
session.commit()
return existing_component.to_model()
@track(AnalyticsEvent.DELETED_STACK_COMPONENT)
def delete_stack_component(self, component_id: UUID) -> None:
"""Delete a stack component.
Args:
component_id: The id of the stack component to delete.
Raises:
KeyError: if the stack component doesn't exist.
IllegalOperationError: if the stack component is part of one or
more stacks.
"""
with Session(self.engine) as session:
try:
stack_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component_id
)
).one()
if len(stack_component.stacks) > 0:
raise IllegalOperationError(
f"Stack Component `{stack_component.name}` of type "
f"`{stack_component.type} can not be "
f"deleted as it is part of "
f"{len(stack_component.stacks)} stacks. "
f"Before deleting this stack "
f"component, make sure to remove it "
f"from all stacks."
)
else:
session.delete(stack_component)
except NoResultFound as error:
raise KeyError from error
session.commit()
def get_stack_component_side_effects(
self,
component_id: UUID,
run_id: UUID,
pipeline_id: UUID,
stack_id: UUID,
) -> Dict[Any, Any]:
"""Get the side effects of a stack component.
Args:
component_id: The id of the stack component to get side effects for.
run_id: The id of the run to get side effects for.
pipeline_id: The id of the pipeline to get side effects for.
stack_id: The id of the stack to get side effects for.
"""
pass # TODO: implement this
@staticmethod
def fail_if_component_with_id_already_exists(
component: ComponentModel, session: Session
) -> None:
"""Raise an exception if a Component with the same id already exists.
Args:
component: The Component
session: The Session
Raises:
StackComponentExistsError: If a component with the same id already
exists
"""
existing_id_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component.id
)
).first()
if existing_id_component is not None:
raise StackComponentExistsError(
f"Unable to register '{component.type.value}' component "
f"with name '{component.name}' and id '{component.id}': "
f" Found an existing component with the same id."
)
@staticmethod
def fail_if_component_with_name_type_exists_for_user(
component: ComponentModel,
project: ProjectSchema,
session: Session,
user: UserSchema,
) -> None:
"""Raise an exception if a Component with same name/type exists for user.
Args:
component: The Component
project: The project scope within which to check
user: The user that owns the Component
session: The Session
Returns:
None
Raises:
StackComponentExistsError: If a component with the given name and
type is already owned by the user
"""
# Check if component with the same domain key (name, type, project,
# owner) already exists
existing_domain_component = session.exec(
select(StackComponentSchema)
.where(StackComponentSchema.name == component.name)
.where(StackComponentSchema.project_id == component.project)
.where(StackComponentSchema.user_id == component.user)
.where(StackComponentSchema.type == component.type)
).first()
if existing_domain_component is not None:
raise StackComponentExistsError(
f"Unable to register '{component.type.value}' component "
f"with name '{component.name}': Found an existing "
f"component with the same name and type in the same "
f" project, '{project.name}', owned by the same "
f" user, '{user.name}'."
)
return None
def fail_if_component_with_name_type_already_shared(
self,
component: ComponentModel,
project: ProjectSchema,
session: Session,
) -> None:
"""Raise an exception if a Component with same name/type already shared.
Args:
component: The Component
project: The project scope within which to check
session: The Session
Raises:
StackComponentExistsError: If a component with the given name and
type is already shared by a user
"""
# Check if component with the same name, type is already shared
# within the project
existing_shared_component = session.exec(
select(StackComponentSchema)
.where(StackComponentSchema.name == component.name)
.where(StackComponentSchema.project_id == component.project)
.where(StackComponentSchema.is_shared == component.is_shared)
.where(StackComponentSchema.type == component.type)
).first()
if existing_shared_component is not None:
owner_of_shared = self._get_user_schema(
existing_shared_component.user_id, session=session
)
raise StackComponentExistsError(
f"Unable to shared component of type '{component.type.value}' "
f"with name '{component.name}': Found an "
f"existing component with the same name and type in project "
f"'{project.name}' shared by "
f"'{owner_of_shared.name}'."
)
# -----------------------
# Stack component flavors
# -----------------------
@track(AnalyticsEvent.CREATED_FLAVOR)
def create_flavor(
self,
flavor: FlavorModel,
) -> FlavorModel:
"""Creates a new stack component flavor.
Args:
flavor: The stack component flavor to create.
Returns:
The newly created flavor.
Raises:
EntityExistsError: If a flavor with the same name and type
is already owned by this user in this project.
"""
with Session(self.engine) as session:
# Check if component with the same domain key (name, type, project,
# owner) already exists
existing_flavor = session.exec(
select(FlavorSchema)
.where(FlavorSchema.name == flavor.name)
.where(FlavorSchema.type == flavor.type)
.where(FlavorSchema.project_id == flavor.project)
.where(FlavorSchema.user_id == flavor.user)
).first()
if existing_flavor is not None:
raise EntityExistsError(
f"Unable to register '{flavor.type.value}' flavor "
f"with name '{flavor.name}': Found an existing "
f"flavor with the same name and type in the same "
f"'{flavor.project}' project owned by the same "
f"'{flavor.user}' user."
)
flavor_in_db = FlavorSchema.from_create_model(flavor=flavor)
session.add(flavor_in_db)
session.commit()
return flavor_in_db.to_model()
def get_flavor(self, flavor_id: UUID) -> FlavorModel:
"""Get a flavor by ID.
Args:
flavor_id: The ID of the flavor to fetch.
Returns:
The stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
with Session(self.engine) as session:
flavor_in_db = session.exec(
select(FlavorSchema).where(FlavorSchema.id == flavor_id)
).first()
if flavor_in_db is None:
raise KeyError(f"Flavor with ID {flavor_id} not found.")
return flavor_in_db.to_model()
def list_flavors(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_type: Optional[StackComponentType] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[FlavorModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
project_name_or_id: Optionally filter by the Project to which the
component flavors belong
component_type: Optionally filter by type of stack component
user_name_or_id: Optionally filter by the owner
component_type: Optionally filter by type of stack component
name: Optionally filter flavors by name
is_shared: Optionally filter out flavors by whether they are
shared or not
Returns:
List of all the stack component flavors matching the given criteria
"""
with Session(self.engine) as session:
query = select(FlavorSchema)
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(FlavorSchema.project_id == project.id)
if component_type:
query = query.where(FlavorSchema.type == component_type)
if name:
query = query.where(FlavorSchema.name == name)
if user_name_or_id:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(FlavorSchema.user_id == user.id)
list_of_flavors_in_db = session.exec(query).all()
return [flavor.to_model() for flavor in list_of_flavors_in_db]
@track(AnalyticsEvent.UPDATED_FLAVOR)
def update_flavor(self, flavor: FlavorModel) -> FlavorModel:
"""Update an existing stack component flavor.
Args:
flavor: The model of the flavor to update.
Returns:
The updated flavor.
Raises:
KeyError: if the flavor doesn't exist.
"""
with Session(self.engine) as session:
existing_flavor = session.exec(
select(FlavorSchema).where(FlavorSchema.id == flavor.id)
).first()
if existing_flavor is None:
raise KeyError(
f"Unable to update flavor with id '{flavor.id}': Found no"
f"existing component with this id."
)
existing_flavor.from_update_model(flavor=flavor)
session.add(existing_flavor)
session.commit()
return existing_flavor.to_model()
@track(AnalyticsEvent.DELETED_FLAVOR)
def delete_flavor(self, flavor_id: UUID) -> None:
"""Delete a flavor.
Args:
flavor_id: The id of the flavor to delete.
Raises:
KeyError: if the flavor doesn't exist.
IllegalOperationError: if the flavor is used by a stack component.
"""
with Session(self.engine) as session:
try:
flavor_in_db = session.exec(
select(FlavorSchema).where(FlavorSchema.id == flavor_id)
).one()
components_of_flavor = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.flavor == flavor_in_db.name
)
).all()
if len(components_of_flavor) > 0:
raise IllegalOperationError(
f"Stack Component `{flavor_in_db.name}` of type "
f"`{flavor_in_db.type} can not be "
f"deleted as it is used by"
f"{len(components_of_flavor)} "
f"components. Before deleting this "
f"flavor, make sure to delete all "
f"associated components."
)
else:
session.delete(flavor_in_db)
except NoResultFound as error:
raise KeyError from error
session.commit()
# -----
# Users
# -----
@property
def active_user_name(self) -> str:
"""Gets the active username.
Returns:
The active username.
"""
return self._default_user_name
@track(AnalyticsEvent.CREATED_USER)
def create_user(self, user: UserModel) -> UserModel:
"""Creates a new user.
Args:
user: User to be created.
Returns:
The newly created user.
Raises:
EntityExistsError: If a user with the given name already exists.
"""
with Session(self.engine) as session:
# Check if user with the given name already exists
existing_user = session.exec(
select(UserSchema).where(UserSchema.name == user.name)
).first()
if existing_user is not None:
raise EntityExistsError(
f"Unable to create user with name '{user.name}': "
f"Found existing user with this name."
)
# Create the user
new_user = UserSchema.from_create_model(user)
session.add(new_user)
session.commit()
return new_user.to_model()
def get_user(self, user_name_or_id: Union[str, UUID]) -> UserModel:
"""Gets a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
"""
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
return user.to_model()
def list_users(self) -> List[UserModel]:
"""List all users.
Returns:
A list of all users.
"""
with Session(self.engine) as session:
users = session.exec(select(UserSchema)).all()
return [user.to_model() for user in users]
@track(AnalyticsEvent.UPDATED_USER)
def update_user(self, user: UserModel) -> UserModel:
"""Updates an existing user.
Args:
user: The User model to use for the update.
Returns:
The updated user.
"""
with Session(self.engine) as session:
existing_user = self._get_user_schema(user.id, session=session)
existing_user.from_update_model(user)
session.add(existing_user)
session.commit()
# Refresh the Model that was just created
session.refresh(existing_user)
return existing_user.to_model()
@track(AnalyticsEvent.DELETED_USER)
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
"""Deletes a user.
Args:
user_name_or_id: The name or the ID of the user to delete.
Raises:
KeyError: If no user with the given name exists.
"""
with Session(self.engine) as session:
try:
user = self._get_user_schema(user_name_or_id, session=session)
session.delete(user)
session.commit()
except NoResultFound as error:
raise KeyError from error
def user_email_opt_in(
self,
user_name_or_id: Union[str, UUID],
user_opt_in_response: bool,
email: Optional[str] = None,
) -> UserModel:
"""Persist user response to the email prompt.
Args:
user_name_or_id: The name or the ID of the user.
user_opt_in_response: Whether this email should be associated
with the user id in the telemetry
email: The users email
Returns:
The updated user.
Raises:
KeyError: If no user with the given name exists.
"""
with Session(self.engine) as session:
try:
user = self._get_user_schema(user_name_or_id, session=session)
except NoResultFound as error:
raise KeyError from error
else:
# TODO: In the future we might want to validate that the email
# is non-empty and valid at this point if user_opt_in_response
# is True
user.email = email
user.email_opted_in = user_opt_in_response
session.add(user)
session.commit()
return user.to_model()
# -----
# Teams
# -----
@track(AnalyticsEvent.CREATED_TEAM)
def create_team(self, team: TeamModel) -> TeamModel:
"""Creates a new team.
Args:
team: The team model to create.
Returns:
The newly created team.
Raises:
EntityExistsError: If a team with the given name already exists.
"""
with Session(self.engine) as session:
# Check if team with the given name already exists
existing_team = session.exec(
select(TeamSchema).where(TeamSchema.name == team.name)
).first()
if existing_team is not None:
raise EntityExistsError(
f"Unable to create team with name '{team.name}': "
f"Found existing team with this name."
)
# Create the team
new_team = TeamSchema.from_create_model(team)
session.add(new_team)
session.commit()
return new_team.to_model()
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
return team.to_model()
def list_teams(self) -> List[TeamModel]:
"""List all teams.
Returns:
A list of all teams.
"""
with Session(self.engine) as session:
teams = session.exec(select(TeamSchema)).all()
return [team.to_model() for team in teams]
@track(AnalyticsEvent.UPDATED_TEAM)
def update_team(self, team: TeamModel) -> TeamModel:
"""Update an existing team.
Args:
team: The team to use for the update.
Returns:
The updated team.
Raises:
KeyError: if the team does not exist.
"""
with Session(self.engine) as session:
existing_team = session.exec(
select(TeamSchema).where(TeamSchema.id == team.id)
).first()
if existing_team is None:
raise KeyError(
f"Unable to update team with id "
f"'{team.id}': Found no"
f"existing teams with this id."
)
# Update the team
existing_team.from_update_model(team)
session.add(existing_team)
session.commit()
# Refresh the Model that was just created
session.refresh(existing_team)
return existing_team.to_model()
@track(AnalyticsEvent.DELETED_TEAM)
def delete_team(self, team_name_or_id: Union[str, UUID]) -> None:
"""Deletes a team.
Args:
team_name_or_id: Name or ID of the team to delete.
Raises:
KeyError: If no team with the given name exists.
"""
with Session(self.engine) as session:
try:
team = self._get_team_schema(team_name_or_id, session=session)
session.delete(team)
session.commit()
except NoResultFound as error:
raise KeyError from error
# ---------------
# Team membership
# ---------------
def get_users_for_team(
self, team_name_or_id: Union[str, UUID]
) -> List[UserModel]:
"""Fetches all users of a team.
Args:
team_name_or_id: The name or ID of the team for which to get users.
Returns:
A list of all users that are part of the team.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
return [user.to_model() for user in team.users]
def get_teams_for_user(
self, user_name_or_id: Union[str, UUID]
) -> List[TeamModel]:
"""Fetches all teams for a user.
Args:
user_name_or_id: The name or ID of the user for which to get all
teams.
Returns:
A list of all teams that the user is part of.
"""
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
return [team.to_model() for team in user.teams]
def add_user_to_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Adds a user to a team.
Args:
user_name_or_id: Name or ID of the user to add to the team.
team_name_or_id: Name or ID of the team to which to add the user to.
Raises:
EntityExistsError: If the user is already a member of the team.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
user = self._get_user_schema(user_name_or_id, session=session)
# Check if user is already in the team
existing_user_in_team = session.exec(
select(TeamAssignmentSchema)
.where(TeamAssignmentSchema.user_id == user.id)
.where(TeamAssignmentSchema.team_id == team.id)
).first()
if existing_user_in_team is not None:
raise EntityExistsError(
f"Unable to add user '{user.name}' to team "
f"'{team.name}': User is already in the team."
)
# Add user to team
team.users = team.users + [user]
session.add(team)
session.commit()
def remove_user_from_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Removes a user from a team.
Args:
user_name_or_id: Name or ID of the user to remove from the team.
team_name_or_id: Name or ID of the team from which to remove the
user.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
user = self._get_user_schema(user_name_or_id, session=session)
# Remove user from team
team.users = [user_ for user_ in team.users if user_.id != user.id]
session.add(team)
session.commit()
# -----
# Roles
# -----
@track(AnalyticsEvent.CREATED_ROLE)
def create_role(self, role: RoleModel) -> RoleModel:
"""Creates a new role.
Args:
role: The role model to create.
Returns:
The newly created role.
Raises:
EntityExistsError: If a role with the given name already exists.
"""
with Session(self.engine) as session:
# Check if role with the given name already exists
existing_role = session.exec(
select(RoleSchema).where(RoleSchema.name == role.name)
).first()
if existing_role is not None:
raise EntityExistsError(
f"Unable to create role '{role.name}': Role already exists."
)
# Create role
role_schema = RoleSchema.from_create_model(role)
session.add(role_schema)
session.commit()
return role_schema.to_model()
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
"""
with Session(self.engine) as session:
role = self._get_role_schema(role_name_or_id, session=session)
return role.to_model()
def list_roles(self) -> List[RoleModel]:
"""List all roles.
Returns:
A list of all roles.
"""
with Session(self.engine) as session:
roles = session.exec(select(RoleSchema)).all()
return [role.to_model() for role in roles]
@track(AnalyticsEvent.UPDATED_ROLE)
def update_role(self, role: RoleModel) -> RoleModel:
"""Update an existing role.
Args:
role: The role to use for the update.
Returns:
The updated role.
Raises:
KeyError: if the role does not exist.
"""
with Session(self.engine) as session:
existing_role = session.exec(
select(RoleSchema).where(RoleSchema.id == role.id)
).first()
if existing_role is None:
raise KeyError(
f"Unable to update role with id "
f"'{role.id}': Found no"
f"existing roles with this id."
)
# Update the role
existing_role.from_update_model(role)
session.add(existing_role)
session.commit()
# Refresh the Model that was just created
session.refresh(existing_role)
return existing_role.to_model()
@track(AnalyticsEvent.DELETED_ROLE)
def delete_role(self, role_name_or_id: Union[str, UUID]) -> None:
"""Deletes a role.
Args:
role_name_or_id: Name or ID of the role to delete.
Raises:
IllegalOperationError: If the role is still assigned to users.
KeyError: If the role does not exist.
"""
with Session(self.engine) as session:
try:
role = self._get_role_schema(role_name_or_id, session=session)
user_role = session.exec(
select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.role_id == role.id
)
).all()
team_role = session.exec(
select(TeamRoleAssignmentSchema).where(
TeamRoleAssignmentSchema.role_id == role.id
)
).all()
if len(user_role) > 0 or len(team_role) > 0:
# TODO: Eventually we might want to allow this deletion
# and simply cascade
raise IllegalOperationError(
f"Role `{role.name}` of type can not be "
f"deleted as it is in use by multiple users and teams. "
f"Before deleting this role make sure to remove all "
f"instances where this role is used."
)
else:
# Delete role
session.delete(role)
session.commit()
except NoResultFound as error:
raise KeyError from error
# ----------------
# Role assignments
# ----------------
def _list_user_role_assignments(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
) -> List[RoleAssignmentModel]:
"""List all user role assignments.
Args:
project_name_or_id: If provided, only return role assignments for
this project.
user_name_or_id: If provided, only list assignments for this user.
Returns:
A list of user role assignments.
"""
with Session(self.engine) as session:
query = select(UserRoleAssignmentSchema)
if project_name_or_id is not None:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(
UserRoleAssignmentSchema.project_id == project.id
)
if user_name_or_id is not None:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(UserRoleAssignmentSchema.user_id == user.id)
assignments = session.exec(query).all()
return [assignment.to_model() for assignment in assignments]
def _list_team_role_assignments(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
team_name_or_id: Optional[Union[str, UUID]] = None,
) -> List[RoleAssignmentModel]:
"""List all team role assignments.
Args:
project_name_or_id: If provided, only return role assignments for
this project.
team_name_or_id: If provided, only list assignments for this team.
Returns:
A list of team role assignments.
"""
with Session(self.engine) as session:
query = select(TeamRoleAssignmentSchema)
if project_name_or_id is not None:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(
TeamRoleAssignmentSchema.project_id == project.id
)
if team_name_or_id is not None:
team = self._get_team_schema(team_name_or_id, session=session)
query = query.where(TeamRoleAssignmentSchema.team_id == team.id)
assignments = session.exec(query).all()
return [assignment.to_model() for assignment in assignments]
def list_role_assignments(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
team_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
) -> List[RoleAssignmentModel]:
"""List all role assignments.
Args:
project_name_or_id: If provided, only return role assignments for
this project.
team_name_or_id: If provided, only list assignments for this team.
user_name_or_id: If provided, only list assignments for this user.
Returns:
A list of all role assignments.
"""
user_role_assignments = self._list_user_role_assignments(
project_name_or_id=project_name_or_id,
user_name_or_id=user_name_or_id,
)
team_role_assignments = self._list_team_role_assignments(
project_name_or_id=project_name_or_id,
team_name_or_id=team_name_or_id,
)
return user_role_assignments + team_role_assignments
def _assign_role_to_user(
self,
role_name_or_id: Union[str, UUID],
user_name_or_id: Union[str, UUID],
project_name_or_id: Optional[Union[str, UUID]] = None,
) -> None:
"""Assigns a role to a user, potentially scoped to a specific project.
Args:
project_name_or_id: Optional ID of a project in which to assign the
role. If this is not provided, the role will be assigned
globally.
role_name_or_id: Name or ID of the role to assign.
user_name_or_id: Name or ID of the user to which to assign the role.
Raises:
EntityExistsError: If the role assignment already exists.
"""
with Session(self.engine) as session:
role = self._get_role_schema(role_name_or_id, session=session)
project: Optional[ProjectSchema] = None
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
user = self._get_user_schema(user_name_or_id, session=session)
query = select(UserRoleAssignmentSchema).where(
UserRoleAssignmentSchema.user_id == user.id,
UserRoleAssignmentSchema.role_id == role.id,
)
if project is not None:
query = query.where(
UserRoleAssignmentSchema.project_id == project.id
)
existing_role_assignment = session.exec(query).first()
if existing_role_assignment is not None:
raise EntityExistsError(
f"Unable to assign role '{role.name}' to user "
f"'{user.name}': Role already assigned in this project."
)
role_assignment = UserRoleAssignmentSchema(
role_id=role.id,
user_id=user.id,
project_id=project.id if project else None,
role=role,
user=user,
project=project,
)
session.add(role_assignment)
session.commit()
def _assign_role_to_team(
self,
role_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
project_name_or_id: Optional[Union[str, UUID]] = None,
) -> None:
"""Assigns a role to a team, potentially scoped to a specific project.
Args:
role_name_or_id: Name or ID of the role to assign.
team_name_or_id: Name or ID of the team to which to assign the role.
project_name_or_id: Optional ID of a project in which to assign the
role. If this is not provided, the role will be assigned
globally.
Raises:
EntityExistsError: If the role assignment already exists.
"""
with Session(self.engine) as session:
role = self._get_role_schema(role_name_or_id, session=session)
project: Optional[ProjectSchema] = None
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
team = self._get_team_schema(team_name_or_id, session=session)
query = select(TeamRoleAssignmentSchema).where(
TeamRoleAssignmentSchema.team_id == team.id,
TeamRoleAssignmentSchema.role_id == role.id,
)
if project is not None:
query = query.where(
TeamRoleAssignmentSchema.project_id == project.id
)
existing_role_assignment = session.exec(query).first()
if existing_role_assignment is not None:
raise EntityExistsError(
f"Unable to assign role '{role.name}' to team "
f"'{team.name}': Role already assigned in this project."
)
role_assignment = TeamRoleAssignmentSchema(
role_id=role.id,
team_id=team.id,
project_id=project.id if project else None,
role=role,
team=team,
project=project,
)
session.add(role_assignment)
session.commit()
def assign_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
project_name_or_id: Optional[Union[str, UUID]] = None,
is_user: bool = True,
) -> None:
"""Assigns a role to a user or team, scoped to a specific project.
Args:
project_name_or_id: Optional ID of a project in which to assign the
role. If this is not provided, the role will be assigned
globally.
role_name_or_id: Name or ID of the role to assign.
user_or_team_name_or_id: Name or ID of the user or team to which to
assign the role.
is_user: Whether `user_or_team_name_or_id` refers to a user or a
team.
"""
if is_user:
self._assign_role_to_user(
role_name_or_id=role_name_or_id,
user_name_or_id=user_or_team_name_or_id,
project_name_or_id=project_name_or_id,
)
else:
self._assign_role_to_team(
role_name_or_id=role_name_or_id,
team_name_or_id=user_or_team_name_or_id,
project_name_or_id=project_name_or_id,
)
def revoke_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
is_user: bool = True,
project_name_or_id: Optional[Union[str, UUID]] = None,
) -> None:
"""Revokes a role from a user or team for a given project.
Args:
project_name_or_id: Optional ID of a project in which to revoke the
role. If this is not provided, the role will be revoked
globally.
role_name_or_id: Name or ID of the role to revoke.
user_or_team_name_or_id: Name or ID of the user or team from which
to revoke the role.
is_user: Whether `user_or_team_name_or_id` refers to a user or a
team.
Raises:
KeyError: If the role, user, team, or project does not exists.
"""
with Session(self.engine) as session:
project: Optional[ProjectSchema] = None
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
role = self._get_role_schema(role_name_or_id, session=session)
role_assignment: Optional[SQLModel] = None
if is_user:
user = self._get_user_schema(
user_or_team_name_or_id, session=session
)
assignee_name = user.name
user_role_query = (
select(UserRoleAssignmentSchema)
.where(UserRoleAssignmentSchema.user_id == user.id)
.where(UserRoleAssignmentSchema.role_id == role.id)
)
if project:
user_role_query = user_role_query.where(
UserRoleAssignmentSchema.project_id == project.id
)
role_assignment = session.exec(user_role_query).first()
else:
team = self._get_team_schema(
user_or_team_name_or_id, session=session
)
assignee_name = team.name
team_role_query = (
select(TeamRoleAssignmentSchema)
.where(TeamRoleAssignmentSchema.team_id == team.id)
.where(TeamRoleAssignmentSchema.role_id == role.id)
)
if project:
team_role_query = team_role_query.where(
TeamRoleAssignmentSchema.project_id == project.id
)
role_assignment = session.exec(team_role_query).first()
if role_assignment is None:
assignee = "user" if is_user else "team"
scope = f" in project '{project.name}'" if project else ""
raise KeyError(
f"Unable to unassign role '{role.name}' from {assignee} "
f"'{assignee_name}'{scope}: The role is currently not "
f"assigned to the {assignee}."
)
session.delete(role_assignment)
session.commit()
# --------
# Projects
# --------
@track(AnalyticsEvent.CREATED_PROJECT)
def create_project(self, project: ProjectModel) -> ProjectModel:
"""Creates a new project.
Args:
project: The project to create.
Returns:
The newly created project.
Raises:
EntityExistsError: If a project with the given name already exists.
"""
with Session(self.engine) as session:
# Check if project with the given name already exists
existing_project = session.exec(
select(ProjectSchema).where(ProjectSchema.name == project.name)
).first()
if existing_project is not None:
raise EntityExistsError(
f"Unable to create project {project.name}: "
"A project with this name already exists."
)
# Create the project
new_project = ProjectSchema.from_create_model(project)
session.add(new_project)
session.commit()
# Explicitly refresh the new_project schema
session.refresh(new_project)
return new_project.to_model()
def get_project(self, project_name_or_id: Union[str, UUID]) -> ProjectModel:
"""Get an existing project by name or ID.
Args:
project_name_or_id: Name or ID of the project to get.
Returns:
The requested project if one was found.
"""
with Session(self.engine) as session:
project = self._get_project_schema(
project_name_or_id, session=session
)
return project.to_model()
def list_projects(self) -> List[ProjectModel]:
"""List all projects.
Returns:
A list of all projects.
"""
with Session(self.engine) as session:
projects = session.exec(select(ProjectSchema)).all()
return [project.to_model() for project in projects]
@track(AnalyticsEvent.UPDATED_PROJECT)
def update_project(self, project: ProjectModel) -> ProjectModel:
"""Update an existing project.
Args:
project: The project to use for the update.
Returns:
The updated project.
Raises:
KeyError: if the project does not exist.
"""
with Session(self.engine) as session:
existing_project = session.exec(
select(ProjectSchema).where(ProjectSchema.id == project.id)
).first()
if existing_project is None:
raise KeyError(
f"Unable to update project with id "
f"'{project.id}': Found no"
f"existing projects with this id."
)
# Update the project
existing_project.from_update_model(project)
session.add(existing_project)
session.commit()
# Refresh the Model that was just created
session.refresh(existing_project)
return existing_project.to_model()
@track(AnalyticsEvent.DELETED_PROJECT)
def delete_project(self, project_name_or_id: Union[str, UUID]) -> None:
"""Deletes a project.
Args:
project_name_or_id: Name or ID of the project to delete.
Raises:
KeyError: If the project does not exist.
"""
with Session(self.engine) as session:
try:
# Check if project with the given name exists
project = self._get_project_schema(
project_name_or_id, session=session
)
session.delete(project)
session.commit()
except NoResultFound as error:
raise KeyError from error
# ------------
# Repositories
# ------------
# ---------
# Pipelines
# ---------
@track(AnalyticsEvent.CREATE_PIPELINE)
def create_pipeline(
self,
pipeline: PipelineModel,
) -> PipelineModel:
"""Creates a new pipeline in a project.
Args:
pipeline: The pipeline to create.
Returns:
The newly created pipeline.
Raises:
EntityExistsError: If an identical pipeline already exists.
"""
with Session(self.engine) as session:
# Check if pipeline with the given name already exists
existing_pipeline = session.exec(
select(PipelineSchema)
.where(PipelineSchema.name == pipeline.name)
.where(PipelineSchema.project_id == pipeline.project)
).first()
if existing_pipeline is not None:
raise EntityExistsError(
f"Unable to create pipeline in project "
f"'{pipeline.project}': A pipeline with this name "
f"already exists."
)
# Create the pipeline
new_pipeline = PipelineSchema.from_create_model(pipeline=pipeline)
session.add(new_pipeline)
session.commit()
# Refresh the Model that was just created
session.refresh(new_pipeline)
return new_pipeline.to_model()
def get_pipeline(self, pipeline_id: UUID) -> PipelineModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
with Session(self.engine) as session:
# Check if pipeline with the given ID exists
pipeline = session.exec(
select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
).first()
if pipeline is None:
raise KeyError(
f"Unable to get pipeline with ID '{pipeline_id}': "
"No pipeline with this ID found."
)
return pipeline.to_model()
def list_pipelines(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
name: Optional[str] = None,
) -> List[PipelineModel]:
"""List all pipelines in the project.
Args:
project_name_or_id: If provided, only list pipelines in this
project.
user_name_or_id: If provided, only list pipelines from this user.
name: If provided, only list pipelines with this name.
Returns:
A list of pipelines.
"""
with Session(self.engine) as session:
# Check if project with the given name exists
query = select(PipelineSchema)
if project_name_or_id is not None:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(PipelineSchema.project_id == project.id)
if user_name_or_id is not None:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(PipelineSchema.user_id == user.id)
if name:
query = query.where(PipelineSchema.name == name)
# Get all pipelines in the project
pipelines = session.exec(query).all()
return [pipeline.to_model() for pipeline in pipelines]
@track(AnalyticsEvent.UPDATE_PIPELINE)
def update_pipeline(self, pipeline: PipelineModel) -> PipelineModel:
"""Updates a pipeline.
Args:
pipeline: The pipeline to use for the update.
Returns:
The updated pipeline.
Raises:
KeyError: if the pipeline doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline with the given ID exists
existing_pipeline = session.exec(
select(PipelineSchema).where(PipelineSchema.id == pipeline.id)
).first()
if existing_pipeline is None:
raise KeyError(
f"Unable to update pipeline with ID {pipeline.id}: "
f"No pipeline with this ID found."
)
# Update the pipeline
existing_pipeline.from_update_model(pipeline)
session.add(existing_pipeline)
session.commit()
return existing_pipeline.to_model()
@track(AnalyticsEvent.DELETE_PIPELINE)
def delete_pipeline(self, pipeline_id: UUID) -> None:
"""Deletes a pipeline.
Args:
pipeline_id: The ID of the pipeline to delete.
Raises:
KeyError: if the pipeline doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline with the given ID exists
pipeline = session.exec(
select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
).first()
if pipeline is None:
raise KeyError(
f"Unable to delete pipeline with ID {pipeline_id}: "
f"No pipeline with this ID found."
)
session.delete(pipeline)
session.commit()
# --------------
# Pipeline steps
# --------------
def list_steps(self, pipeline_id: UUID) -> List[StepRunModel]:
"""List all steps.
Args:
pipeline_id: The ID of the pipeline to list steps for.
"""
pass # TODO
# --------------
# Pipeline runs
# --------------
def get_run(self, run_id: UUID) -> PipelineRunModel:
"""Gets a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
Returns:
The pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
if not self.runs_inside_server:
self._sync_runs() # Sync with MLMD
with Session(self.engine) as session:
# Check if pipeline run with the given ID exists
run = session.exec(
select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
).first()
if run is None:
raise KeyError(
f"Unable to get pipeline run with ID {run_id}: "
f"No pipeline run with this ID found."
)
return run.to_model()
def get_run_component_side_effects(
self,
run_id: UUID,
component_id: Optional[UUID] = None,
) -> Dict[str, Any]:
"""Gets the side effects for a component in a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
component_id: The ID of the component to get.
"""
# TODO: raise KeyError if run doesn't exist
pass # TODO
def list_runs(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
stack_id: Optional[UUID] = None,
component_id: Optional[UUID] = None,
run_name: Optional[str] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
pipeline_id: Optional[UUID] = None,
unlisted: bool = False,
) -> List[PipelineRunModel]:
"""Gets all pipeline runs.
Args:
project_name_or_id: If provided, only return runs for this project.
stack_id: If provided, only return runs for this stack.
component_id: Optionally filter for runs that used the
component
run_name: Run name if provided
user_name_or_id: If provided, only return runs for this user.
pipeline_id: If provided, only return runs for this pipeline.
unlisted: If True, only return unlisted runs that are not
associated with any pipeline (filter by pipeline_id==None).
Returns:
A list of all pipeline runs.
"""
if not self.runs_inside_server:
self._sync_runs() # Sync with MLMD
with Session(self.engine) as session:
query = select(PipelineRunSchema)
if project_name_or_id is not None:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(StackSchema.project_id == project.id)
query = query.where(
PipelineRunSchema.stack_id == StackSchema.id
)
if stack_id is not None:
query = query.where(PipelineRunSchema.stack_id == stack_id)
if component_id:
query = query.where(
StackCompositionSchema.stack_id
== PipelineRunSchema.stack_id
).where(StackCompositionSchema.component_id == component_id)
if run_name is not None:
query = query.where(PipelineRunSchema.name == run_name)
if pipeline_id is not None:
query = query.where(
PipelineRunSchema.pipeline_id == pipeline_id
)
elif unlisted:
query = query.where(is_(PipelineRunSchema.pipeline_id, None))
if user_name_or_id is not None:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(PipelineRunSchema.user_id == user.id)
query = query.order_by(PipelineRunSchema.created)
runs = session.exec(query).all()
return [run.to_model() for run in runs]
def get_run_status(self, run_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a pipeline run.
Args:
run_id: The ID of the pipeline run to get the status for.
Returns:
The status of the pipeline run.
"""
steps = self.list_run_steps(run_id)
# If any step is failed or running, return that status respectively
for step in steps:
step_status = self.get_run_step_status(step.id)
if step_status == ExecutionStatus.FAILED:
return ExecutionStatus.FAILED
if step_status == ExecutionStatus.RUNNING:
return ExecutionStatus.RUNNING
# If not all steps have started yet, return running
if len(steps) < self.get_run(run_id).num_steps:
return ExecutionStatus.RUNNING
# Otherwise, return succeeded
return ExecutionStatus.COMPLETED
# ------------------
# Pipeline run steps
# ------------------
def get_run_step(self, step_id: UUID) -> StepRunModel:
"""Get a step by ID.
Args:
step_id: The ID of the step to get.
Returns:
The step.
Raises:
KeyError: if the step doesn't exist.
"""
with Session(self.engine) as session:
step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_id)
).first()
if step is None:
raise KeyError(
f"Unable to get step with ID {step_id}: No step with this "
"ID found."
)
parent_steps = session.exec(
select(StepRunSchema)
.where(StepRunOrderSchema.child_id == step.id)
.where(StepRunOrderSchema.parent_id == StepRunSchema.id)
).all()
parent_step_ids = [parent_step.id for parent_step in parent_steps]
mlmd_parent_step_ids = [
parent_step.mlmd_id for parent_step in parent_steps
]
step_model = step.to_model(parent_step_ids, mlmd_parent_step_ids)
return step_model
def get_run_step_outputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get the outputs for a specific step.
Args:
step_id: The id of the step to get outputs for.
Returns:
A dict mapping artifact names to the output artifacts for the step.
Raises:
KeyError: if the step doesn't exist.
"""
with Session(self.engine) as session:
step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_id)
).first()
if step is None:
raise KeyError(
f"Unable to get output artifacts for step with ID "
f"{step_id}: No step with this ID found."
)
artifacts = session.exec(
select(ArtifactSchema).where(
ArtifactSchema.parent_step_id == step_id
)
).all()
return {
artifact.name: artifact.to_model() for artifact in artifacts
}
def get_run_step_inputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get the inputs for a specific step.
Args:
step_id: The id of the step to get inputs for.
Returns:
A dict mapping artifact names to the input artifacts for the step.
Raises:
KeyError: if the step doesn't exist.
"""
with Session(self.engine) as session:
step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_id)
).first()
if step is None:
raise KeyError(
f"Unable to get input artifacts for step with ID "
f"{step_id}: No step with this ID found."
)
query_result = session.exec(
select(ArtifactSchema, StepInputArtifactSchema)
.where(ArtifactSchema.id == StepInputArtifactSchema.artifact_id)
.where(StepInputArtifactSchema.step_id == step_id)
).all()
return {
step_input_artifact.name: artifact.to_model()
for artifact, step_input_artifact in query_result
}
def get_run_step_status(self, step_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a single step.
Args:
step_id: The ID of the step to get the status for.
Returns:
ExecutionStatus: The status of the step.
"""
step = self.get_run_step(step_id)
return self.metadata_store.get_step_status(step.mlmd_id)
def list_run_steps(self, run_id: UUID) -> List[StepRunModel]:
"""Gets all steps in a pipeline run.
Args:
run_id: The ID of the pipeline run for which to list runs.
Returns:
A mapping from step names to step models for all steps in the run.
"""
if not self.runs_inside_server:
self._sync_run_steps(run_id)
with Session(self.engine) as session:
steps = session.exec(
select(StepRunSchema).where(
StepRunSchema.pipeline_run_id == run_id
)
).all()
return [self.get_run_step(step.id) for step in steps]
def list_artifacts(
self, artifact_uri: Optional[str] = None
) -> List[ArtifactModel]:
"""Lists all artifacts.
Args:
artifact_uri: If specified, only artifacts with the given URI will
be returned.
Returns:
A list of all artifacts.
"""
if not self.runs_inside_server:
self._sync_runs()
with Session(self.engine) as session:
query = select(ArtifactSchema)
if artifact_uri is not None:
query = query.where(ArtifactSchema.uri == artifact_uri)
artifacts = session.exec(query).all()
return [artifact.to_model() for artifact in artifacts]
# =======================
# Internal helper methods
# =======================
def _get_schema_by_name_or_id(
self,
object_name_or_id: Union[str, UUID],
schema_class: Type[SQLModel],
schema_name: str,
session: Session,
) -> SQLModel:
"""Query a schema by its 'name' or 'id' field.
Args:
object_name_or_id: The name or ID of the object to query.
schema_class: The schema class to query. E.g., `ProjectSchema`.
schema_name: The name of the schema used for error messages.
E.g., "project".
session: The database session to use.
Returns:
The schema object.
Raises:
KeyError: if the object couldn't be found.
ValueError: if the schema_name isn't provided.
"""
if object_name_or_id is None:
raise ValueError(
f"Unable to get {schema_name}: No {schema_name} ID or name "
"provided."
)
if uuid_utils.is_valid_uuid(object_name_or_id):
filter = schema_class.id == object_name_or_id # type: ignore[attr-defined]
error_msg = (
f"Unable to get {schema_name} with name or ID "
f"'{object_name_or_id}': No {schema_name} with this ID found."
)
else:
filter = schema_class.name == object_name_or_id # type: ignore[attr-defined]
error_msg = (
f"Unable to get {schema_name} with name or ID "
f"'{object_name_or_id}': '{object_name_or_id}' is not a valid "
f" UUID and no {schema_name} with this name exists."
)
schema = session.exec(select(schema_class).where(filter)).first()
if schema is None:
raise KeyError(error_msg)
return schema
def _get_project_schema(
self,
project_name_or_id: Union[str, UUID],
session: Session,
) -> ProjectSchema:
"""Gets a project schema by name or ID.
This is a helper method that is used in various places to find the
project associated to some other object.
Args:
project_name_or_id: The name or ID of the project to get.
session: The database session to use.
Returns:
The project schema.
"""
return cast(
ProjectSchema,
self._get_schema_by_name_or_id(
object_name_or_id=project_name_or_id,
schema_class=ProjectSchema,
schema_name="project",
session=session,
),
)
def _get_user_schema(
self,
user_name_or_id: Union[str, UUID],
session: Session,
) -> UserSchema:
"""Gets a user schema by name or ID.
This is a helper method that is used in various places to find the
user associated to some other object.
Args:
user_name_or_id: The name or ID of the user to get.
session: The database session to use.
Returns:
The user schema.
"""
return cast(
UserSchema,
self._get_schema_by_name_or_id(
object_name_or_id=user_name_or_id,
schema_class=UserSchema,
schema_name="user",
session=session,
),
)
def _get_team_schema(
self,
team_name_or_id: Union[str, UUID],
session: Session,
) -> TeamSchema:
"""Gets a team schema by name or ID.
This is a helper method that is used in various places to find a team
by its name or ID.
Args:
team_name_or_id: The name or ID of the team to get.
session: The database session to use.
Returns:
The team schema.
"""
return cast(
TeamSchema,
self._get_schema_by_name_or_id(
object_name_or_id=team_name_or_id,
schema_class=TeamSchema,
schema_name="team",
session=session,
),
)
def _get_role_schema(
self,
role_name_or_id: Union[str, UUID],
session: Session,
) -> RoleSchema:
"""Gets a role schema by name or ID.
This is a helper method that is used in various places to find a role
by its name or ID.
Args:
role_name_or_id: The name or ID of the role to get.
session: The database session to use.
Returns:
The role schema.
"""
return cast(
RoleSchema,
self._get_schema_by_name_or_id(
object_name_or_id=role_name_or_id,
schema_class=RoleSchema,
schema_name="role",
session=session,
),
)
# MLMD Stuff
def _resolve_mlmd_step_id(self, mlmd_id: int) -> UUID:
"""Resolves a step ID from MLMD to a ZenML step ID.
Args:
mlmd_id: The MLMD ID of the step.
Returns:
The ZenML step ID.
Raises:
KeyError: if the step couldn't be found.
"""
with Session(self.engine) as session:
step = session.exec(
select(StepRunSchema).where(StepRunSchema.mlmd_id == mlmd_id)
).first()
if step is None:
raise KeyError(
f"Unable to resolve MLMD step ID {mlmd_id}: "
f"No step with this ID found."
)
return step.id
def _resolve_mlmd_artifact_id(
self, mlmd_id: int, mlmd_parent_step_id: int
) -> UUID:
"""Resolves an artifact ID from MLMD to a ZenML artifact ID.
Since a single MLMD artifact can map to multiple ZenML artifacts, we
also need to know the parent step to resolve this correctly.
Args:
mlmd_id: The MLMD ID of the artifact.
mlmd_parent_step_id: The MLMD ID of the parent step.
Returns:
The ZenML artifact ID.
Raises:
KeyError: if the artifact couldn't be found.
"""
with Session(self.engine) as session:
artifact = session.exec(
select(ArtifactSchema)
.where(ArtifactSchema.mlmd_id == mlmd_id)
.where(
ArtifactSchema.mlmd_parent_step_id == mlmd_parent_step_id
)
).first()
if artifact is None:
raise KeyError(
f"Unable to resolve MLMD artifact ID {mlmd_id}: "
f"No artifact with this ID found."
)
return artifact.id
def _sync_runs(self) -> None:
"""Sync runs from MLMD into the database.
This queries all runs from MLMD, checks for each whether it already
exists in the database, and if not, creates it.
"""
# Get all runs from ZenML.
with Session(self.engine) as session:
zenml_runs_list = session.exec(select(PipelineRunSchema)).all()
zenml_runs = {run.name: run.to_model() for run in zenml_runs_list}
# Get all runs from MLMD.
mlmd_runs = self.metadata_store.get_all_runs()
# Sync all MLMD runs that don't exist in ZenML.
for run_name, mlmd_run in mlmd_runs.items():
# If the run is in MLMD but not in ZenML, we create it
if run_name not in zenml_runs:
new_run = PipelineRunModel(
name=run_name,
mlmd_id=mlmd_run.mlmd_id,
project=mlmd_run.project,
user=mlmd_run.user,
stack_id=mlmd_run.stack_id,
pipeline_id=mlmd_run.pipeline_id,
pipeline_configuration=mlmd_run.pipeline_configuration,
num_steps=mlmd_run.num_steps,
)
new_run = self._create_run(new_run)
zenml_runs[run_name] = new_run
for run_ in zenml_runs.values():
self._sync_run_steps(run_.id)
def _sync_run_steps(self, run_id: UUID) -> None:
"""Sync run steps from MLMD into the database.
Since we do not allow to create steps in the database directly, this is
a one-way sync from MLMD to the database.
Args:
run_id: The ID of the pipeline run to sync steps for.
Raises:
KeyError: if the run couldn't be found.
"""
# Get all steps from ZenML.
with Session(self.engine) as session:
run = session.exec(
select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
).first()
if run is None:
raise KeyError(
f"Unable to sync run steps for run with ID {run_id}: "
f"No run with this ID found."
)
zenml_steps_list = session.exec(
select(StepRunSchema).where(
StepRunSchema.pipeline_run_id == run_id
)
).all()
zenml_steps = {
step.name: self.get_run_step(step.id) for step in zenml_steps_list
}
# Get all steps from MLMD.
mlmd_steps = self.metadata_store.get_pipeline_run_steps(run.mlmd_id)
# For each step in MLMD, sync it into ZenML if it doesn't exist yet.
for step_name, mlmd_step in mlmd_steps.items():
if step_name not in zenml_steps:
docstring = mlmd_step.step_configuration["config"]["docstring"]
new_step = StepRunModel(
name=step_name,
mlmd_id=mlmd_step.mlmd_id,
mlmd_parent_step_ids=mlmd_step.mlmd_parent_step_ids,
entrypoint_name=mlmd_step.entrypoint_name,
parameters=mlmd_step.parameters,
step_configuration=mlmd_step.step_configuration,
docstring=docstring,
pipeline_run_id=run_id,
parent_step_ids=[
self._resolve_mlmd_step_id(parent_step_id)
for parent_step_id in mlmd_step.mlmd_parent_step_ids
],
)
new_step = self._create_run_step(new_step)
zenml_steps[step_name] = new_step
# Save parent step IDs into the database.
for step in zenml_steps.values():
for parent_step_id in step.parent_step_ids:
self._set_parent_step(
child_id=step.id, parent_id=parent_step_id
)
# Sync Artifacts.
for step in zenml_steps.values():
self._sync_run_step_artifacts(step.id)
def _sync_run_step_artifacts(self, run_step_id: UUID) -> None:
"""Sync run step artifacts from MLMD into the database.
Since we do not allow to create artifacts in the database directly, this
is a one-way sync from MLMD to the database.
Args:
run_step_id: The ID of the step run to sync artifacts for.
"""
# Get all ZenML artifacts.
zenml_inputs = self.get_run_step_inputs(run_step_id)
zenml_outputs = self.get_run_step_outputs(run_step_id)
# Get all MLMD artifacts.
step_model = self.get_run_step(run_step_id)
mlmd_inputs, mlmd_outputs = self.metadata_store.get_step_artifacts(
step_id=step_model.mlmd_id,
step_parent_step_ids=step_model.mlmd_parent_step_ids,
step_name=step_model.entrypoint_name,
)
# For each output in MLMD, sync it into ZenML if it doesn't exist yet.
for output_name, mlmd_artifact in mlmd_outputs.items():
if output_name not in zenml_outputs:
new_artifact = ArtifactModel(
name=output_name,
mlmd_id=mlmd_artifact.mlmd_id,
type=mlmd_artifact.type,
uri=mlmd_artifact.uri,
materializer=mlmd_artifact.materializer,
data_type=mlmd_artifact.data_type,
mlmd_parent_step_id=mlmd_artifact.mlmd_parent_step_id,
mlmd_producer_step_id=mlmd_artifact.mlmd_producer_step_id,
is_cached=mlmd_artifact.is_cached,
parent_step_id=self._resolve_mlmd_step_id(
mlmd_artifact.mlmd_parent_step_id
),
producer_step_id=self._resolve_mlmd_step_id(
mlmd_artifact.mlmd_producer_step_id
),
)
self._create_run_step_artifact(new_artifact)
# For each input in MLMD, sync it into ZenML if it doesn't exist yet.
for input_name, mlmd_artifact in mlmd_inputs.items():
if input_name not in zenml_inputs:
artifact_id = self._resolve_mlmd_artifact_id(
mlmd_id=mlmd_artifact.mlmd_id,
mlmd_parent_step_id=mlmd_artifact.mlmd_parent_step_id,
)
self._set_run_step_input_artifact(
step_id=run_step_id,
artifact_id=artifact_id,
name=input_name,
)
def _create_run(self, pipeline_run: PipelineRunModel) -> PipelineRunModel:
"""Creates a pipeline run.
Args:
pipeline_run: The pipeline run to create.
Returns:
The created pipeline run.
Raises:
EntityExistsError: If an identical pipeline run already exists.
KeyError: If the pipeline does not exist.
"""
# TODO: fix for when creating without associating to a project
with Session(self.engine) as session:
# Check if pipeline run already exists
existing_domain_run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.name == pipeline_run.name
)
).first()
if existing_domain_run is not None:
raise EntityExistsError(
f"Unable to create pipeline run {pipeline_run.name}: "
f"A pipeline run with this name already exists."
)
existing_id_run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.id == pipeline_run.id
)
).first()
if existing_id_run is not None:
raise EntityExistsError(
f"Unable to create pipeline run {pipeline_run.id}: "
f"A pipeline run with this id already exists."
)
# Query pipeline
if pipeline_run.pipeline_id is not None:
pipeline = session.exec(
select(PipelineSchema).where(
PipelineSchema.id == pipeline_run.pipeline_id
)
).first()
if pipeline is None:
raise KeyError(
f"Unable to create pipeline run: {pipeline_run.name}: "
f"No pipeline with ID {pipeline_run.pipeline_id} found."
)
new_run = PipelineRunSchema.from_create_model(
run=pipeline_run, pipeline=pipeline
)
else:
new_run = PipelineRunSchema.from_create_model(run=pipeline_run)
# Create the pipeline run
session.add(new_run)
session.commit()
return new_run.to_model()
def _update_run(self, run: PipelineRunModel) -> PipelineRunModel:
"""Updates a pipeline run.
Args:
run: The pipeline run to use for the update.
Returns:
The updated pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
with Session(self.engine) as session:
# Check if pipeline run with the given ID exists
existing_run = session.exec(
select(PipelineRunSchema).where(PipelineRunSchema.id == run.id)
).first()
if existing_run is None:
raise KeyError(
f"Unable to update pipeline run with ID {run.id}: "
f"No pipeline run with this ID found."
)
# Update the pipeline run
existing_run.from_update_model(run)
session.add(existing_run)
session.commit()
session.refresh(existing_run)
return existing_run.to_model()
def _create_run_step(self, step: StepRunModel) -> StepRunModel:
"""Creates a step.
Args:
step: The step to create.
Returns:
The created step.
Raises:
EntityExistsError: if the step already exists.
KeyError: if the pipeline run doesn't exist.
"""
with Session(self.engine) as session:
# Check if the step already exists
existing_step = session.exec(
select(StepRunSchema).where(
StepRunSchema.mlmd_id == step.mlmd_id
)
).first()
if existing_step is not None:
raise EntityExistsError(
f"Unable to create step '{step.name}': A step with MLMD ID "
f"'{step.mlmd_id}' already exists."
)
# Check if the pipeline run exists
run = session.exec(
select(PipelineRunSchema).where(
PipelineRunSchema.id == step.pipeline_run_id
)
).first()
if run is None:
raise KeyError(
f"Unable to create step '{step.name}': No pipeline run "
f"with ID '{step.pipeline_run_id}' found."
)
# Check if the step name already exists in the pipeline run
existing_step = session.exec(
select(StepRunSchema)
.where(StepRunSchema.name == step.name)
.where(StepRunSchema.pipeline_run_id == step.pipeline_run_id)
).first()
if existing_step is not None:
raise EntityExistsError(
f"Unable to create step '{step.name}': A step with this "
f"name already exists in the pipeline run with ID "
f"'{step.pipeline_run_id}'."
)
# Create the step
step_schema = StepRunSchema.from_create_model(step)
session.add(step_schema)
session.commit()
assert step.parent_step_ids is not None
return step_schema.to_model(
parent_step_ids=step.parent_step_ids,
mlmd_parent_step_ids=step.mlmd_parent_step_ids,
)
def _set_parent_step(self, child_id: UUID, parent_id: UUID) -> None:
"""Sets the parent step for a step.
Args:
child_id: The ID of the child step to set the parent for.
parent_id: The ID of the parent step to set a child for.
Raises:
KeyError: if the child step or parent step doesn't exist.
"""
with Session(self.engine) as session:
# Check if the child step exists.
child_step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == child_id)
).first()
if child_step is None:
raise KeyError(
f"Unable to set parent step for step with ID "
f"{child_id}: No step with this ID found."
)
# Check if the parent step exists.
parent_step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == parent_id)
).first()
if parent_step is None:
raise KeyError(
f"Unable to set parent step for step with ID "
f"{child_id}: No parent step with ID {parent_id} "
"found."
)
# Check if the parent step is already set.
assignment = session.exec(
select(StepRunOrderSchema)
.where(StepRunOrderSchema.child_id == child_id)
.where(StepRunOrderSchema.parent_id == parent_id)
).first()
if assignment is not None:
return
# Save the parent step assignment in the database.
assignment = StepRunOrderSchema(
child_id=child_id, parent_id=parent_id
)
session.add(assignment)
session.commit()
def _create_run_step_artifact(
self, artifact: ArtifactModel
) -> ArtifactModel:
"""Creates an artifact of a step.
Args:
artifact: The artifact to create.
Returns:
The created artifact.
Raises:
KeyError: if the step doesn't exist.
"""
with Session(self.engine) as session:
# Check if the step exists
step = session.exec(
select(StepRunSchema).where(
StepRunSchema.id == artifact.parent_step_id
)
).first()
if step is None:
raise KeyError(
f"Unable to create artifact: Could not find parent step "
f"with ID '{artifact.parent_step_id}'."
)
# Create the artifact
artifact_schema = ArtifactSchema.from_create_model(artifact)
session.add(artifact_schema)
session.commit()
return artifact_schema.to_model()
def _set_run_step_input_artifact(
self, step_id: UUID, artifact_id: UUID, name: str
) -> None:
"""Sets an artifact as an input of a step.
Args:
step_id: The ID of the step.
artifact_id: The ID of the artifact.
name: The name of the input in the step.
Raises:
KeyError: if the step or artifact doesn't exist.
"""
with Session(self.engine) as session:
# Check if the step exists.
step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_id)
).first()
if step is None:
raise KeyError(
f"Unable to set input artifact: No step with ID "
f"'{step_id}' found."
)
# Check if the artifact exists.
artifact = session.exec(
select(ArtifactSchema).where(ArtifactSchema.id == artifact_id)
).first()
if artifact is None:
raise KeyError(
f"Unable to set input artifact: No artifact with ID "
f"'{artifact_id}' found."
)
# Save the input artifact assignment in the database.
step_input = StepInputArtifactSchema(
step_id=step_id,
artifact_id=artifact_id,
name=name,
)
session.add(step_input)
session.commit()
active_user_name: str
property
readonly
Gets the active username.
Returns:
| Type | Description |
|---|---|
str |
The active username. |
engine: Engine
property
readonly
The SQLAlchemy engine.
Returns:
| Type | Description |
|---|---|
Engine |
The SQLAlchemy engine. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the store is not initialized. |
metadata_store: MetadataStore
property
readonly
The metadata store.
Returns:
| Type | Description |
|---|---|
MetadataStore |
The metadata store. |
Exceptions:
| Type | Description |
|---|---|
ValueError |
If the store is not initialized. |
runs_inside_server: bool
property
readonly
Whether the store is running inside a server.
Returns:
| Type | Description |
|---|---|
bool |
Whether the store is running inside a server. |
CONFIG_TYPE (StoreConfiguration)
pydantic-model
SQL ZenML store configuration.
Attributes:
| Name | Type | Description |
|---|---|---|
type |
StoreType |
The type of the store. |
database |
Optional[str] |
database name. If not already present on the server, it will be created automatically on first access. |
username |
Optional[str] |
The database username. |
password |
Optional[str] |
The database password. |
ssl_ca |
Optional[str] |
certificate authority certificate. Required for SSL enabled authentication if the CA certificate is not part of the certificates shipped by the operating system. |
ssl_cert |
Optional[str] |
client certificate. Required for SSL enabled authentication if client certificates are used. |
ssl_key |
Optional[str] |
client certificate private key. Required for SSL enabled if client certificates are used. |
ssl_verify_server_cert |
bool |
set to verify the identity of the server against the provided server certificate. |
Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStoreConfiguration(StoreConfiguration):
"""SQL ZenML store configuration.
Attributes:
type: The type of the store.
database: database name. If not already present on the server, it will
be created automatically on first access.
username: The database username.
password: The database password.
ssl_ca: certificate authority certificate. Required for SSL
enabled authentication if the CA certificate is not part of the
certificates shipped by the operating system.
ssl_cert: client certificate. Required for SSL enabled
authentication if client certificates are used.
ssl_key: client certificate private key. Required for SSL
enabled if client certificates are used.
ssl_verify_server_cert: set to verify the identity of the server
against the provided server certificate.
"""
type: StoreType = StoreType.SQL
database: Optional[str] = None
username: Optional[str] = None
password: Optional[str] = None
ssl_ca: Optional[str] = None
ssl_cert: Optional[str] = None
ssl_key: Optional[str] = None
ssl_verify_server_cert: bool = False
pool_size: int = 20
max_overflow: int = 20
@root_validator
def _validate_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validate the SQL URL.
The validator also moves the MySQL username, password and database
parameters from the URL into the other configuration arguments, if they
are present in the URL.
Args:
values: The values to validate.
Returns:
The validated values.
Raises:
ValueError: If the URL is invalid or the SQL driver is not
supported.
"""
# flake8: noqa: C901
url = values.get("url")
if url is None:
return values
try:
sql_url = make_url(url)
except ArgumentError as e:
raise ValueError(
"Invalid SQL URL `%s`: %s. The URL must be in the format "
"`driver://[[username:password@]hostname:port]/database["
"?<extra-args>]`.",
url,
str(e),
)
if sql_url.drivername not in SQLDatabaseDriver.values():
raise ValueError(
"Invalid SQL driver value `%s`: The driver must be one of: %s.",
url,
", ".join(SQLDatabaseDriver.values()),
)
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
if (
sql_url.username
or sql_url.password
or sql_url.query
or sql_url.database is None
):
raise ValueError(
"Invalid SQLite URL `%s`: The URL must be in the "
"format `sqlite:///path/to/database.db`.",
url,
)
if values.get("username") or values.get("password"):
raise ValueError(
"Invalid SQLite configuration: The username and password "
"must not be set",
url,
)
values["database"] = sql_url.database
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
if sql_url.username:
values["username"] = sql_url.username
sql_url = sql_url._replace(username=None)
if sql_url.password:
values["password"] = sql_url.password
sql_url = sql_url._replace(password=None)
if sql_url.database:
values["database"] = sql_url.database
sql_url = sql_url._replace(database=None)
if sql_url.query:
for k, v in sql_url.query.items():
if k == "ssl_ca":
values["ssl_ca"] = v
elif k == "ssl_cert":
values["ssl_cert"] = v
elif k == "ssl_key":
values["ssl_key"] = v
elif k == "ssl_verify_server_cert":
values["ssl_verify_server_cert"] = v
else:
raise ValueError(
"Invalid MySQL URL query parameter `%s`: The "
"parameter must be one of: ssl_ca, ssl_cert, "
"ssl_key, or ssl_verify_server_cert.",
k,
)
sql_url = sql_url._replace(query={})
database = values.get("database")
if (
not values.get("username")
or not values.get("password")
or not database
):
raise ValueError(
"Invalid MySQL configuration: The username, password and "
"database must be set in the URL or as configuration "
"attributes",
)
regexp = r"^[^\\/?%*:|\"<>.-]{1,64}$"
match = re.match(regexp, database)
if not match:
raise ValueError(
f"The database name does not conform to the required "
f"format "
f"rules ({regexp}): {database}"
)
# Save the certificates in a secure location on disk
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
content = values.get(key)
if content and not os.path.isfile(content):
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, f"{key}.pem")
with open(file_path, "w") as f:
f.write(content)
file_path.chmod(0o600)
values[key] = str(file_path)
values["url"] = str(sql_url)
return values
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the metadata configuration for the SQL ZenML store.
Args:
expand_certs: Whether to expand the certificate paths to their
contents.
Returns:
The metadata configuration.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
from ml_metadata.proto.metadata_store_pb2 import MySQLDatabaseConfig
from tfx.orchestration import metadata
sql_url = make_url(self.url)
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
mlmd_config = metadata.sqlite_metadata_connection_config(
self.database
)
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
mlmd_config = metadata.mysql_metadata_connection_config(
host=sql_url.host,
port=sql_url.port or 3306,
database=self.database,
username=self.username,
password=self.password,
)
mlmd_ssl_options = {}
# Handle certificate params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if not ssl_setting:
continue
if expand_certs and os.path.isfile(ssl_setting):
with open(ssl_setting, "r") as f:
ssl_setting = f.read()
mlmd_ssl_options[key.lstrip("ssl_")] = ssl_setting
# Handle additional params
if mlmd_ssl_options:
mlmd_ssl_options[
"verify_server_cert"
] = self.ssl_verify_server_cert
mlmd_config.mysql.ssl_options.CopyFrom(
MySQLDatabaseConfig.SSLOptions(**mlmd_ssl_options)
)
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return mlmd_config
def get_sqlmodel_config(self) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
"""Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
The URL and connection arguments for the SQLModel engine.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
sql_url = make_url(self.url)
sqlalchemy_connect_args = {}
engine_args = {}
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
# The following default value is needed for sqlite to avoid the Error:
# sqlite3.ProgrammingError: SQLite objects created in a thread can
# only be used in that same thread.
sqlalchemy_connect_args = {"check_same_thread": False}
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
engine_args = {
"pool_size": self.pool_size,
"max_overflow": self.max_overflow,
}
sql_url = sql_url._replace(
drivername="mysql+pymysql",
username=self.username,
password=self.password,
database=self.database,
)
# Handle certificate params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if ssl_setting and os.path.isfile(ssl_setting):
if key == "ssl_cert":
sqlalchemy_connect_args["ssl_capath"] = ssl_setting
else:
sqlalchemy_connect_args[key] = ssl_setting
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return str(sql_url), sqlalchemy_connect_args, engine_args
class Config:
"""Pydantic configuration class."""
# Validate attributes when assigning them. We need to set this in order
# to have a mix of mutable and immutable attributes
validate_assignment = True
# Forbid extra attributes set in the class.
extra = "forbid"
Config
Pydantic configuration class.
Source code in zenml/zen_stores/sql_zen_store.py
class Config:
"""Pydantic configuration class."""
# Validate attributes when assigning them. We need to set this in order
# to have a mix of mutable and immutable attributes
validate_assignment = True
# Forbid extra attributes set in the class.
extra = "forbid"
get_metadata_config(self, expand_certs=False)
Get the metadata configuration for the SQL ZenML store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
expand_certs |
bool |
Whether to expand the certificate paths to their contents. |
False |
Returns:
| Type | Description |
|---|---|
ConnectionConfig |
The metadata configuration. |
Exceptions:
| Type | Description |
|---|---|
NotImplementedError |
If the SQL driver is not supported. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the metadata configuration for the SQL ZenML store.
Args:
expand_certs: Whether to expand the certificate paths to their
contents.
Returns:
The metadata configuration.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
from ml_metadata.proto.metadata_store_pb2 import MySQLDatabaseConfig
from tfx.orchestration import metadata
sql_url = make_url(self.url)
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
mlmd_config = metadata.sqlite_metadata_connection_config(
self.database
)
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
mlmd_config = metadata.mysql_metadata_connection_config(
host=sql_url.host,
port=sql_url.port or 3306,
database=self.database,
username=self.username,
password=self.password,
)
mlmd_ssl_options = {}
# Handle certificate params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if not ssl_setting:
continue
if expand_certs and os.path.isfile(ssl_setting):
with open(ssl_setting, "r") as f:
ssl_setting = f.read()
mlmd_ssl_options[key.lstrip("ssl_")] = ssl_setting
# Handle additional params
if mlmd_ssl_options:
mlmd_ssl_options[
"verify_server_cert"
] = self.ssl_verify_server_cert
mlmd_config.mysql.ssl_options.CopyFrom(
MySQLDatabaseConfig.SSLOptions(**mlmd_ssl_options)
)
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return mlmd_config
get_sqlmodel_config(self)
Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
| Type | Description |
|---|---|
Tuple[str, Dict[str, Any], Dict[str, Any]] |
The URL and connection arguments for the SQLModel engine. |
Exceptions:
| Type | Description |
|---|---|
NotImplementedError |
If the SQL driver is not supported. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_sqlmodel_config(self) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
"""Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
The URL and connection arguments for the SQLModel engine.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
sql_url = make_url(self.url)
sqlalchemy_connect_args = {}
engine_args = {}
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
# The following default value is needed for sqlite to avoid the Error:
# sqlite3.ProgrammingError: SQLite objects created in a thread can
# only be used in that same thread.
sqlalchemy_connect_args = {"check_same_thread": False}
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
engine_args = {
"pool_size": self.pool_size,
"max_overflow": self.max_overflow,
}
sql_url = sql_url._replace(
drivername="mysql+pymysql",
username=self.username,
password=self.password,
database=self.database,
)
# Handle certificate params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if ssl_setting and os.path.isfile(ssl_setting):
if key == "ssl_cert":
sqlalchemy_connect_args["ssl_capath"] = ssl_setting
else:
sqlalchemy_connect_args[key] = ssl_setting
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return str(sql_url), sqlalchemy_connect_args, engine_args
add_user_to_team(self, user_name_or_id, team_name_or_id)
Adds a user to a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user to add to the team. |
required |
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to which to add the user to. |
required |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If the user is already a member of the team. |
Source code in zenml/zen_stores/sql_zen_store.py
def add_user_to_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Adds a user to a team.
Args:
user_name_or_id: Name or ID of the user to add to the team.
team_name_or_id: Name or ID of the team to which to add the user to.
Raises:
EntityExistsError: If the user is already a member of the team.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
user = self._get_user_schema(user_name_or_id, session=session)
# Check if user is already in the team
existing_user_in_team = session.exec(
select(TeamAssignmentSchema)
.where(TeamAssignmentSchema.user_id == user.id)
.where(TeamAssignmentSchema.team_id == team.id)
).first()
if existing_user_in_team is not None:
raise EntityExistsError(
f"Unable to add user '{user.name}' to team "
f"'{team.name}': User is already in the team."
)
# Add user to team
team.users = team.users + [user]
session.add(team)
session.commit()
assign_role(self, role_name_or_id, user_or_team_name_or_id, project_name_or_id=None, is_user=True)
Assigns a role to a user or team, scoped to a specific project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Optional ID of a project in which to assign the role. If this is not provided, the role will be assigned globally. |
None |
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to assign. |
required |
user_or_team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user or team to which to assign the role. |
required |
is_user |
bool |
Whether |
True |
Source code in zenml/zen_stores/sql_zen_store.py
def assign_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
project_name_or_id: Optional[Union[str, UUID]] = None,
is_user: bool = True,
) -> None:
"""Assigns a role to a user or team, scoped to a specific project.
Args:
project_name_or_id: Optional ID of a project in which to assign the
role. If this is not provided, the role will be assigned
globally.
role_name_or_id: Name or ID of the role to assign.
user_or_team_name_or_id: Name or ID of the user or team to which to
assign the role.
is_user: Whether `user_or_team_name_or_id` refers to a user or a
team.
"""
if is_user:
self._assign_role_to_user(
role_name_or_id=role_name_or_id,
user_name_or_id=user_or_team_name_or_id,
project_name_or_id=project_name_or_id,
)
else:
self._assign_role_to_team(
role_name_or_id=role_name_or_id,
team_name_or_id=user_or_team_name_or_id,
project_name_or_id=project_name_or_id,
)
copy_local_store(config, path, load_config_path=None)
classmethod
Copy a local store to a new location.
Use this method to create a copy of a store database to a new location and return a new store configuration pointing to the database copy. This only applies to stores that use the local filesystem to store their data. Calling this method for remote stores simply returns the input store configuration unaltered.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The configuration of the store to copy. |
required |
path |
str |
The new local path where the store DB will be copied. |
required |
load_config_path |
Optional[pathlib.PurePath] |
path that will be used to load the copied store
database. This can be set to a value different from |
None |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
The store configuration of the copied store. |
Source code in zenml/zen_stores/sql_zen_store.py
@classmethod
def copy_local_store(
cls,
config: StoreConfiguration,
path: str,
load_config_path: Optional[PurePath] = None,
) -> StoreConfiguration:
"""Copy a local store to a new location.
Use this method to create a copy of a store database to a new location
and return a new store configuration pointing to the database copy. This
only applies to stores that use the local filesystem to store their
data. Calling this method for remote stores simply returns the input
store configuration unaltered.
Args:
config: The configuration of the store to copy.
path: The new local path where the store DB will be copied.
load_config_path: path that will be used to load the copied store
database. This can be set to a value different from `path`
if the local database copy will be loaded from a different
environment, e.g. when the database is copied to a container
image and loaded using a different absolute path. This will be
reflected in the paths and URLs encoded in the copied store
configuration.
Returns:
The store configuration of the copied store.
"""
config_copy = config.copy()
sql_url = make_url(config.url)
if sql_url.drivername != SQLDatabaseDriver.SQLITE:
# this is not a configuration backed by a local filesystem
return config_copy
assert sql_url.database
local_path = Path(sql_url.database)
io_utils.create_dir_recursive_if_not_exists(path)
fileio.copy(
str(local_path), str(Path(path) / local_path.name), overwrite=True
)
if load_config_path:
config_copy.url = cls.get_local_url(str(load_config_path))
else:
config_copy.url = cls.get_local_url(path)
return config_copy
create_flavor(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_pipeline(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_project(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_role(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_stack(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_stack_component(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_team(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
create_user(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_flavor(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_pipeline(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_project(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_role(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_stack(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_stack_component(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_team(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
delete_user(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
fail_if_component_with_id_already_exists(component, session)
staticmethod
Raise an exception if a Component with the same id already exists.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
ComponentModel |
The Component |
required |
session |
Session |
The Session |
required |
Exceptions:
| Type | Description |
|---|---|
StackComponentExistsError |
If a component with the same id already exists |
Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def fail_if_component_with_id_already_exists(
component: ComponentModel, session: Session
) -> None:
"""Raise an exception if a Component with the same id already exists.
Args:
component: The Component
session: The Session
Raises:
StackComponentExistsError: If a component with the same id already
exists
"""
existing_id_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component.id
)
).first()
if existing_id_component is not None:
raise StackComponentExistsError(
f"Unable to register '{component.type.value}' component "
f"with name '{component.name}' and id '{component.id}': "
f" Found an existing component with the same id."
)
fail_if_component_with_name_type_already_shared(self, component, project, session)
Raise an exception if a Component with same name/type already shared.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
ComponentModel |
The Component |
required |
project |
ProjectSchema |
The project scope within which to check |
required |
session |
Session |
The Session |
required |
Exceptions:
| Type | Description |
|---|---|
StackComponentExistsError |
If a component with the given name and type is already shared by a user |
Source code in zenml/zen_stores/sql_zen_store.py
def fail_if_component_with_name_type_already_shared(
self,
component: ComponentModel,
project: ProjectSchema,
session: Session,
) -> None:
"""Raise an exception if a Component with same name/type already shared.
Args:
component: The Component
project: The project scope within which to check
session: The Session
Raises:
StackComponentExistsError: If a component with the given name and
type is already shared by a user
"""
# Check if component with the same name, type is already shared
# within the project
existing_shared_component = session.exec(
select(StackComponentSchema)
.where(StackComponentSchema.name == component.name)
.where(StackComponentSchema.project_id == component.project)
.where(StackComponentSchema.is_shared == component.is_shared)
.where(StackComponentSchema.type == component.type)
).first()
if existing_shared_component is not None:
owner_of_shared = self._get_user_schema(
existing_shared_component.user_id, session=session
)
raise StackComponentExistsError(
f"Unable to shared component of type '{component.type.value}' "
f"with name '{component.name}': Found an "
f"existing component with the same name and type in project "
f"'{project.name}' shared by "
f"'{owner_of_shared.name}'."
)
fail_if_component_with_name_type_exists_for_user(component, project, session, user)
staticmethod
Raise an exception if a Component with same name/type exists for user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
ComponentModel |
The Component |
required |
project |
ProjectSchema |
The project scope within which to check |
required |
user |
UserSchema |
The user that owns the Component |
required |
session |
Session |
The Session |
required |
Returns:
| Type | Description |
|---|---|
None |
None |
Exceptions:
| Type | Description |
|---|---|
StackComponentExistsError |
If a component with the given name and type is already owned by the user |
Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def fail_if_component_with_name_type_exists_for_user(
component: ComponentModel,
project: ProjectSchema,
session: Session,
user: UserSchema,
) -> None:
"""Raise an exception if a Component with same name/type exists for user.
Args:
component: The Component
project: The project scope within which to check
user: The user that owns the Component
session: The Session
Returns:
None
Raises:
StackComponentExistsError: If a component with the given name and
type is already owned by the user
"""
# Check if component with the same domain key (name, type, project,
# owner) already exists
existing_domain_component = session.exec(
select(StackComponentSchema)
.where(StackComponentSchema.name == component.name)
.where(StackComponentSchema.project_id == component.project)
.where(StackComponentSchema.user_id == component.user)
.where(StackComponentSchema.type == component.type)
).first()
if existing_domain_component is not None:
raise StackComponentExistsError(
f"Unable to register '{component.type.value}' component "
f"with name '{component.name}': Found an existing "
f"component with the same name and type in the same "
f" project, '{project.name}', owned by the same "
f" user, '{user.name}'."
)
return None
fail_if_stack_with_id_already_exists(stack, session)
staticmethod
Raise an exception if a Stack with the same id already exists.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack |
StackModel |
The Stack |
required |
session |
Session |
The Session |
required |
Exceptions:
| Type | Description |
|---|---|
StackExistsError |
If a stack with the same id already exists |
Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def fail_if_stack_with_id_already_exists(
stack: StackModel, session: Session
) -> None:
"""Raise an exception if a Stack with the same id already exists.
Args:
stack: The Stack
session: The Session
Raises:
StackExistsError: If a stack with the same id already
exists
"""
existing_id_stack = session.exec(
select(StackSchema).where(StackSchema.id == stack.id)
).first()
if existing_id_stack is not None:
raise StackExistsError(
f"Unable to register stack with name "
f"'{stack.name}' and id '{stack.id}': "
f" Found an existing component with the same id."
)
fail_if_stack_with_name_already_shared(self, stack, project, session)
Raise an exception if a Stack with same name is already shared.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack |
StackModel |
The Stack |
required |
project |
ProjectSchema |
The project scope within which to check |
required |
session |
Session |
The Session |
required |
Exceptions:
| Type | Description |
|---|---|
StackExistsError |
If a stack with the given name is already shared by a user. |
Source code in zenml/zen_stores/sql_zen_store.py
def fail_if_stack_with_name_already_shared(
self, stack: StackModel, project: ProjectSchema, session: Session
) -> None:
"""Raise an exception if a Stack with same name is already shared.
Args:
stack: The Stack
project: The project scope within which to check
session: The Session
Raises:
StackExistsError: If a stack with the given name is already shared
by a user.
"""
# Check if component with the same name, type is already shared
# within the project
existing_shared_stack = session.exec(
select(StackSchema)
.where(StackSchema.name == stack.name)
.where(StackSchema.project_id == stack.project)
.where(StackSchema.is_shared == stack.is_shared)
).first()
if existing_shared_stack is not None:
owner_of_shared = self._get_user_schema(
existing_shared_stack.user_id, session=session
)
raise StackExistsError(
f"Unable to share stack with name '{stack.name}': Found an "
f"existing stack with the same name in project "
f"'{project.name}' shared by '{owner_of_shared.name}'."
)
fail_if_stack_with_name_exists_for_user(stack, project, session, user)
staticmethod
Raise an exception if a Component with same name exists for user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack |
StackModel |
The Stack |
required |
project |
ProjectSchema |
The project scope within which to check |
required |
user |
UserSchema |
The user that owns the Stack |
required |
session |
Session |
The Session |
required |
Returns:
| Type | Description |
|---|---|
None |
None |
Exceptions:
| Type | Description |
|---|---|
StackExistsError |
If a Stack with the given name is already owned by the user |
Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def fail_if_stack_with_name_exists_for_user(
stack: StackModel,
project: ProjectSchema,
session: Session,
user: UserSchema,
) -> None:
"""Raise an exception if a Component with same name exists for user.
Args:
stack: The Stack
project: The project scope within which to check
user: The user that owns the Stack
session: The Session
Returns:
None
Raises:
StackExistsError: If a Stack with the given name is already
owned by the user
"""
existing_domain_stack = session.exec(
select(StackSchema)
.where(StackSchema.name == stack.name)
.where(StackSchema.project_id == stack.project)
.where(StackSchema.user_id == stack.user)
).first()
if existing_domain_stack is not None:
raise StackExistsError(
f"Unable to register stack with name "
f"'{stack.name}': Found an existing stack with the same "
f"name in the active project, '{project.name}', owned by the "
f"same user, '{user.name}'."
)
return None
get_flavor(self, flavor_id)
Get a flavor by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
UUID |
The ID of the flavor to fetch. |
required |
Returns:
| Type | Description |
|---|---|
FlavorModel |
The stack component flavor. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component flavor doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_flavor(self, flavor_id: UUID) -> FlavorModel:
"""Get a flavor by ID.
Args:
flavor_id: The ID of the flavor to fetch.
Returns:
The stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
with Session(self.engine) as session:
flavor_in_db = session.exec(
select(FlavorSchema).where(FlavorSchema.id == flavor_id)
).first()
if flavor_in_db is None:
raise KeyError(f"Flavor with ID {flavor_id} not found.")
return flavor_in_db.to_model()
get_local_url(path)
staticmethod
Get a local SQL url for a given local path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
The path to the local sqlite file. |
required |
Returns:
| Type | Description |
|---|---|
str |
The local SQL url for the given path. |
Source code in zenml/zen_stores/sql_zen_store.py
@staticmethod
def get_local_url(path: str) -> str:
"""Get a local SQL url for a given local path.
Args:
path: The path to the local sqlite file.
Returns:
The local SQL url for the given path.
"""
return f"sqlite:///{path}/{ZENML_SQLITE_DB_FILENAME}"
get_metadata_config(self, expand_certs=False)
Get the TFX metadata config of this ZenStore.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
expand_certs |
bool |
Whether to expand the certificate paths in the connection config to their value. |
False |
Returns:
| Type | Description |
|---|---|
ConnectionConfig |
The TFX metadata config of this ZenStore. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the TFX metadata config of this ZenStore.
Args:
expand_certs: Whether to expand the certificate paths in the
connection config to their value.
Returns:
The TFX metadata config of this ZenStore.
"""
return self.config.get_metadata_config(expand_certs=expand_certs)
get_pipeline(self, pipeline_id)
Get a pipeline with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
ID of the pipeline. |
required |
Returns:
| Type | Description |
|---|---|
PipelineModel |
The pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline does not exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_pipeline(self, pipeline_id: UUID) -> PipelineModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
with Session(self.engine) as session:
# Check if pipeline with the given ID exists
pipeline = session.exec(
select(PipelineSchema).where(PipelineSchema.id == pipeline_id)
).first()
if pipeline is None:
raise KeyError(
f"Unable to get pipeline with ID '{pipeline_id}': "
"No pipeline with this ID found."
)
return pipeline.to_model()
get_project(self, project_name_or_id)
Get an existing project by name or ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the project to get. |
required |
Returns:
| Type | Description |
|---|---|
ProjectModel |
The requested project if one was found. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_project(self, project_name_or_id: Union[str, UUID]) -> ProjectModel:
"""Get an existing project by name or ID.
Args:
project_name_or_id: Name or ID of the project to get.
Returns:
The requested project if one was found.
"""
with Session(self.engine) as session:
project = self._get_project_schema(
project_name_or_id, session=session
)
return project.to_model()
get_role(self, role_name_or_id)
Gets a specific role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to get. |
required |
Returns:
| Type | Description |
|---|---|
RoleModel |
The requested role. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
"""
with Session(self.engine) as session:
role = self._get_role_schema(role_name_or_id, session=session)
return role.to_model()
get_run(self, run_id)
Gets a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunModel |
The pipeline run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run(self, run_id: UUID) -> PipelineRunModel:
"""Gets a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
Returns:
The pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
if not self.runs_inside_server:
self._sync_runs() # Sync with MLMD
with Session(self.engine) as session:
# Check if pipeline run with the given ID exists
run = session.exec(
select(PipelineRunSchema).where(PipelineRunSchema.id == run_id)
).first()
if run is None:
raise KeyError(
f"Unable to get pipeline run with ID {run_id}: "
f"No pipeline run with this ID found."
)
return run.to_model()
get_run_component_side_effects(self, run_id, component_id=None)
Gets the side effects for a component in a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get. |
required |
component_id |
Optional[uuid.UUID] |
The ID of the component to get. |
None |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run_component_side_effects(
self,
run_id: UUID,
component_id: Optional[UUID] = None,
) -> Dict[str, Any]:
"""Gets the side effects for a component in a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
component_id: The ID of the component to get.
"""
# TODO: raise KeyError if run doesn't exist
pass # TODO
get_run_status(self, run_id)
Gets the execution status of a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get the status for. |
required |
Returns:
| Type | Description |
|---|---|
ExecutionStatus |
The status of the pipeline run. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run_status(self, run_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a pipeline run.
Args:
run_id: The ID of the pipeline run to get the status for.
Returns:
The status of the pipeline run.
"""
steps = self.list_run_steps(run_id)
# If any step is failed or running, return that status respectively
for step in steps:
step_status = self.get_run_step_status(step.id)
if step_status == ExecutionStatus.FAILED:
return ExecutionStatus.FAILED
if step_status == ExecutionStatus.RUNNING:
return ExecutionStatus.RUNNING
# If not all steps have started yet, return running
if len(steps) < self.get_run(run_id).num_steps:
return ExecutionStatus.RUNNING
# Otherwise, return succeeded
return ExecutionStatus.COMPLETED
get_run_step(self, step_id)
Get a step by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The ID of the step to get. |
required |
Returns:
| Type | Description |
|---|---|
StepRunModel |
The step. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the step doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run_step(self, step_id: UUID) -> StepRunModel:
"""Get a step by ID.
Args:
step_id: The ID of the step to get.
Returns:
The step.
Raises:
KeyError: if the step doesn't exist.
"""
with Session(self.engine) as session:
step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_id)
).first()
if step is None:
raise KeyError(
f"Unable to get step with ID {step_id}: No step with this "
"ID found."
)
parent_steps = session.exec(
select(StepRunSchema)
.where(StepRunOrderSchema.child_id == step.id)
.where(StepRunOrderSchema.parent_id == StepRunSchema.id)
).all()
parent_step_ids = [parent_step.id for parent_step in parent_steps]
mlmd_parent_step_ids = [
parent_step.mlmd_id for parent_step in parent_steps
]
step_model = step.to_model(parent_step_ids, mlmd_parent_step_ids)
return step_model
get_run_step_inputs(self, step_id)
Get the inputs for a specific step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The id of the step to get inputs for. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, zenml.models.pipeline_models.ArtifactModel] |
A dict mapping artifact names to the input artifacts for the step. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the step doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run_step_inputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get the inputs for a specific step.
Args:
step_id: The id of the step to get inputs for.
Returns:
A dict mapping artifact names to the input artifacts for the step.
Raises:
KeyError: if the step doesn't exist.
"""
with Session(self.engine) as session:
step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_id)
).first()
if step is None:
raise KeyError(
f"Unable to get input artifacts for step with ID "
f"{step_id}: No step with this ID found."
)
query_result = session.exec(
select(ArtifactSchema, StepInputArtifactSchema)
.where(ArtifactSchema.id == StepInputArtifactSchema.artifact_id)
.where(StepInputArtifactSchema.step_id == step_id)
).all()
return {
step_input_artifact.name: artifact.to_model()
for artifact, step_input_artifact in query_result
}
get_run_step_outputs(self, step_id)
Get the outputs for a specific step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The id of the step to get outputs for. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, zenml.models.pipeline_models.ArtifactModel] |
A dict mapping artifact names to the output artifacts for the step. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the step doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run_step_outputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get the outputs for a specific step.
Args:
step_id: The id of the step to get outputs for.
Returns:
A dict mapping artifact names to the output artifacts for the step.
Raises:
KeyError: if the step doesn't exist.
"""
with Session(self.engine) as session:
step = session.exec(
select(StepRunSchema).where(StepRunSchema.id == step_id)
).first()
if step is None:
raise KeyError(
f"Unable to get output artifacts for step with ID "
f"{step_id}: No step with this ID found."
)
artifacts = session.exec(
select(ArtifactSchema).where(
ArtifactSchema.parent_step_id == step_id
)
).all()
return {
artifact.name: artifact.to_model() for artifact in artifacts
}
get_run_step_status(self, step_id)
Gets the execution status of a single step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The ID of the step to get the status for. |
required |
Returns:
| Type | Description |
|---|---|
ExecutionStatus |
The status of the step. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_run_step_status(self, step_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a single step.
Args:
step_id: The ID of the step to get the status for.
Returns:
ExecutionStatus: The status of the step.
"""
step = self.get_run_step(step_id)
return self.metadata_store.get_step_status(step.mlmd_id)
get_stack(self, stack_id)
Get a stack by its unique ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack to get. |
required |
Returns:
| Type | Description |
|---|---|
StackModel |
The stack with the given ID. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_stack(self, stack_id: UUID) -> StackModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
Raises:
KeyError: if the stack doesn't exist.
"""
with Session(self.engine) as session:
stack = session.exec(
select(StackSchema).where(StackSchema.id == stack_id)
).first()
if stack is None:
raise KeyError(f"Stack with ID {stack_id} not found.")
return stack.to_model()
get_stack_component(self, component_id)
Get a stack component by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to get. |
required |
Returns:
| Type | Description |
|---|---|
ComponentModel |
The stack component. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_stack_component(self, component_id: UUID) -> ComponentModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
with Session(self.engine) as session:
stack_component = session.exec(
select(StackComponentSchema).where(
StackComponentSchema.id == component_id
)
).first()
if stack_component is None:
raise KeyError(
f"Stack component with ID {component_id} not found."
)
return stack_component.to_model()
get_stack_component_side_effects(self, component_id, run_id, pipeline_id, stack_id)
Get the side effects of a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The id of the stack component to get side effects for. |
required |
run_id |
UUID |
The id of the run to get side effects for. |
required |
pipeline_id |
UUID |
The id of the pipeline to get side effects for. |
required |
stack_id |
UUID |
The id of the stack to get side effects for. |
required |
Source code in zenml/zen_stores/sql_zen_store.py
def get_stack_component_side_effects(
self,
component_id: UUID,
run_id: UUID,
pipeline_id: UUID,
stack_id: UUID,
) -> Dict[Any, Any]:
"""Get the side effects of a stack component.
Args:
component_id: The id of the stack component to get side effects for.
run_id: The id of the run to get side effects for.
pipeline_id: The id of the pipeline to get side effects for.
stack_id: The id of the stack to get side effects for.
"""
pass # TODO: implement this
get_store_info(self)
Get information about the store.
Returns:
| Type | Description |
|---|---|
ServerModel |
Information about the store. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
model = super().get_store_info()
sql_url = make_url(self.config.url)
model.database_type = ServerDatabaseType(sql_url.drivername)
return model
get_team(self, team_name_or_id)
Gets a specific team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamModel |
The requested team. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
return team.to_model()
get_teams_for_user(self, user_name_or_id)
Fetches all teams for a user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user for which to get all teams. |
required |
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.TeamModel] |
A list of all teams that the user is part of. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_teams_for_user(
self, user_name_or_id: Union[str, UUID]
) -> List[TeamModel]:
"""Fetches all teams for a user.
Args:
user_name_or_id: The name or ID of the user for which to get all
teams.
Returns:
A list of all teams that the user is part of.
"""
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
return [team.to_model() for team in user.teams]
get_user(self, user_name_or_id)
Gets a specific user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
required |
Returns:
| Type | Description |
|---|---|
UserModel |
The requested user, if it was found. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_user(self, user_name_or_id: Union[str, UUID]) -> UserModel:
"""Gets a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
"""
with Session(self.engine) as session:
user = self._get_user_schema(user_name_or_id, session=session)
return user.to_model()
get_users_for_team(self, team_name_or_id)
Fetches all users of a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the team for which to get users. |
required |
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.UserModel] |
A list of all users that are part of the team. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_users_for_team(
self, team_name_or_id: Union[str, UUID]
) -> List[UserModel]:
"""Fetches all users of a team.
Args:
team_name_or_id: The name or ID of the team for which to get users.
Returns:
A list of all users that are part of the team.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
return [user.to_model() for user in team.users]
list_artifacts(self, artifact_uri=None)
Lists all artifacts.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_uri |
Optional[str] |
If specified, only artifacts with the given URI will be returned. |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.ArtifactModel] |
A list of all artifacts. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_artifacts(
self, artifact_uri: Optional[str] = None
) -> List[ArtifactModel]:
"""Lists all artifacts.
Args:
artifact_uri: If specified, only artifacts with the given URI will
be returned.
Returns:
A list of all artifacts.
"""
if not self.runs_inside_server:
self._sync_runs()
with Session(self.engine) as session:
query = select(ArtifactSchema)
if artifact_uri is not None:
query = query.where(ArtifactSchema.uri == artifact_uri)
artifacts = session.exec(query).all()
return [artifact.to_model() for artifact in artifacts]
list_flavors(self, project_name_or_id=None, user_name_or_id=None, component_type=None, name=None, is_shared=None)
List all stack component flavors matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Optionally filter by the Project to which the component flavors belong |
None |
component_type |
Optional[zenml.enums.StackComponentType] |
Optionally filter by type of stack component |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter by the owner |
None |
component_type |
Optional[zenml.enums.StackComponentType] |
Optionally filter by type of stack component |
None |
name |
Optional[str] |
Optionally filter flavors by name |
None |
is_shared |
Optional[bool] |
Optionally filter out flavors by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.flavor_models.FlavorModel] |
List of all the stack component flavors matching the given criteria |
Source code in zenml/zen_stores/sql_zen_store.py
def list_flavors(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_type: Optional[StackComponentType] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[FlavorModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
project_name_or_id: Optionally filter by the Project to which the
component flavors belong
component_type: Optionally filter by type of stack component
user_name_or_id: Optionally filter by the owner
component_type: Optionally filter by type of stack component
name: Optionally filter flavors by name
is_shared: Optionally filter out flavors by whether they are
shared or not
Returns:
List of all the stack component flavors matching the given criteria
"""
with Session(self.engine) as session:
query = select(FlavorSchema)
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(FlavorSchema.project_id == project.id)
if component_type:
query = query.where(FlavorSchema.type == component_type)
if name:
query = query.where(FlavorSchema.name == name)
if user_name_or_id:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(FlavorSchema.user_id == user.id)
list_of_flavors_in_db = session.exec(query).all()
return [flavor.to_model() for flavor in list_of_flavors_in_db]
list_pipelines(self, project_name_or_id=None, user_name_or_id=None, name=None)
List all pipelines in the project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only list pipelines in this project. |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only list pipelines from this user. |
None |
name |
Optional[str] |
If provided, only list pipelines with this name. |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.PipelineModel] |
A list of pipelines. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_pipelines(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
name: Optional[str] = None,
) -> List[PipelineModel]:
"""List all pipelines in the project.
Args:
project_name_or_id: If provided, only list pipelines in this
project.
user_name_or_id: If provided, only list pipelines from this user.
name: If provided, only list pipelines with this name.
Returns:
A list of pipelines.
"""
with Session(self.engine) as session:
# Check if project with the given name exists
query = select(PipelineSchema)
if project_name_or_id is not None:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(PipelineSchema.project_id == project.id)
if user_name_or_id is not None:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(PipelineSchema.user_id == user.id)
if name:
query = query.where(PipelineSchema.name == name)
# Get all pipelines in the project
pipelines = session.exec(query).all()
return [pipeline.to_model() for pipeline in pipelines]
list_projects(self)
List all projects.
Returns:
| Type | Description |
|---|---|
List[zenml.models.project_models.ProjectModel] |
A list of all projects. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_projects(self) -> List[ProjectModel]:
"""List all projects.
Returns:
A list of all projects.
"""
with Session(self.engine) as session:
projects = session.exec(select(ProjectSchema)).all()
return [project.to_model() for project in projects]
list_role_assignments(self, project_name_or_id=None, team_name_or_id=None, user_name_or_id=None)
List all role assignments.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only return role assignments for this project. |
None |
team_name_or_id |
Union[str, uuid.UUID] |
If provided, only list assignments for this team. |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only list assignments for this user. |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.RoleAssignmentModel] |
A list of all role assignments. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_role_assignments(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
team_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
) -> List[RoleAssignmentModel]:
"""List all role assignments.
Args:
project_name_or_id: If provided, only return role assignments for
this project.
team_name_or_id: If provided, only list assignments for this team.
user_name_or_id: If provided, only list assignments for this user.
Returns:
A list of all role assignments.
"""
user_role_assignments = self._list_user_role_assignments(
project_name_or_id=project_name_or_id,
user_name_or_id=user_name_or_id,
)
team_role_assignments = self._list_team_role_assignments(
project_name_or_id=project_name_or_id,
team_name_or_id=team_name_or_id,
)
return user_role_assignments + team_role_assignments
list_roles(self)
List all roles.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.RoleModel] |
A list of all roles. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_roles(self) -> List[RoleModel]:
"""List all roles.
Returns:
A list of all roles.
"""
with Session(self.engine) as session:
roles = session.exec(select(RoleSchema)).all()
return [role.to_model() for role in roles]
list_run_steps(self, run_id)
Gets all steps in a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run for which to list runs. |
required |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.StepRunModel] |
A mapping from step names to step models for all steps in the run. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_run_steps(self, run_id: UUID) -> List[StepRunModel]:
"""Gets all steps in a pipeline run.
Args:
run_id: The ID of the pipeline run for which to list runs.
Returns:
A mapping from step names to step models for all steps in the run.
"""
if not self.runs_inside_server:
self._sync_run_steps(run_id)
with Session(self.engine) as session:
steps = session.exec(
select(StepRunSchema).where(
StepRunSchema.pipeline_run_id == run_id
)
).all()
return [self.get_run_step(step.id) for step in steps]
list_runs(self, project_name_or_id=None, stack_id=None, component_id=None, run_name=None, user_name_or_id=None, pipeline_id=None, unlisted=False)
Gets all pipeline runs.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only return runs for this project. |
None |
stack_id |
Optional[uuid.UUID] |
If provided, only return runs for this stack. |
None |
component_id |
Optional[uuid.UUID] |
Optionally filter for runs that used the component |
None |
run_name |
Optional[str] |
Run name if provided |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only return runs for this user. |
None |
pipeline_id |
Optional[uuid.UUID] |
If provided, only return runs for this pipeline. |
None |
unlisted |
bool |
If True, only return unlisted runs that are not associated with any pipeline (filter by pipeline_id==None). |
False |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.PipelineRunModel] |
A list of all pipeline runs. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_runs(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
stack_id: Optional[UUID] = None,
component_id: Optional[UUID] = None,
run_name: Optional[str] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
pipeline_id: Optional[UUID] = None,
unlisted: bool = False,
) -> List[PipelineRunModel]:
"""Gets all pipeline runs.
Args:
project_name_or_id: If provided, only return runs for this project.
stack_id: If provided, only return runs for this stack.
component_id: Optionally filter for runs that used the
component
run_name: Run name if provided
user_name_or_id: If provided, only return runs for this user.
pipeline_id: If provided, only return runs for this pipeline.
unlisted: If True, only return unlisted runs that are not
associated with any pipeline (filter by pipeline_id==None).
Returns:
A list of all pipeline runs.
"""
if not self.runs_inside_server:
self._sync_runs() # Sync with MLMD
with Session(self.engine) as session:
query = select(PipelineRunSchema)
if project_name_or_id is not None:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(StackSchema.project_id == project.id)
query = query.where(
PipelineRunSchema.stack_id == StackSchema.id
)
if stack_id is not None:
query = query.where(PipelineRunSchema.stack_id == stack_id)
if component_id:
query = query.where(
StackCompositionSchema.stack_id
== PipelineRunSchema.stack_id
).where(StackCompositionSchema.component_id == component_id)
if run_name is not None:
query = query.where(PipelineRunSchema.name == run_name)
if pipeline_id is not None:
query = query.where(
PipelineRunSchema.pipeline_id == pipeline_id
)
elif unlisted:
query = query.where(is_(PipelineRunSchema.pipeline_id, None))
if user_name_or_id is not None:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(PipelineRunSchema.user_id == user.id)
query = query.order_by(PipelineRunSchema.created)
runs = session.exec(query).all()
return [run.to_model() for run in runs]
list_stack_components(self, project_name_or_id=None, user_name_or_id=None, type=None, flavor_name=None, name=None, is_shared=None)
List all stack components matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
The ID or name of the Project to which the stack components belong |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter stack components by the owner |
None |
type |
Optional[str] |
Optionally filter by type of stack component |
None |
flavor_name |
Optional[str] |
Optionally filter by flavor |
None |
name |
Optional[str] |
Optionally filter stack component by name |
None |
is_shared |
Optional[bool] |
Optionally filter out stack component by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.component_model.ComponentModel] |
A list of all stack components matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_stack_components(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
type: Optional[str] = None,
flavor_name: Optional[str] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[ComponentModel]:
"""List all stack components matching the given filter criteria.
Args:
project_name_or_id: The ID or name of the Project to which the stack
components belong
user_name_or_id: Optionally filter stack components by the owner
type: Optionally filter by type of stack component
flavor_name: Optionally filter by flavor
name: Optionally filter stack component by name
is_shared: Optionally filter out stack component by whether they are
shared or not
Returns:
A list of all stack components matching the filter criteria.
"""
with Session(self.engine) as session:
# Get a list of all stacks
query = select(StackComponentSchema)
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(
StackComponentSchema.project_id == project.id
)
if user_name_or_id:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(StackComponentSchema.user_id == user.id)
if type:
query = query.where(StackComponentSchema.type == type)
if flavor_name:
query = query.where(StackComponentSchema.flavor == flavor_name)
if name:
query = query.where(StackComponentSchema.name == name)
if is_shared is not None:
query = query.where(StackComponentSchema.is_shared == is_shared)
list_of_stack_components_in_db = session.exec(query).all()
return [comp.to_model() for comp in list_of_stack_components_in_db]
list_stacks(self, project_name_or_id=None, user_name_or_id=None, component_id=None, name=None, is_shared=None)
List all stacks matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Id or name of the Project containing the stack |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter stacks by their owner |
None |
component_id |
Optional[uuid.UUID] |
Optionally filter for stacks that contain the component |
None |
name |
Optional[str] |
Optionally filter stacks by their name |
None |
is_shared |
Optional[bool] |
Optionally filter out stacks by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.stack_models.StackModel] |
A list of all stacks matching the filter criteria. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_stacks(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_id: Optional[UUID] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[StackModel]:
"""List all stacks matching the given filter criteria.
Args:
project_name_or_id: Id or name of the Project containing the stack
user_name_or_id: Optionally filter stacks by their owner
component_id: Optionally filter for stacks that contain the
component
name: Optionally filter stacks by their name
is_shared: Optionally filter out stacks by whether they are shared
or not
Returns:
A list of all stacks matching the filter criteria.
"""
with Session(self.engine) as session:
# Get a list of all stacks
query = select(StackSchema)
# TODO: prettify
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
query = query.where(StackSchema.project_id == project.id)
if user_name_or_id:
user = self._get_user_schema(user_name_or_id, session=session)
query = query.where(StackSchema.user_id == user.id)
if component_id:
query = query.where(
StackCompositionSchema.stack_id == StackSchema.id
).where(StackCompositionSchema.component_id == component_id)
if name:
query = query.where(StackSchema.name == name)
if is_shared is not None:
query = query.where(StackSchema.is_shared == is_shared)
stacks = session.exec(query.order_by(StackSchema.name)).all()
return [stack.to_model() for stack in stacks]
list_steps(self, pipeline_id)
List all steps.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
The ID of the pipeline to list steps for. |
required |
Source code in zenml/zen_stores/sql_zen_store.py
def list_steps(self, pipeline_id: UUID) -> List[StepRunModel]:
"""List all steps.
Args:
pipeline_id: The ID of the pipeline to list steps for.
"""
pass # TODO
list_teams(self)
List all teams.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.TeamModel] |
A list of all teams. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_teams(self) -> List[TeamModel]:
"""List all teams.
Returns:
A list of all teams.
"""
with Session(self.engine) as session:
teams = session.exec(select(TeamSchema)).all()
return [team.to_model() for team in teams]
list_users(self)
List all users.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.UserModel] |
A list of all users. |
Source code in zenml/zen_stores/sql_zen_store.py
def list_users(self) -> List[UserModel]:
"""List all users.
Returns:
A list of all users.
"""
with Session(self.engine) as session:
users = session.exec(select(UserSchema)).all()
return [user.to_model() for user in users]
remove_user_from_team(self, user_name_or_id, team_name_or_id)
Removes a user from a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user to remove from the team. |
required |
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team from which to remove the user. |
required |
Source code in zenml/zen_stores/sql_zen_store.py
def remove_user_from_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Removes a user from a team.
Args:
user_name_or_id: Name or ID of the user to remove from the team.
team_name_or_id: Name or ID of the team from which to remove the
user.
"""
with Session(self.engine) as session:
team = self._get_team_schema(team_name_or_id, session=session)
user = self._get_user_schema(user_name_or_id, session=session)
# Remove user from team
team.users = [user_ for user_ in team.users if user_.id != user.id]
session.add(team)
session.commit()
revoke_role(self, role_name_or_id, user_or_team_name_or_id, is_user=True, project_name_or_id=None)
Revokes a role from a user or team for a given project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Optional ID of a project in which to revoke the role. If this is not provided, the role will be revoked globally. |
None |
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to revoke. |
required |
user_or_team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user or team from which to revoke the role. |
required |
is_user |
bool |
Whether |
True |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If the role, user, team, or project does not exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def revoke_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
is_user: bool = True,
project_name_or_id: Optional[Union[str, UUID]] = None,
) -> None:
"""Revokes a role from a user or team for a given project.
Args:
project_name_or_id: Optional ID of a project in which to revoke the
role. If this is not provided, the role will be revoked
globally.
role_name_or_id: Name or ID of the role to revoke.
user_or_team_name_or_id: Name or ID of the user or team from which
to revoke the role.
is_user: Whether `user_or_team_name_or_id` refers to a user or a
team.
Raises:
KeyError: If the role, user, team, or project does not exists.
"""
with Session(self.engine) as session:
project: Optional[ProjectSchema] = None
if project_name_or_id:
project = self._get_project_schema(
project_name_or_id, session=session
)
role = self._get_role_schema(role_name_or_id, session=session)
role_assignment: Optional[SQLModel] = None
if is_user:
user = self._get_user_schema(
user_or_team_name_or_id, session=session
)
assignee_name = user.name
user_role_query = (
select(UserRoleAssignmentSchema)
.where(UserRoleAssignmentSchema.user_id == user.id)
.where(UserRoleAssignmentSchema.role_id == role.id)
)
if project:
user_role_query = user_role_query.where(
UserRoleAssignmentSchema.project_id == project.id
)
role_assignment = session.exec(user_role_query).first()
else:
team = self._get_team_schema(
user_or_team_name_or_id, session=session
)
assignee_name = team.name
team_role_query = (
select(TeamRoleAssignmentSchema)
.where(TeamRoleAssignmentSchema.team_id == team.id)
.where(TeamRoleAssignmentSchema.role_id == role.id)
)
if project:
team_role_query = team_role_query.where(
TeamRoleAssignmentSchema.project_id == project.id
)
role_assignment = session.exec(team_role_query).first()
if role_assignment is None:
assignee = "user" if is_user else "team"
scope = f" in project '{project.name}'" if project else ""
raise KeyError(
f"Unable to unassign role '{role.name}' from {assignee} "
f"'{assignee_name}'{scope}: The role is currently not "
f"assigned to the {assignee}."
)
session.delete(role_assignment)
session.commit()
update_flavor(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_pipeline(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_project(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_role(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_stack(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_stack_component(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_team(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
update_user(*args, **kwargs)
Inner decorator function.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
*args |
Any |
Arguments to be passed to the function. |
() |
**kwargs |
Any |
Keyword arguments to be passed to the function. |
{} |
Returns:
| Type | Description |
|---|---|
Any |
Result of the function. |
Source code in zenml/zen_stores/sql_zen_store.py
def inner_func(*args: Any, **kwargs: Any) -> Any:
"""Inner decorator function.
Args:
*args: Arguments to be passed to the function.
**kwargs: Keyword arguments to be passed to the function.
Returns:
Result of the function.
"""
result = func(*args, **kwargs)
try:
tracker: Optional[AnalyticsTrackerMixin] = None
if len(args) and isinstance(args[0], AnalyticsTrackerMixin):
tracker = args[0]
for obj in [result] + list(args) + list(kwargs.values()):
if isinstance(obj, AnalyticsTrackedModelMixin):
obj.track_event(event_name, tracker=tracker)
break
else:
if tracker:
tracker.track_event(event_name, metadata)
else:
track_event(event_name, metadata)
except Exception as e:
logger.debug(f"Analytics tracking failure for {func}: {e}")
return result
user_email_opt_in(self, user_name_or_id, user_opt_in_response, email=None)
Persist user response to the email prompt.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or the ID of the user. |
required |
user_opt_in_response |
bool |
Whether this email should be associated with the user id in the telemetry |
required |
email |
Optional[str] |
The users email |
None |
Returns:
| Type | Description |
|---|---|
UserModel |
The updated user. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given name exists. |
Source code in zenml/zen_stores/sql_zen_store.py
def user_email_opt_in(
self,
user_name_or_id: Union[str, UUID],
user_opt_in_response: bool,
email: Optional[str] = None,
) -> UserModel:
"""Persist user response to the email prompt.
Args:
user_name_or_id: The name or the ID of the user.
user_opt_in_response: Whether this email should be associated
with the user id in the telemetry
email: The users email
Returns:
The updated user.
Raises:
KeyError: If no user with the given name exists.
"""
with Session(self.engine) as session:
try:
user = self._get_user_schema(user_name_or_id, session=session)
except NoResultFound as error:
raise KeyError from error
else:
# TODO: In the future we might want to validate that the email
# is non-empty and valid at this point if user_opt_in_response
# is True
user.email = email
user.email_opted_in = user_opt_in_response
session.add(user)
session.commit()
return user.to_model()
SqlZenStoreConfiguration (StoreConfiguration)
pydantic-model
SQL ZenML store configuration.
Attributes:
| Name | Type | Description |
|---|---|---|
type |
StoreType |
The type of the store. |
database |
Optional[str] |
database name. If not already present on the server, it will be created automatically on first access. |
username |
Optional[str] |
The database username. |
password |
Optional[str] |
The database password. |
ssl_ca |
Optional[str] |
certificate authority certificate. Required for SSL enabled authentication if the CA certificate is not part of the certificates shipped by the operating system. |
ssl_cert |
Optional[str] |
client certificate. Required for SSL enabled authentication if client certificates are used. |
ssl_key |
Optional[str] |
client certificate private key. Required for SSL enabled if client certificates are used. |
ssl_verify_server_cert |
bool |
set to verify the identity of the server against the provided server certificate. |
Source code in zenml/zen_stores/sql_zen_store.py
class SqlZenStoreConfiguration(StoreConfiguration):
"""SQL ZenML store configuration.
Attributes:
type: The type of the store.
database: database name. If not already present on the server, it will
be created automatically on first access.
username: The database username.
password: The database password.
ssl_ca: certificate authority certificate. Required for SSL
enabled authentication if the CA certificate is not part of the
certificates shipped by the operating system.
ssl_cert: client certificate. Required for SSL enabled
authentication if client certificates are used.
ssl_key: client certificate private key. Required for SSL
enabled if client certificates are used.
ssl_verify_server_cert: set to verify the identity of the server
against the provided server certificate.
"""
type: StoreType = StoreType.SQL
database: Optional[str] = None
username: Optional[str] = None
password: Optional[str] = None
ssl_ca: Optional[str] = None
ssl_cert: Optional[str] = None
ssl_key: Optional[str] = None
ssl_verify_server_cert: bool = False
pool_size: int = 20
max_overflow: int = 20
@root_validator
def _validate_url(cls, values: Dict[str, Any]) -> Dict[str, Any]:
"""Validate the SQL URL.
The validator also moves the MySQL username, password and database
parameters from the URL into the other configuration arguments, if they
are present in the URL.
Args:
values: The values to validate.
Returns:
The validated values.
Raises:
ValueError: If the URL is invalid or the SQL driver is not
supported.
"""
# flake8: noqa: C901
url = values.get("url")
if url is None:
return values
try:
sql_url = make_url(url)
except ArgumentError as e:
raise ValueError(
"Invalid SQL URL `%s`: %s. The URL must be in the format "
"`driver://[[username:password@]hostname:port]/database["
"?<extra-args>]`.",
url,
str(e),
)
if sql_url.drivername not in SQLDatabaseDriver.values():
raise ValueError(
"Invalid SQL driver value `%s`: The driver must be one of: %s.",
url,
", ".join(SQLDatabaseDriver.values()),
)
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
if (
sql_url.username
or sql_url.password
or sql_url.query
or sql_url.database is None
):
raise ValueError(
"Invalid SQLite URL `%s`: The URL must be in the "
"format `sqlite:///path/to/database.db`.",
url,
)
if values.get("username") or values.get("password"):
raise ValueError(
"Invalid SQLite configuration: The username and password "
"must not be set",
url,
)
values["database"] = sql_url.database
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
if sql_url.username:
values["username"] = sql_url.username
sql_url = sql_url._replace(username=None)
if sql_url.password:
values["password"] = sql_url.password
sql_url = sql_url._replace(password=None)
if sql_url.database:
values["database"] = sql_url.database
sql_url = sql_url._replace(database=None)
if sql_url.query:
for k, v in sql_url.query.items():
if k == "ssl_ca":
values["ssl_ca"] = v
elif k == "ssl_cert":
values["ssl_cert"] = v
elif k == "ssl_key":
values["ssl_key"] = v
elif k == "ssl_verify_server_cert":
values["ssl_verify_server_cert"] = v
else:
raise ValueError(
"Invalid MySQL URL query parameter `%s`: The "
"parameter must be one of: ssl_ca, ssl_cert, "
"ssl_key, or ssl_verify_server_cert.",
k,
)
sql_url = sql_url._replace(query={})
database = values.get("database")
if (
not values.get("username")
or not values.get("password")
or not database
):
raise ValueError(
"Invalid MySQL configuration: The username, password and "
"database must be set in the URL or as configuration "
"attributes",
)
regexp = r"^[^\\/?%*:|\"<>.-]{1,64}$"
match = re.match(regexp, database)
if not match:
raise ValueError(
f"The database name does not conform to the required "
f"format "
f"rules ({regexp}): {database}"
)
# Save the certificates in a secure location on disk
secret_folder = Path(
GlobalConfiguration().local_stores_path,
"certificates",
)
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
content = values.get(key)
if content and not os.path.isfile(content):
fileio.makedirs(str(secret_folder))
file_path = Path(secret_folder, f"{key}.pem")
with open(file_path, "w") as f:
f.write(content)
file_path.chmod(0o600)
values[key] = str(file_path)
values["url"] = str(sql_url)
return values
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the metadata configuration for the SQL ZenML store.
Args:
expand_certs: Whether to expand the certificate paths to their
contents.
Returns:
The metadata configuration.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
from ml_metadata.proto.metadata_store_pb2 import MySQLDatabaseConfig
from tfx.orchestration import metadata
sql_url = make_url(self.url)
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
mlmd_config = metadata.sqlite_metadata_connection_config(
self.database
)
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
mlmd_config = metadata.mysql_metadata_connection_config(
host=sql_url.host,
port=sql_url.port or 3306,
database=self.database,
username=self.username,
password=self.password,
)
mlmd_ssl_options = {}
# Handle certificate params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if not ssl_setting:
continue
if expand_certs and os.path.isfile(ssl_setting):
with open(ssl_setting, "r") as f:
ssl_setting = f.read()
mlmd_ssl_options[key.lstrip("ssl_")] = ssl_setting
# Handle additional params
if mlmd_ssl_options:
mlmd_ssl_options[
"verify_server_cert"
] = self.ssl_verify_server_cert
mlmd_config.mysql.ssl_options.CopyFrom(
MySQLDatabaseConfig.SSLOptions(**mlmd_ssl_options)
)
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return mlmd_config
def get_sqlmodel_config(self) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
"""Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
The URL and connection arguments for the SQLModel engine.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
sql_url = make_url(self.url)
sqlalchemy_connect_args = {}
engine_args = {}
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
# The following default value is needed for sqlite to avoid the Error:
# sqlite3.ProgrammingError: SQLite objects created in a thread can
# only be used in that same thread.
sqlalchemy_connect_args = {"check_same_thread": False}
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
engine_args = {
"pool_size": self.pool_size,
"max_overflow": self.max_overflow,
}
sql_url = sql_url._replace(
drivername="mysql+pymysql",
username=self.username,
password=self.password,
database=self.database,
)
# Handle certificate params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if ssl_setting and os.path.isfile(ssl_setting):
if key == "ssl_cert":
sqlalchemy_connect_args["ssl_capath"] = ssl_setting
else:
sqlalchemy_connect_args[key] = ssl_setting
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return str(sql_url), sqlalchemy_connect_args, engine_args
class Config:
"""Pydantic configuration class."""
# Validate attributes when assigning them. We need to set this in order
# to have a mix of mutable and immutable attributes
validate_assignment = True
# Forbid extra attributes set in the class.
extra = "forbid"
Config
Pydantic configuration class.
Source code in zenml/zen_stores/sql_zen_store.py
class Config:
"""Pydantic configuration class."""
# Validate attributes when assigning them. We need to set this in order
# to have a mix of mutable and immutable attributes
validate_assignment = True
# Forbid extra attributes set in the class.
extra = "forbid"
get_metadata_config(self, expand_certs=False)
Get the metadata configuration for the SQL ZenML store.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
expand_certs |
bool |
Whether to expand the certificate paths to their contents. |
False |
Returns:
| Type | Description |
|---|---|
ConnectionConfig |
The metadata configuration. |
Exceptions:
| Type | Description |
|---|---|
NotImplementedError |
If the SQL driver is not supported. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the metadata configuration for the SQL ZenML store.
Args:
expand_certs: Whether to expand the certificate paths to their
contents.
Returns:
The metadata configuration.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
from ml_metadata.proto.metadata_store_pb2 import MySQLDatabaseConfig
from tfx.orchestration import metadata
sql_url = make_url(self.url)
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
mlmd_config = metadata.sqlite_metadata_connection_config(
self.database
)
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
mlmd_config = metadata.mysql_metadata_connection_config(
host=sql_url.host,
port=sql_url.port or 3306,
database=self.database,
username=self.username,
password=self.password,
)
mlmd_ssl_options = {}
# Handle certificate params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if not ssl_setting:
continue
if expand_certs and os.path.isfile(ssl_setting):
with open(ssl_setting, "r") as f:
ssl_setting = f.read()
mlmd_ssl_options[key.lstrip("ssl_")] = ssl_setting
# Handle additional params
if mlmd_ssl_options:
mlmd_ssl_options[
"verify_server_cert"
] = self.ssl_verify_server_cert
mlmd_config.mysql.ssl_options.CopyFrom(
MySQLDatabaseConfig.SSLOptions(**mlmd_ssl_options)
)
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return mlmd_config
get_sqlmodel_config(self)
Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
| Type | Description |
|---|---|
Tuple[str, Dict[str, Any], Dict[str, Any]] |
The URL and connection arguments for the SQLModel engine. |
Exceptions:
| Type | Description |
|---|---|
NotImplementedError |
If the SQL driver is not supported. |
Source code in zenml/zen_stores/sql_zen_store.py
def get_sqlmodel_config(self) -> Tuple[str, Dict[str, Any], Dict[str, Any]]:
"""Get the SQLModel engine configuration for the SQL ZenML store.
Returns:
The URL and connection arguments for the SQLModel engine.
Raises:
NotImplementedError: If the SQL driver is not supported.
"""
sql_url = make_url(self.url)
sqlalchemy_connect_args = {}
engine_args = {}
if sql_url.drivername == SQLDatabaseDriver.SQLITE:
assert self.database is not None
# The following default value is needed for sqlite to avoid the Error:
# sqlite3.ProgrammingError: SQLite objects created in a thread can
# only be used in that same thread.
sqlalchemy_connect_args = {"check_same_thread": False}
elif sql_url.drivername == SQLDatabaseDriver.MYSQL:
# all these are guaranteed by our root validator
assert self.database is not None
assert self.username is not None
assert self.password is not None
assert sql_url.host is not None
engine_args = {
"pool_size": self.pool_size,
"max_overflow": self.max_overflow,
}
sql_url = sql_url._replace(
drivername="mysql+pymysql",
username=self.username,
password=self.password,
database=self.database,
)
# Handle certificate params
for key in ["ssl_key", "ssl_ca", "ssl_cert"]:
ssl_setting = getattr(self, key)
if ssl_setting and os.path.isfile(ssl_setting):
if key == "ssl_cert":
sqlalchemy_connect_args["ssl_capath"] = ssl_setting
else:
sqlalchemy_connect_args[key] = ssl_setting
else:
raise NotImplementedError(
f"SQL driver `{sql_url.drivername}` is not supported."
)
return str(sql_url), sqlalchemy_connect_args, engine_args
zen_store_interface
ZenML Store interface.
ZenStoreInterface (ABC)
ZenML store interface.
All ZenML stores must implement the methods in this interface.
The methods in this interface are organized in the following way:
-
they are grouped into categories based on the type of resource that they operate on (e.g. stacks, stack components, etc.)
-
each category has a set of CRUD methods (create, read, update, delete) that operate on the resources in that category. The order of the methods in each category should be:
-
create methods - store a new resource. These methods should fill in generated fields (e.g. UUIDs, creation timestamps) in the resource and return the updated resource.
- get methods - retrieve a single existing resource identified by a unique key or identifier from the store. These methods should always return a resource and raise an exception if the resource does not exist.
- list methods - retrieve a list of resources from the store. These methods should accept a set of filter parameters that can be used to filter the list of resources retrieved from the store.
- update methods - update an existing resource in the store. These methods should expect the updated resource to be correctly identified by its unique key or identifier and raise an exception if the resource does not exist.
- delete methods - delete an existing resource from the store. These methods should expect the resource to be correctly identified by its unique key or identifier. If the resource does not exist, an exception should be raised.
Best practices for implementing and keeping this interface clean and easy to maintain and extend:
- keep methods organized by resource type and ordered by CRUD operation
- for resources with multiple keys, don't implement multiple get or list methods here if the same functionality can be achieved by a single get or list method. Instead, implement them in the BaseZenStore class and have them call the generic get or list method in this interface.
- keep the logic required to convert between ZenML domain Model classes and internal store representations outside the ZenML domain Model classes
- methods for resources that have two or more unique keys (e.g. a Project
is uniquely identified by its name as well as its UUID) should reflect
that in the method variants and/or method arguments:
- methods that take in a resource identifier as argument should accept
all variants of the identifier (e.g.
project_name_or_uuidfor methods that get/list/update/delete Projects) - if a compound key is involved, separate get methods should be
implemented (e.g.
get_pipelineto get a pipeline by ID andget_pipeline_in_projectto get a pipeline by its name and the ID of the project it belongs to)
- methods that take in a resource identifier as argument should accept
all variants of the identifier (e.g.
- methods for resources that are scoped as children of other resources
(e.g. a Stack is always owned by a Project) should reflect the
key(s) of the parent resource in the provided methods and method
arguments:
- create methods should take the parent resource UUID(s) as an argument
(e.g.
create_stacktakes in the project ID) - get methods should be provided to retrieve a resource by the compound key that includes the parent resource key(s)
- list methods should feature optional filter arguments that reflect the parent resource key(s)
- create methods should take the parent resource UUID(s) as an argument
(e.g.
Source code in zenml/zen_stores/zen_store_interface.py
class ZenStoreInterface(ABC):
"""ZenML store interface.
All ZenML stores must implement the methods in this interface.
The methods in this interface are organized in the following way:
* they are grouped into categories based on the type of resource
that they operate on (e.g. stacks, stack components, etc.)
* each category has a set of CRUD methods (create, read, update, delete)
that operate on the resources in that category. The order of the methods
in each category should be:
* create methods - store a new resource. These methods
should fill in generated fields (e.g. UUIDs, creation timestamps) in
the resource and return the updated resource.
* get methods - retrieve a single existing resource identified by a
unique key or identifier from the store. These methods should always
return a resource and raise an exception if the resource does not exist.
* list methods - retrieve a list of resources from the store. These
methods should accept a set of filter parameters that can be used to
filter the list of resources retrieved from the store.
* update methods - update an existing resource in the store. These
methods should expect the updated resource to be correctly identified
by its unique key or identifier and raise an exception if the resource
does not exist.
* delete methods - delete an existing resource from the store. These
methods should expect the resource to be correctly identified by its
unique key or identifier. If the resource does not exist,
an exception should be raised.
Best practices for implementing and keeping this interface clean and easy to
maintain and extend:
* keep methods organized by resource type and ordered by CRUD operation
* for resources with multiple keys, don't implement multiple get or list
methods here if the same functionality can be achieved by a single get or
list method. Instead, implement them in the BaseZenStore class and have
them call the generic get or list method in this interface.
* keep the logic required to convert between ZenML domain Model classes
and internal store representations outside the ZenML domain Model classes
* methods for resources that have two or more unique keys (e.g. a Project
is uniquely identified by its name as well as its UUID) should reflect
that in the method variants and/or method arguments:
* methods that take in a resource identifier as argument should accept
all variants of the identifier (e.g. `project_name_or_uuid` for methods
that get/list/update/delete Projects)
* if a compound key is involved, separate get methods should be
implemented (e.g. `get_pipeline` to get a pipeline by ID and
`get_pipeline_in_project` to get a pipeline by its name and the ID of
the project it belongs to)
* methods for resources that are scoped as children of other resources
(e.g. a Stack is always owned by a Project) should reflect the
key(s) of the parent resource in the provided methods and method
arguments:
* create methods should take the parent resource UUID(s) as an argument
(e.g. `create_stack` takes in the project ID)
* get methods should be provided to retrieve a resource by the compound
key that includes the parent resource key(s)
* list methods should feature optional filter arguments that reflect
the parent resource key(s)
"""
# ---------------------------------
# Initialization and configuration
# ---------------------------------
@abstractmethod
def _initialize(self) -> None:
"""Initialize the store.
This method is called immediately after the store is created. It should
be used to set up the backend (database, connection etc.).
"""
@staticmethod
@abstractmethod
def get_local_url(path: str) -> str:
"""Get a local URL for a given local path.
Args:
path: the path string to build a URL out of.
Returns:
Url pointing to the path for the store type.
"""
@classmethod
@abstractmethod
def copy_local_store(
cls,
config: StoreConfiguration,
path: str,
load_config_path: Optional[PurePath] = None,
) -> StoreConfiguration:
"""Copy a local store to a new location.
Use this method to create a copy of a store database to a new location
and return a new store configuration pointing to the database copy. This
only applies to stores that use the local filesystem to store their
data. Calling this method for remote stores simply returns the input
store configuration unaltered.
Args:
config: The configuration of the store to copy.
path: The new local path where the store DB will be copied.
load_config_path: path that will be used to load the copied store
database. This can be set to a value different from `path`
if the local database copy will be loaded from a different
environment, e.g. when the database is copied to a container
image and loaded using a different absolute path. This will be
reflected in the paths and URLs encoded in the copied store
configuration.
Returns:
The store configuration of the copied store.
"""
@abstractmethod
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
# ------------
# TFX Metadata
# ------------
@abstractmethod
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the TFX metadata config of this ZenStore.
Args:
expand_certs: Whether to expand the certificate paths in the
connection config to their value.
Returns:
The TFX metadata config of this ZenStore.
"""
# ------
# Stacks
# ------
@abstractmethod
def create_stack(
self,
stack: StackModel,
) -> StackModel:
"""Create a new stack.
Args:
stack: The stack to create.
Returns:
The created stack.
Raises:
StackExistsError: If a stack with the same name is already owned
by this user in this project.
"""
@abstractmethod
def get_stack(self, stack_id: UUID) -> StackModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
Raises:
KeyError: if the stack doesn't exist.
"""
@abstractmethod
def list_stacks(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_id: Optional[UUID] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[StackModel]:
"""List all stacks matching the given filter criteria.
Args:
project_name_or_id: Id or name of the Project containing the stack
user_name_or_id: Optionally filter stacks by their owner
component_id: Optionally filter for stacks that contain the
component
name: Optionally filter stacks by their name
is_shared: Optionally filter out stacks by whether they are shared
or not
Returns:
A list of all stacks matching the filter criteria.
Raises:
KeyError: if the project doesn't exist.
"""
@abstractmethod
def update_stack(
self,
stack: StackModel,
) -> StackModel:
"""Update a stack.
Args:
stack: The stack to use for the update.
Returns:
The updated stack.
Raises:
KeyError: if the stack doesn't exist.
"""
@abstractmethod
def delete_stack(self, stack_id: UUID) -> None:
"""Delete a stack.
Args:
stack_id: The ID of the stack to delete.
Raises:
KeyError: if the stack doesn't exist.
"""
# ----------------
# Stack components
# ----------------
@abstractmethod
def create_stack_component(
self,
component: ComponentModel,
) -> ComponentModel:
"""Create a stack component.
Args:
component: The stack component to create.
Returns:
The created stack component.
Raises:
StackComponentExistsError: If a stack component with the same name
and type is already owned by this user in this project.
"""
@abstractmethod
def list_stack_components(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
type: Optional[str] = None,
flavor_name: Optional[str] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[ComponentModel]:
"""List all stack components matching the given filter criteria.
Args:
project_name_or_id: The ID or name of the Project to which the stack
components belong
user_name_or_id: Optionally filter stack components by the owner
type: Optionally filter by type of stack component
flavor_name: Optionally filter by flavor
name: Optionally filter stack component by name
is_shared: Optionally filter out stack component by whether they are
shared or not
Returns:
A list of all stack components matching the filter criteria.
Raises:
KeyError: if the project doesn't exist.
"""
@abstractmethod
def get_stack_component(self, component_id: UUID) -> ComponentModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
@abstractmethod
def update_stack_component(
self,
component: ComponentModel,
) -> ComponentModel:
"""Update an existing stack component.
Args:
component: The stack component to use for the update.
Returns:
The updated stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
@abstractmethod
def delete_stack_component(self, component_id: UUID) -> None:
"""Delete a stack component.
Args:
component_id: The ID of the stack component to delete.
Raises:
KeyError: if the stack component doesn't exist.
ValueError: if the stack component is part of one or more stacks.
"""
@abstractmethod
def get_stack_component_side_effects(
self,
component_id: UUID,
run_id: UUID,
pipeline_id: UUID,
stack_id: UUID,
) -> Dict[Any, Any]:
"""Get the side effects of a stack component.
Args:
component_id: The ID of the stack component to get side effects for.
run_id: The ID of the run to get side effects for.
pipeline_id: The ID of the pipeline to get side effects for.
stack_id: The ID of the stack to get side effects for.
"""
# -----------------------
# Stack component flavors
# -----------------------
@abstractmethod
def create_flavor(
self,
flavor: FlavorModel,
) -> FlavorModel:
"""Creates a new stack component flavor.
Args:
flavor: The stack component flavor to create.
Returns:
The newly created flavor.
Raises:
EntityExistsError: If a flavor with the same name and type
is already owned by this user in this project.
"""
@abstractmethod
def get_flavor(self, flavor_id: UUID) -> FlavorModel:
"""Get a stack component flavor by ID.
Args:
flavor_id: The ID of the stack component flavor to get.
Returns:
The stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
@abstractmethod
def list_flavors(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_type: Optional[StackComponentType] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[FlavorModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
project_name_or_id: Optionally filter by the Project to which the
component flavors belong
user_name_or_id: Optionally filter by the owner
component_type: Optionally filter by type of stack component
name: Optionally filter flavors by name
is_shared: Optionally filter out flavors by whether they are
shared or not
Returns:
List of all the stack component flavors matching the given criteria.
Raises:
KeyError: if the project doesn't exist.
"""
@abstractmethod
def update_flavor(self, flavor: FlavorModel) -> FlavorModel:
"""Update an existing stack component flavor.
Args:
flavor: The stack component flavor to use for the update.
Returns:
The updated stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
@abstractmethod
def delete_flavor(self, flavor_id: UUID) -> None:
"""Delete a stack component flavor.
Args:
flavor_id: The ID of the stack component flavor to delete.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
# -----
# Users
# -----
@property
@abstractmethod
def active_user_name(self) -> str:
"""Gets the active username.
Returns:
The active username.
"""
@abstractmethod
def create_user(self, user: UserModel) -> UserModel:
"""Creates a new user.
Args:
user: User to be created.
Returns:
The newly created user.
Raises:
EntityExistsError: If a user with the given name already exists.
"""
@abstractmethod
def get_user(self, user_name_or_id: Union[str, UUID]) -> UserModel:
"""Gets a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
Raises:
KeyError: If no user with the given name or ID exists.
"""
# TODO: [ALEX] add filtering param(s)
@abstractmethod
def list_users(self) -> List[UserModel]:
"""List all users.
Returns:
A list of all users.
"""
@abstractmethod
def update_user(self, user: UserModel) -> UserModel:
"""Updates an existing user.
Args:
user: The user model to use for the update.
Returns:
The updated user.
Raises:
KeyError: If no user with the given name exists.
"""
@abstractmethod
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
"""Deletes a user.
Args:
user_name_or_id: The name or ID of the user to delete.
Raises:
KeyError: If no user with the given ID exists.
"""
@abstractmethod
def user_email_opt_in(
self,
user_name_or_id: Union[str, UUID],
user_opt_in_response: bool,
email: Optional[str] = None,
) -> UserModel:
"""Persist user response to the email prompt.
Args:
user_name_or_id: The name or the ID of the user.
user_opt_in_response: Whether this email should be associated
with the user id in the telemetry
email: The users email
Returns:
The updated user.
Raises:
KeyError: If no user with the given name exists.
"""
# -----
# Teams
# -----
@abstractmethod
def create_team(self, team: TeamModel) -> TeamModel:
"""Creates a new team.
Args:
team: The team model to create.
Returns:
The newly created team.
"""
@abstractmethod
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
Raises:
KeyError: If no team with the given name or ID exists.
"""
@abstractmethod
def list_teams(self) -> List[TeamModel]:
"""List all teams.
Returns:
A list of all teams.
"""
@abstractmethod
def update_team(self, team: TeamModel) -> TeamModel:
"""Update an existing team.
Args:
team: The team to use for the update.
Returns:
The updated team.
Raises:
KeyError: if the team does not exist.
"""
@abstractmethod
def delete_team(self, team_name_or_id: Union[str, UUID]) -> None:
"""Deletes a team.
Args:
team_name_or_id: Name or ID of the team to delete.
Raises:
KeyError: If no team with the given ID exists.
"""
# ---------------
# Team membership
# ---------------
@abstractmethod
def get_users_for_team(
self, team_name_or_id: Union[str, UUID]
) -> List[UserModel]:
"""Fetches all users of a team.
Args:
team_name_or_id: The name or ID of the team for which to get users.
Returns:
A list of all users that are part of the team.
Raises:
KeyError: If no team with the given ID exists.
"""
@abstractmethod
def get_teams_for_user(
self, user_name_or_id: Union[str, UUID]
) -> List[TeamModel]:
"""Fetches all teams for a user.
Args:
user_name_or_id: The name or ID of the user for which to get all
teams.
Returns:
A list of all teams that the user is part of.
Raises:
KeyError: If no user with the given ID exists.
"""
@abstractmethod
def add_user_to_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Adds a user to a team.
Args:
user_name_or_id: Name or ID of the user to add to the team.
team_name_or_id: Name or ID of the team to which to add the user to.
Raises:
KeyError: If the team or user does not exist.
"""
@abstractmethod
def remove_user_from_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Removes a user from a team.
Args:
user_name_or_id: Name or ID of the user to remove from the team.
team_name_or_id: Name or ID of the team from which to remove the user.
Raises:
KeyError: If the team or user does not exist.
"""
# -----
# Roles
# -----
# TODO: consider using team_id instead
@abstractmethod
def create_role(self, role: RoleModel) -> RoleModel:
"""Creates a new role.
Args:
role: The role model to create.
Returns:
The newly created role.
Raises:
EntityExistsError: If a role with the given name already exists.
"""
# TODO: consider using team_id instead
@abstractmethod
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
Raises:
KeyError: If no role with the given name exists.
"""
# TODO: [ALEX] add filtering param(s)
@abstractmethod
def list_roles(self) -> List[RoleModel]:
"""List all roles.
Returns:
A list of all roles.
"""
@abstractmethod
def update_role(self, role: RoleModel) -> RoleModel:
"""Update an existing role.
Args:
role: The role to use for the update.
Returns:
The updated role.
Raises:
KeyError: if the role does not exist.
"""
@abstractmethod
def delete_role(self, role_name_or_id: Union[str, UUID]) -> None:
"""Deletes a role.
Args:
role_name_or_id: Name or ID of the role to delete.
Raises:
KeyError: If no role with the given ID exists.
"""
# ----------------
# Role assignments
# ----------------
@abstractmethod
def list_role_assignments(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
team_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
) -> List[RoleAssignmentModel]:
"""List all role assignments.
Args:
project_name_or_id: If provided, only list assignments for the given
project
team_name_or_id: If provided, only list assignments for the given
team
user_name_or_id: If provided, only list assignments for the given
user
Returns:
A list of all role assignments.
"""
@abstractmethod
def assign_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
project_name_or_id: Optional[Union[str, UUID]] = None,
is_user: bool = True,
) -> None:
"""Assigns a role to a user or team, scoped to a specific project.
Args:
role_name_or_id: Name or ID of the role to assign.
user_or_team_name_or_id: Name or ID of the user or team to which to
assign the role.
is_user: Whether `user_or_team_id` refers to a user or a team.
project_name_or_id: Optional Name or ID of a project in which to
assign the role. If this is not provided, the role will be
assigned globally.
Raises:
EntityExistsError: If the role assignment already exists.
"""
@abstractmethod
def revoke_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
is_user: bool = True,
project_name_or_id: Optional[Union[str, UUID]] = None,
) -> None:
"""Revokes a role from a user or team for a given project.
Args:
role_name_or_id: ID of the role to revoke.
user_or_team_name_or_id: Name or ID of the user or team from which
to revoke the role.
is_user: Whether `user_or_team_id` refers to a user or a team.
project_name_or_id: Optional ID of a project in which to revoke
the role. If this is not provided, the role will be revoked
globally.
Raises:
KeyError: If the role, user, team, or project does not exists.
"""
# --------
# Projects
# --------
@abstractmethod
def create_project(self, project: ProjectModel) -> ProjectModel:
"""Creates a new project.
Args:
project: The project to create.
Returns:
The newly created project.
Raises:
EntityExistsError: If a project with the given name already exists.
"""
@abstractmethod
def get_project(self, project_name_or_id: Union[UUID, str]) -> ProjectModel:
"""Get an existing project by name or ID.
Args:
project_name_or_id: Name or ID of the project to get.
Returns:
The requested project.
Raises:
KeyError: If there is no such project.
"""
# TODO: [ALEX] add filtering param(s)
@abstractmethod
def list_projects(self) -> List[ProjectModel]:
"""List all projects.
Returns:
A list of all projects.
"""
@abstractmethod
def update_project(self, project: ProjectModel) -> ProjectModel:
"""Update an existing project.
Args:
project: The project to use for the update.
Returns:
The updated project.
Raises:
KeyError: if the project does not exist.
"""
@abstractmethod
def delete_project(self, project_name_or_id: Union[str, UUID]) -> None:
"""Deletes a project.
Args:
project_name_or_id: Name or ID of the project to delete.
Raises:
KeyError: If no project with the given name exists.
"""
# ---------
# Pipelines
# ---------
@abstractmethod
def create_pipeline(
self,
pipeline: PipelineModel,
) -> PipelineModel:
"""Creates a new pipeline in a project.
Args:
pipeline: The pipeline to create.
Returns:
The newly created pipeline.
Raises:
KeyError: if the project does not exist.
EntityExistsError: If an identical pipeline already exists.
"""
@abstractmethod
def get_pipeline(self, pipeline_id: UUID) -> PipelineModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
@abstractmethod
def list_pipelines(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
name: Optional[str] = None,
) -> List[PipelineModel]:
"""List all pipelines in the project.
Args:
project_name_or_id: If provided, only list pipelines in this project.
user_name_or_id: If provided, only list pipelines from this user.
name: If provided, only list pipelines with this name.
Returns:
A list of pipelines.
Raises:
KeyError: if the project does not exist.
"""
@abstractmethod
def update_pipeline(self, pipeline: PipelineModel) -> PipelineModel:
"""Updates a pipeline.
Args:
pipeline: The pipeline to use for the update.
Returns:
The updated pipeline.
Raises:
KeyError: if the pipeline doesn't exist.
"""
@abstractmethod
def delete_pipeline(self, pipeline_id: UUID) -> None:
"""Deletes a pipeline.
Args:
pipeline_id: The ID of the pipeline to delete.
Raises:
KeyError: if the pipeline doesn't exist.
"""
# --------------
# Pipeline steps
# --------------
# TODO: Note that this doesn't have a corresponding API endpoint (consider adding?)
# TODO: Discuss whether we even need this, given that the endpoint is on
# pipeline runs
# TODO: [ALEX] add filtering param(s)
@abstractmethod
def list_steps(self, pipeline_id: UUID) -> List[StepRunModel]:
"""List all steps.
Args:
pipeline_id: The ID of the pipeline to list steps for.
Returns:
A list of all steps.
"""
# --------------
# Pipeline runs
# --------------
@abstractmethod
def get_run(self, run_id: UUID) -> PipelineRunModel:
"""Gets a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
Returns:
The pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
# TODO: Figure out what exactly gets returned from this
@abstractmethod
def get_run_component_side_effects(
self,
run_id: UUID,
component_id: Optional[UUID] = None,
) -> Dict[str, Any]:
"""Gets the side effects for a component in a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
component_id: The ID of the component to get.
Returns:
The side effects for the component in the pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
@abstractmethod
def list_runs(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
stack_id: Optional[UUID] = None,
component_id: Optional[UUID] = None,
run_name: Optional[str] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
pipeline_id: Optional[UUID] = None,
unlisted: bool = False,
) -> List[PipelineRunModel]:
"""Gets all pipeline runs.
Args:
project_name_or_id: If provided, only return runs for this project.
stack_id: If provided, only return runs for this stack.
component_id: Optionally filter for runs that used the
component
run_name: Run name if provided
user_name_or_id: If provided, only return runs for this user.
pipeline_id: If provided, only return runs for this pipeline.
unlisted: If True, only return unlisted runs that are not
associated with any pipeline (filter by `pipeline_id==None`).
Returns:
A list of all pipeline runs.
"""
@abstractmethod
def get_run_status(self, run_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a pipeline run.
Args:
run_id: The ID of the pipeline run to get the status for.
Returns:
The status of the pipeline run.
"""
# ------------------
# Pipeline run steps
# ------------------
@abstractmethod
def get_run_step(self, step_id: UUID) -> StepRunModel:
"""Get a step by ID.
Args:
step_id: The ID of the step to get.
Returns:
The step.
Raises:
KeyError: if the step doesn't exist.
"""
@abstractmethod
def get_run_step_outputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get a list of outputs for a specific step.
Args:
step_id: The id of the step to get outputs for.
Returns:
A dict mapping artifact names to the output artifacts for the step.
"""
@abstractmethod
def get_run_step_inputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get a list of inputs for a specific step.
Args:
step_id: The id of the step to get inputs for.
Returns:
A dict mapping artifact names to the input artifacts for the step.
"""
@abstractmethod
def get_run_step_status(self, step_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a single step.
Args:
step_id: The ID of the step to get the status for.
Returns:
ExecutionStatus: The status of the step.
"""
@abstractmethod
def list_run_steps(self, run_id: UUID) -> List[StepRunModel]:
"""Gets all steps in a pipeline run.
Args:
run_id: The ID of the pipeline run for which to list runs.
Returns:
A mapping from step names to step models for all steps in the run.
"""
@abstractmethod
def list_artifacts(
self, artifact_uri: Optional[str] = None
) -> List[ArtifactModel]:
"""Lists all artifacts.
Args:
artifact_uri: If specified, only artifacts with the given URI will
be returned.
Returns:
A list of all artifacts.
"""
@abstractmethod
def _sync_runs(self) -> None:
"""Syncs runs from MLMD."""
active_user_name: str
property
readonly
Gets the active username.
Returns:
| Type | Description |
|---|---|
str |
The active username. |
add_user_to_team(self, user_name_or_id, team_name_or_id)
Adds a user to a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user to add to the team. |
required |
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to which to add the user to. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If the team or user does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def add_user_to_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Adds a user to a team.
Args:
user_name_or_id: Name or ID of the user to add to the team.
team_name_or_id: Name or ID of the team to which to add the user to.
Raises:
KeyError: If the team or user does not exist.
"""
assign_role(self, role_name_or_id, user_or_team_name_or_id, project_name_or_id=None, is_user=True)
Assigns a role to a user or team, scoped to a specific project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to assign. |
required |
user_or_team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user or team to which to assign the role. |
required |
is_user |
bool |
Whether |
True |
project_name_or_id |
Union[str, uuid.UUID] |
Optional Name or ID of a project in which to assign the role. If this is not provided, the role will be assigned globally. |
None |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If the role assignment already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def assign_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
project_name_or_id: Optional[Union[str, UUID]] = None,
is_user: bool = True,
) -> None:
"""Assigns a role to a user or team, scoped to a specific project.
Args:
role_name_or_id: Name or ID of the role to assign.
user_or_team_name_or_id: Name or ID of the user or team to which to
assign the role.
is_user: Whether `user_or_team_id` refers to a user or a team.
project_name_or_id: Optional Name or ID of a project in which to
assign the role. If this is not provided, the role will be
assigned globally.
Raises:
EntityExistsError: If the role assignment already exists.
"""
copy_local_store(config, path, load_config_path=None)
classmethod
Copy a local store to a new location.
Use this method to create a copy of a store database to a new location and return a new store configuration pointing to the database copy. This only applies to stores that use the local filesystem to store their data. Calling this method for remote stores simply returns the input store configuration unaltered.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
config |
StoreConfiguration |
The configuration of the store to copy. |
required |
path |
str |
The new local path where the store DB will be copied. |
required |
load_config_path |
Optional[pathlib.PurePath] |
path that will be used to load the copied store
database. This can be set to a value different from |
None |
Returns:
| Type | Description |
|---|---|
StoreConfiguration |
The store configuration of the copied store. |
Source code in zenml/zen_stores/zen_store_interface.py
@classmethod
@abstractmethod
def copy_local_store(
cls,
config: StoreConfiguration,
path: str,
load_config_path: Optional[PurePath] = None,
) -> StoreConfiguration:
"""Copy a local store to a new location.
Use this method to create a copy of a store database to a new location
and return a new store configuration pointing to the database copy. This
only applies to stores that use the local filesystem to store their
data. Calling this method for remote stores simply returns the input
store configuration unaltered.
Args:
config: The configuration of the store to copy.
path: The new local path where the store DB will be copied.
load_config_path: path that will be used to load the copied store
database. This can be set to a value different from `path`
if the local database copy will be loaded from a different
environment, e.g. when the database is copied to a container
image and loaded using a different absolute path. This will be
reflected in the paths and URLs encoded in the copied store
configuration.
Returns:
The store configuration of the copied store.
"""
create_flavor(self, flavor)
Creates a new stack component flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor |
FlavorModel |
The stack component flavor to create. |
required |
Returns:
| Type | Description |
|---|---|
FlavorModel |
The newly created flavor. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a flavor with the same name and type is already owned by this user in this project. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_flavor(
self,
flavor: FlavorModel,
) -> FlavorModel:
"""Creates a new stack component flavor.
Args:
flavor: The stack component flavor to create.
Returns:
The newly created flavor.
Raises:
EntityExistsError: If a flavor with the same name and type
is already owned by this user in this project.
"""
create_pipeline(self, pipeline)
Creates a new pipeline in a project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline |
PipelineModel |
The pipeline to create. |
required |
Returns:
| Type | Description |
|---|---|
PipelineModel |
The newly created pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the project does not exist. |
EntityExistsError |
If an identical pipeline already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_pipeline(
self,
pipeline: PipelineModel,
) -> PipelineModel:
"""Creates a new pipeline in a project.
Args:
pipeline: The pipeline to create.
Returns:
The newly created pipeline.
Raises:
KeyError: if the project does not exist.
EntityExistsError: If an identical pipeline already exists.
"""
create_project(self, project)
Creates a new project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project |
ProjectModel |
The project to create. |
required |
Returns:
| Type | Description |
|---|---|
ProjectModel |
The newly created project. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a project with the given name already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_project(self, project: ProjectModel) -> ProjectModel:
"""Creates a new project.
Args:
project: The project to create.
Returns:
The newly created project.
Raises:
EntityExistsError: If a project with the given name already exists.
"""
create_role(self, role)
Creates a new role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role |
RoleModel |
The role model to create. |
required |
Returns:
| Type | Description |
|---|---|
RoleModel |
The newly created role. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a role with the given name already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_role(self, role: RoleModel) -> RoleModel:
"""Creates a new role.
Args:
role: The role model to create.
Returns:
The newly created role.
Raises:
EntityExistsError: If a role with the given name already exists.
"""
create_stack(self, stack)
Create a new stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack |
StackModel |
The stack to create. |
required |
Returns:
| Type | Description |
|---|---|
StackModel |
The created stack. |
Exceptions:
| Type | Description |
|---|---|
StackExistsError |
If a stack with the same name is already owned by this user in this project. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_stack(
self,
stack: StackModel,
) -> StackModel:
"""Create a new stack.
Args:
stack: The stack to create.
Returns:
The created stack.
Raises:
StackExistsError: If a stack with the same name is already owned
by this user in this project.
"""
create_stack_component(self, component)
Create a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
ComponentModel |
The stack component to create. |
required |
Returns:
| Type | Description |
|---|---|
ComponentModel |
The created stack component. |
Exceptions:
| Type | Description |
|---|---|
StackComponentExistsError |
If a stack component with the same name and type is already owned by this user in this project. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_stack_component(
self,
component: ComponentModel,
) -> ComponentModel:
"""Create a stack component.
Args:
component: The stack component to create.
Returns:
The created stack component.
Raises:
StackComponentExistsError: If a stack component with the same name
and type is already owned by this user in this project.
"""
create_team(self, team)
Creates a new team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team |
TeamModel |
The team model to create. |
required |
Returns:
| Type | Description |
|---|---|
TeamModel |
The newly created team. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_team(self, team: TeamModel) -> TeamModel:
"""Creates a new team.
Args:
team: The team model to create.
Returns:
The newly created team.
"""
create_user(self, user)
Creates a new user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user |
UserModel |
User to be created. |
required |
Returns:
| Type | Description |
|---|---|
UserModel |
The newly created user. |
Exceptions:
| Type | Description |
|---|---|
EntityExistsError |
If a user with the given name already exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def create_user(self, user: UserModel) -> UserModel:
"""Creates a new user.
Args:
user: User to be created.
Returns:
The newly created user.
Raises:
EntityExistsError: If a user with the given name already exists.
"""
delete_flavor(self, flavor_id)
Delete a stack component flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
UUID |
The ID of the stack component flavor to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component flavor doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_flavor(self, flavor_id: UUID) -> None:
"""Delete a stack component flavor.
Args:
flavor_id: The ID of the stack component flavor to delete.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
delete_pipeline(self, pipeline_id)
Deletes a pipeline.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
The ID of the pipeline to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_pipeline(self, pipeline_id: UUID) -> None:
"""Deletes a pipeline.
Args:
pipeline_id: The ID of the pipeline to delete.
Raises:
KeyError: if the pipeline doesn't exist.
"""
delete_project(self, project_name_or_id)
Deletes a project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the project to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no project with the given name exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_project(self, project_name_or_id: Union[str, UUID]) -> None:
"""Deletes a project.
Args:
project_name_or_id: Name or ID of the project to delete.
Raises:
KeyError: If no project with the given name exists.
"""
delete_role(self, role_name_or_id)
Deletes a role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no role with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_role(self, role_name_or_id: Union[str, UUID]) -> None:
"""Deletes a role.
Args:
role_name_or_id: Name or ID of the role to delete.
Raises:
KeyError: If no role with the given ID exists.
"""
delete_stack(self, stack_id)
Delete a stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_stack(self, stack_id: UUID) -> None:
"""Delete a stack.
Args:
stack_id: The ID of the stack to delete.
Raises:
KeyError: if the stack doesn't exist.
"""
delete_stack_component(self, component_id)
Delete a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
ValueError |
if the stack component is part of one or more stacks. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_stack_component(self, component_id: UUID) -> None:
"""Delete a stack component.
Args:
component_id: The ID of the stack component to delete.
Raises:
KeyError: if the stack component doesn't exist.
ValueError: if the stack component is part of one or more stacks.
"""
delete_team(self, team_name_or_id)
Deletes a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no team with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_team(self, team_name_or_id: Union[str, UUID]) -> None:
"""Deletes a team.
Args:
team_name_or_id: Name or ID of the team to delete.
Raises:
KeyError: If no team with the given ID exists.
"""
delete_user(self, user_name_or_id)
Deletes a user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to delete. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def delete_user(self, user_name_or_id: Union[str, UUID]) -> None:
"""Deletes a user.
Args:
user_name_or_id: The name or ID of the user to delete.
Raises:
KeyError: If no user with the given ID exists.
"""
get_flavor(self, flavor_id)
Get a stack component flavor by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor_id |
UUID |
The ID of the stack component flavor to get. |
required |
Returns:
| Type | Description |
|---|---|
FlavorModel |
The stack component flavor. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component flavor doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_flavor(self, flavor_id: UUID) -> FlavorModel:
"""Get a stack component flavor by ID.
Args:
flavor_id: The ID of the stack component flavor to get.
Returns:
The stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
get_local_url(path)
staticmethod
Get a local URL for a given local path.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
path |
str |
the path string to build a URL out of. |
required |
Returns:
| Type | Description |
|---|---|
str |
Url pointing to the path for the store type. |
Source code in zenml/zen_stores/zen_store_interface.py
@staticmethod
@abstractmethod
def get_local_url(path: str) -> str:
"""Get a local URL for a given local path.
Args:
path: the path string to build a URL out of.
Returns:
Url pointing to the path for the store type.
"""
get_metadata_config(self, expand_certs=False)
Get the TFX metadata config of this ZenStore.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
expand_certs |
bool |
Whether to expand the certificate paths in the connection config to their value. |
False |
Returns:
| Type | Description |
|---|---|
ConnectionConfig |
The TFX metadata config of this ZenStore. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_metadata_config(
self, expand_certs: bool = False
) -> "ConnectionConfig":
"""Get the TFX metadata config of this ZenStore.
Args:
expand_certs: Whether to expand the certificate paths in the
connection config to their value.
Returns:
The TFX metadata config of this ZenStore.
"""
get_pipeline(self, pipeline_id)
Get a pipeline with a given ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
ID of the pipeline. |
required |
Returns:
| Type | Description |
|---|---|
PipelineModel |
The pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_pipeline(self, pipeline_id: UUID) -> PipelineModel:
"""Get a pipeline with a given ID.
Args:
pipeline_id: ID of the pipeline.
Returns:
The pipeline.
Raises:
KeyError: if the pipeline does not exist.
"""
get_project(self, project_name_or_id)
Get an existing project by name or ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[uuid.UUID, str] |
Name or ID of the project to get. |
required |
Returns:
| Type | Description |
|---|---|
ProjectModel |
The requested project. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If there is no such project. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_project(self, project_name_or_id: Union[UUID, str]) -> ProjectModel:
"""Get an existing project by name or ID.
Args:
project_name_or_id: Name or ID of the project to get.
Returns:
The requested project.
Raises:
KeyError: If there is no such project.
"""
get_role(self, role_name_or_id)
Gets a specific role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the role to get. |
required |
Returns:
| Type | Description |
|---|---|
RoleModel |
The requested role. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no role with the given name exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_role(self, role_name_or_id: Union[str, UUID]) -> RoleModel:
"""Gets a specific role.
Args:
role_name_or_id: Name or ID of the role to get.
Returns:
The requested role.
Raises:
KeyError: If no role with the given name exists.
"""
get_run(self, run_id)
Gets a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get. |
required |
Returns:
| Type | Description |
|---|---|
PipelineRunModel |
The pipeline run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run(self, run_id: UUID) -> PipelineRunModel:
"""Gets a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
Returns:
The pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
get_run_component_side_effects(self, run_id, component_id=None)
Gets the side effects for a component in a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get. |
required |
component_id |
Optional[uuid.UUID] |
The ID of the component to get. |
None |
Returns:
| Type | Description |
|---|---|
Dict[str, Any] |
The side effects for the component in the pipeline run. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline run doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_component_side_effects(
self,
run_id: UUID,
component_id: Optional[UUID] = None,
) -> Dict[str, Any]:
"""Gets the side effects for a component in a pipeline run.
Args:
run_id: The ID of the pipeline run to get.
component_id: The ID of the component to get.
Returns:
The side effects for the component in the pipeline run.
Raises:
KeyError: if the pipeline run doesn't exist.
"""
get_run_status(self, run_id)
Gets the execution status of a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run to get the status for. |
required |
Returns:
| Type | Description |
|---|---|
ExecutionStatus |
The status of the pipeline run. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_status(self, run_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a pipeline run.
Args:
run_id: The ID of the pipeline run to get the status for.
Returns:
The status of the pipeline run.
"""
get_run_step(self, step_id)
Get a step by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The ID of the step to get. |
required |
Returns:
| Type | Description |
|---|---|
StepRunModel |
The step. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the step doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_step(self, step_id: UUID) -> StepRunModel:
"""Get a step by ID.
Args:
step_id: The ID of the step to get.
Returns:
The step.
Raises:
KeyError: if the step doesn't exist.
"""
get_run_step_inputs(self, step_id)
Get a list of inputs for a specific step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The id of the step to get inputs for. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, zenml.models.pipeline_models.ArtifactModel] |
A dict mapping artifact names to the input artifacts for the step. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_step_inputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get a list of inputs for a specific step.
Args:
step_id: The id of the step to get inputs for.
Returns:
A dict mapping artifact names to the input artifacts for the step.
"""
get_run_step_outputs(self, step_id)
Get a list of outputs for a specific step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The id of the step to get outputs for. |
required |
Returns:
| Type | Description |
|---|---|
Dict[str, zenml.models.pipeline_models.ArtifactModel] |
A dict mapping artifact names to the output artifacts for the step. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_step_outputs(self, step_id: UUID) -> Dict[str, ArtifactModel]:
"""Get a list of outputs for a specific step.
Args:
step_id: The id of the step to get outputs for.
Returns:
A dict mapping artifact names to the output artifacts for the step.
"""
get_run_step_status(self, step_id)
Gets the execution status of a single step.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
step_id |
UUID |
The ID of the step to get the status for. |
required |
Returns:
| Type | Description |
|---|---|
ExecutionStatus |
The status of the step. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_run_step_status(self, step_id: UUID) -> ExecutionStatus:
"""Gets the execution status of a single step.
Args:
step_id: The ID of the step to get the status for.
Returns:
ExecutionStatus: The status of the step.
"""
get_stack(self, stack_id)
Get a stack by its unique ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack_id |
UUID |
The ID of the stack to get. |
required |
Returns:
| Type | Description |
|---|---|
StackModel |
The stack with the given ID. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_stack(self, stack_id: UUID) -> StackModel:
"""Get a stack by its unique ID.
Args:
stack_id: The ID of the stack to get.
Returns:
The stack with the given ID.
Raises:
KeyError: if the stack doesn't exist.
"""
get_stack_component(self, component_id)
Get a stack component by ID.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to get. |
required |
Returns:
| Type | Description |
|---|---|
ComponentModel |
The stack component. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_stack_component(self, component_id: UUID) -> ComponentModel:
"""Get a stack component by ID.
Args:
component_id: The ID of the stack component to get.
Returns:
The stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
get_stack_component_side_effects(self, component_id, run_id, pipeline_id, stack_id)
Get the side effects of a stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component_id |
UUID |
The ID of the stack component to get side effects for. |
required |
run_id |
UUID |
The ID of the run to get side effects for. |
required |
pipeline_id |
UUID |
The ID of the pipeline to get side effects for. |
required |
stack_id |
UUID |
The ID of the stack to get side effects for. |
required |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_stack_component_side_effects(
self,
component_id: UUID,
run_id: UUID,
pipeline_id: UUID,
stack_id: UUID,
) -> Dict[Any, Any]:
"""Get the side effects of a stack component.
Args:
component_id: The ID of the stack component to get side effects for.
run_id: The ID of the run to get side effects for.
pipeline_id: The ID of the pipeline to get side effects for.
stack_id: The ID of the stack to get side effects for.
"""
get_store_info(self)
Get information about the store.
Returns:
| Type | Description |
|---|---|
ServerModel |
Information about the store. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_store_info(self) -> ServerModel:
"""Get information about the store.
Returns:
Information about the store.
"""
get_team(self, team_name_or_id)
Gets a specific team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team to get. |
required |
Returns:
| Type | Description |
|---|---|
TeamModel |
The requested team. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no team with the given name or ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_team(self, team_name_or_id: Union[str, UUID]) -> TeamModel:
"""Gets a specific team.
Args:
team_name_or_id: Name or ID of the team to get.
Returns:
The requested team.
Raises:
KeyError: If no team with the given name or ID exists.
"""
get_teams_for_user(self, user_name_or_id)
Fetches all teams for a user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user for which to get all teams. |
required |
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.TeamModel] |
A list of all teams that the user is part of. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_teams_for_user(
self, user_name_or_id: Union[str, UUID]
) -> List[TeamModel]:
"""Fetches all teams for a user.
Args:
user_name_or_id: The name or ID of the user for which to get all
teams.
Returns:
A list of all teams that the user is part of.
Raises:
KeyError: If no user with the given ID exists.
"""
get_user(self, user_name_or_id)
Gets a specific user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the user to get. |
required |
Returns:
| Type | Description |
|---|---|
UserModel |
The requested user, if it was found. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given name or ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_user(self, user_name_or_id: Union[str, UUID]) -> UserModel:
"""Gets a specific user.
Args:
user_name_or_id: The name or ID of the user to get.
Returns:
The requested user, if it was found.
Raises:
KeyError: If no user with the given name or ID exists.
"""
get_users_for_team(self, team_name_or_id)
Fetches all users of a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team_name_or_id |
Union[str, uuid.UUID] |
The name or ID of the team for which to get users. |
required |
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.UserModel] |
A list of all users that are part of the team. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no team with the given ID exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def get_users_for_team(
self, team_name_or_id: Union[str, UUID]
) -> List[UserModel]:
"""Fetches all users of a team.
Args:
team_name_or_id: The name or ID of the team for which to get users.
Returns:
A list of all users that are part of the team.
Raises:
KeyError: If no team with the given ID exists.
"""
list_artifacts(self, artifact_uri=None)
Lists all artifacts.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
artifact_uri |
Optional[str] |
If specified, only artifacts with the given URI will be returned. |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.ArtifactModel] |
A list of all artifacts. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_artifacts(
self, artifact_uri: Optional[str] = None
) -> List[ArtifactModel]:
"""Lists all artifacts.
Args:
artifact_uri: If specified, only artifacts with the given URI will
be returned.
Returns:
A list of all artifacts.
"""
list_flavors(self, project_name_or_id=None, user_name_or_id=None, component_type=None, name=None, is_shared=None)
List all stack component flavors matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Optionally filter by the Project to which the component flavors belong |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter by the owner |
None |
component_type |
Optional[zenml.enums.StackComponentType] |
Optionally filter by type of stack component |
None |
name |
Optional[str] |
Optionally filter flavors by name |
None |
is_shared |
Optional[bool] |
Optionally filter out flavors by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.flavor_models.FlavorModel] |
List of all the stack component flavors matching the given criteria. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the project doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_flavors(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_type: Optional[StackComponentType] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[FlavorModel]:
"""List all stack component flavors matching the given filter criteria.
Args:
project_name_or_id: Optionally filter by the Project to which the
component flavors belong
user_name_or_id: Optionally filter by the owner
component_type: Optionally filter by type of stack component
name: Optionally filter flavors by name
is_shared: Optionally filter out flavors by whether they are
shared or not
Returns:
List of all the stack component flavors matching the given criteria.
Raises:
KeyError: if the project doesn't exist.
"""
list_pipelines(self, project_name_or_id=None, user_name_or_id=None, name=None)
List all pipelines in the project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only list pipelines in this project. |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only list pipelines from this user. |
None |
name |
Optional[str] |
If provided, only list pipelines with this name. |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.PipelineModel] |
A list of pipelines. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the project does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_pipelines(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
name: Optional[str] = None,
) -> List[PipelineModel]:
"""List all pipelines in the project.
Args:
project_name_or_id: If provided, only list pipelines in this project.
user_name_or_id: If provided, only list pipelines from this user.
name: If provided, only list pipelines with this name.
Returns:
A list of pipelines.
Raises:
KeyError: if the project does not exist.
"""
list_projects(self)
List all projects.
Returns:
| Type | Description |
|---|---|
List[zenml.models.project_models.ProjectModel] |
A list of all projects. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_projects(self) -> List[ProjectModel]:
"""List all projects.
Returns:
A list of all projects.
"""
list_role_assignments(self, project_name_or_id=None, team_name_or_id=None, user_name_or_id=None)
List all role assignments.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only list assignments for the given project |
None |
team_name_or_id |
Union[str, uuid.UUID] |
If provided, only list assignments for the given team |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only list assignments for the given user |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.RoleAssignmentModel] |
A list of all role assignments. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_role_assignments(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
team_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
) -> List[RoleAssignmentModel]:
"""List all role assignments.
Args:
project_name_or_id: If provided, only list assignments for the given
project
team_name_or_id: If provided, only list assignments for the given
team
user_name_or_id: If provided, only list assignments for the given
user
Returns:
A list of all role assignments.
"""
list_roles(self)
List all roles.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.RoleModel] |
A list of all roles. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_roles(self) -> List[RoleModel]:
"""List all roles.
Returns:
A list of all roles.
"""
list_run_steps(self, run_id)
Gets all steps in a pipeline run.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
run_id |
UUID |
The ID of the pipeline run for which to list runs. |
required |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.StepRunModel] |
A mapping from step names to step models for all steps in the run. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_run_steps(self, run_id: UUID) -> List[StepRunModel]:
"""Gets all steps in a pipeline run.
Args:
run_id: The ID of the pipeline run for which to list runs.
Returns:
A mapping from step names to step models for all steps in the run.
"""
list_runs(self, project_name_or_id=None, stack_id=None, component_id=None, run_name=None, user_name_or_id=None, pipeline_id=None, unlisted=False)
Gets all pipeline runs.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
If provided, only return runs for this project. |
None |
stack_id |
Optional[uuid.UUID] |
If provided, only return runs for this stack. |
None |
component_id |
Optional[uuid.UUID] |
Optionally filter for runs that used the component |
None |
run_name |
Optional[str] |
Run name if provided |
None |
user_name_or_id |
Union[str, uuid.UUID] |
If provided, only return runs for this user. |
None |
pipeline_id |
Optional[uuid.UUID] |
If provided, only return runs for this pipeline. |
None |
unlisted |
bool |
If True, only return unlisted runs that are not
associated with any pipeline (filter by |
False |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.PipelineRunModel] |
A list of all pipeline runs. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_runs(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
stack_id: Optional[UUID] = None,
component_id: Optional[UUID] = None,
run_name: Optional[str] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
pipeline_id: Optional[UUID] = None,
unlisted: bool = False,
) -> List[PipelineRunModel]:
"""Gets all pipeline runs.
Args:
project_name_or_id: If provided, only return runs for this project.
stack_id: If provided, only return runs for this stack.
component_id: Optionally filter for runs that used the
component
run_name: Run name if provided
user_name_or_id: If provided, only return runs for this user.
pipeline_id: If provided, only return runs for this pipeline.
unlisted: If True, only return unlisted runs that are not
associated with any pipeline (filter by `pipeline_id==None`).
Returns:
A list of all pipeline runs.
"""
list_stack_components(self, project_name_or_id=None, user_name_or_id=None, type=None, flavor_name=None, name=None, is_shared=None)
List all stack components matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
The ID or name of the Project to which the stack components belong |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter stack components by the owner |
None |
type |
Optional[str] |
Optionally filter by type of stack component |
None |
flavor_name |
Optional[str] |
Optionally filter by flavor |
None |
name |
Optional[str] |
Optionally filter stack component by name |
None |
is_shared |
Optional[bool] |
Optionally filter out stack component by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.component_model.ComponentModel] |
A list of all stack components matching the filter criteria. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the project doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_stack_components(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
type: Optional[str] = None,
flavor_name: Optional[str] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[ComponentModel]:
"""List all stack components matching the given filter criteria.
Args:
project_name_or_id: The ID or name of the Project to which the stack
components belong
user_name_or_id: Optionally filter stack components by the owner
type: Optionally filter by type of stack component
flavor_name: Optionally filter by flavor
name: Optionally filter stack component by name
is_shared: Optionally filter out stack component by whether they are
shared or not
Returns:
A list of all stack components matching the filter criteria.
Raises:
KeyError: if the project doesn't exist.
"""
list_stacks(self, project_name_or_id=None, user_name_or_id=None, component_id=None, name=None, is_shared=None)
List all stacks matching the given filter criteria.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project_name_or_id |
Union[str, uuid.UUID] |
Id or name of the Project containing the stack |
None |
user_name_or_id |
Union[str, uuid.UUID] |
Optionally filter stacks by their owner |
None |
component_id |
Optional[uuid.UUID] |
Optionally filter for stacks that contain the component |
None |
name |
Optional[str] |
Optionally filter stacks by their name |
None |
is_shared |
Optional[bool] |
Optionally filter out stacks by whether they are shared or not |
None |
Returns:
| Type | Description |
|---|---|
List[zenml.models.stack_models.StackModel] |
A list of all stacks matching the filter criteria. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the project doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_stacks(
self,
project_name_or_id: Optional[Union[str, UUID]] = None,
user_name_or_id: Optional[Union[str, UUID]] = None,
component_id: Optional[UUID] = None,
name: Optional[str] = None,
is_shared: Optional[bool] = None,
) -> List[StackModel]:
"""List all stacks matching the given filter criteria.
Args:
project_name_or_id: Id or name of the Project containing the stack
user_name_or_id: Optionally filter stacks by their owner
component_id: Optionally filter for stacks that contain the
component
name: Optionally filter stacks by their name
is_shared: Optionally filter out stacks by whether they are shared
or not
Returns:
A list of all stacks matching the filter criteria.
Raises:
KeyError: if the project doesn't exist.
"""
list_steps(self, pipeline_id)
List all steps.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline_id |
UUID |
The ID of the pipeline to list steps for. |
required |
Returns:
| Type | Description |
|---|---|
List[zenml.models.pipeline_models.StepRunModel] |
A list of all steps. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_steps(self, pipeline_id: UUID) -> List[StepRunModel]:
"""List all steps.
Args:
pipeline_id: The ID of the pipeline to list steps for.
Returns:
A list of all steps.
"""
list_teams(self)
List all teams.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.TeamModel] |
A list of all teams. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_teams(self) -> List[TeamModel]:
"""List all teams.
Returns:
A list of all teams.
"""
list_users(self)
List all users.
Returns:
| Type | Description |
|---|---|
List[zenml.models.user_management_models.UserModel] |
A list of all users. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def list_users(self) -> List[UserModel]:
"""List all users.
Returns:
A list of all users.
"""
remove_user_from_team(self, user_name_or_id, team_name_or_id)
Removes a user from a team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user to remove from the team. |
required |
team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the team from which to remove the user. |
required |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If the team or user does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def remove_user_from_team(
self,
user_name_or_id: Union[str, UUID],
team_name_or_id: Union[str, UUID],
) -> None:
"""Removes a user from a team.
Args:
user_name_or_id: Name or ID of the user to remove from the team.
team_name_or_id: Name or ID of the team from which to remove the user.
Raises:
KeyError: If the team or user does not exist.
"""
revoke_role(self, role_name_or_id, user_or_team_name_or_id, is_user=True, project_name_or_id=None)
Revokes a role from a user or team for a given project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role_name_or_id |
Union[str, uuid.UUID] |
ID of the role to revoke. |
required |
user_or_team_name_or_id |
Union[str, uuid.UUID] |
Name or ID of the user or team from which to revoke the role. |
required |
is_user |
bool |
Whether |
True |
project_name_or_id |
Union[str, uuid.UUID] |
Optional ID of a project in which to revoke the role. If this is not provided, the role will be revoked globally. |
None |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If the role, user, team, or project does not exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def revoke_role(
self,
role_name_or_id: Union[str, UUID],
user_or_team_name_or_id: Union[str, UUID],
is_user: bool = True,
project_name_or_id: Optional[Union[str, UUID]] = None,
) -> None:
"""Revokes a role from a user or team for a given project.
Args:
role_name_or_id: ID of the role to revoke.
user_or_team_name_or_id: Name or ID of the user or team from which
to revoke the role.
is_user: Whether `user_or_team_id` refers to a user or a team.
project_name_or_id: Optional ID of a project in which to revoke
the role. If this is not provided, the role will be revoked
globally.
Raises:
KeyError: If the role, user, team, or project does not exists.
"""
update_flavor(self, flavor)
Update an existing stack component flavor.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
flavor |
FlavorModel |
The stack component flavor to use for the update. |
required |
Returns:
| Type | Description |
|---|---|
FlavorModel |
The updated stack component flavor. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component flavor doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_flavor(self, flavor: FlavorModel) -> FlavorModel:
"""Update an existing stack component flavor.
Args:
flavor: The stack component flavor to use for the update.
Returns:
The updated stack component flavor.
Raises:
KeyError: if the stack component flavor doesn't exist.
"""
update_pipeline(self, pipeline)
Updates a pipeline.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
pipeline |
PipelineModel |
The pipeline to use for the update. |
required |
Returns:
| Type | Description |
|---|---|
PipelineModel |
The updated pipeline. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the pipeline doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_pipeline(self, pipeline: PipelineModel) -> PipelineModel:
"""Updates a pipeline.
Args:
pipeline: The pipeline to use for the update.
Returns:
The updated pipeline.
Raises:
KeyError: if the pipeline doesn't exist.
"""
update_project(self, project)
Update an existing project.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
project |
ProjectModel |
The project to use for the update. |
required |
Returns:
| Type | Description |
|---|---|
ProjectModel |
The updated project. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the project does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_project(self, project: ProjectModel) -> ProjectModel:
"""Update an existing project.
Args:
project: The project to use for the update.
Returns:
The updated project.
Raises:
KeyError: if the project does not exist.
"""
update_role(self, role)
Update an existing role.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
role |
RoleModel |
The role to use for the update. |
required |
Returns:
| Type | Description |
|---|---|
RoleModel |
The updated role. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the role does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_role(self, role: RoleModel) -> RoleModel:
"""Update an existing role.
Args:
role: The role to use for the update.
Returns:
The updated role.
Raises:
KeyError: if the role does not exist.
"""
update_stack(self, stack)
Update a stack.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
stack |
StackModel |
The stack to use for the update. |
required |
Returns:
| Type | Description |
|---|---|
StackModel |
The updated stack. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_stack(
self,
stack: StackModel,
) -> StackModel:
"""Update a stack.
Args:
stack: The stack to use for the update.
Returns:
The updated stack.
Raises:
KeyError: if the stack doesn't exist.
"""
update_stack_component(self, component)
Update an existing stack component.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
component |
ComponentModel |
The stack component to use for the update. |
required |
Returns:
| Type | Description |
|---|---|
ComponentModel |
The updated stack component. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the stack component doesn't exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_stack_component(
self,
component: ComponentModel,
) -> ComponentModel:
"""Update an existing stack component.
Args:
component: The stack component to use for the update.
Returns:
The updated stack component.
Raises:
KeyError: if the stack component doesn't exist.
"""
update_team(self, team)
Update an existing team.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
team |
TeamModel |
The team to use for the update. |
required |
Returns:
| Type | Description |
|---|---|
TeamModel |
The updated team. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
if the team does not exist. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_team(self, team: TeamModel) -> TeamModel:
"""Update an existing team.
Args:
team: The team to use for the update.
Returns:
The updated team.
Raises:
KeyError: if the team does not exist.
"""
update_user(self, user)
Updates an existing user.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user |
UserModel |
The user model to use for the update. |
required |
Returns:
| Type | Description |
|---|---|
UserModel |
The updated user. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given name exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def update_user(self, user: UserModel) -> UserModel:
"""Updates an existing user.
Args:
user: The user model to use for the update.
Returns:
The updated user.
Raises:
KeyError: If no user with the given name exists.
"""
user_email_opt_in(self, user_name_or_id, user_opt_in_response, email=None)
Persist user response to the email prompt.
Parameters:
| Name | Type | Description | Default |
|---|---|---|---|
user_name_or_id |
Union[str, uuid.UUID] |
The name or the ID of the user. |
required |
user_opt_in_response |
bool |
Whether this email should be associated with the user id in the telemetry |
required |
email |
Optional[str] |
The users email |
None |
Returns:
| Type | Description |
|---|---|
UserModel |
The updated user. |
Exceptions:
| Type | Description |
|---|---|
KeyError |
If no user with the given name exists. |
Source code in zenml/zen_stores/zen_store_interface.py
@abstractmethod
def user_email_opt_in(
self,
user_name_or_id: Union[str, UUID],
user_opt_in_response: bool,
email: Optional[str] = None,
) -> UserModel:
"""Persist user response to the email prompt.
Args:
user_name_or_id: The name or the ID of the user.
user_opt_in_response: Whether this email should be associated
with the user id in the telemetry
email: The users email
Returns:
The updated user.
Raises:
KeyError: If no user with the given name exists.
"""