Skypilot Aws
zenml.integrations.skypilot_aws
special
Initialization of the Skypilot AWS integration for ZenML.
The Skypilot integration sub-module powers an alternative to the local orchestrator for a remote orchestration of ZenML pipelines on VMs.
SkypilotAWSIntegration (Integration)
Definition of Skypilot AWS Integration for ZenML.
Source code in zenml/integrations/skypilot_aws/__init__.py
class SkypilotAWSIntegration(Integration):
"""Definition of Skypilot AWS Integration for ZenML."""
NAME = SKYPILOT_AWS
# all 0.6.x versions of skypilot[aws] are compatible
REQUIREMENTS = ["skypilot[aws]~=0.6.0"]
APT_PACKAGES = ["openssh-client", "rsync"]
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
"""Declare the stack component flavors for the Skypilot AWS integration.
Returns:
List of stack component flavors for this integration.
"""
from zenml.integrations.skypilot_aws.flavors import (
SkypilotAWSOrchestratorFlavor,
)
return [SkypilotAWSOrchestratorFlavor]
flavors()
classmethod
Declare the stack component flavors for the Skypilot AWS integration.
Returns:
Type | Description |
---|---|
List[Type[zenml.stack.flavor.Flavor]] |
List of stack component flavors for this integration. |
Source code in zenml/integrations/skypilot_aws/__init__.py
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
"""Declare the stack component flavors for the Skypilot AWS integration.
Returns:
List of stack component flavors for this integration.
"""
from zenml.integrations.skypilot_aws.flavors import (
SkypilotAWSOrchestratorFlavor,
)
return [SkypilotAWSOrchestratorFlavor]
flavors
special
Skypilot integration flavor for Skypilot AWS orchestrator.
skypilot_orchestrator_aws_vm_flavor
Skypilot orchestrator AWS flavor.
SkypilotAWSOrchestratorConfig (SkypilotBaseOrchestratorConfig, SkypilotAWSOrchestratorSettings)
Skypilot orchestrator config.
Source code in zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py
class SkypilotAWSOrchestratorConfig(
SkypilotBaseOrchestratorConfig, SkypilotAWSOrchestratorSettings
):
"""Skypilot orchestrator config."""
SkypilotAWSOrchestratorFlavor (BaseOrchestratorFlavor)
Flavor for the Skypilot AWS orchestrator.
Source code in zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py
class SkypilotAWSOrchestratorFlavor(BaseOrchestratorFlavor):
"""Flavor for the Skypilot AWS orchestrator."""
@property
def name(self) -> str:
"""Name of the orchestrator flavor.
Returns:
Name of the orchestrator flavor.
"""
return SKYPILOT_AWS_ORCHESTRATOR_FLAVOR
@property
def service_connector_requirements(
self,
) -> Optional[ServiceConnectorRequirements]:
"""Service connector resource requirements for service connectors.
Specifies resource requirements that are used to filter the available
service connector types that are compatible with this flavor.
Returns:
Requirements for compatible service connectors, if a service
connector is required for this flavor.
"""
return ServiceConnectorRequirements(
resource_type="aws-generic",
)
@property
def docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_docs_url()
@property
def sdk_docs_url(self) -> Optional[str]:
"""A url to point at SDK docs explaining this flavor.
Returns:
A flavor SDK docs url.
"""
return self.generate_default_sdk_docs_url()
@property
def logo_url(self) -> str:
"""A url to represent the flavor in the dashboard.
Returns:
The flavor logo.
"""
return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/orchestrator/aws-skypilot.png"
@property
def config_class(self) -> Type[BaseOrchestratorConfig]:
"""Config class for the base orchestrator flavor.
Returns:
The config class.
"""
return SkypilotAWSOrchestratorConfig
@property
def implementation_class(self) -> Type["SkypilotAWSOrchestrator"]:
"""Implementation class for this flavor.
Returns:
Implementation class for this flavor.
"""
from zenml.integrations.skypilot_aws.orchestrators import (
SkypilotAWSOrchestrator,
)
return SkypilotAWSOrchestrator
config_class: Type[zenml.orchestrators.base_orchestrator.BaseOrchestratorConfig]
property
readonly
Config class for the base orchestrator flavor.
Returns:
Type | Description |
---|---|
Type[zenml.orchestrators.base_orchestrator.BaseOrchestratorConfig] |
The config class. |
docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
Type | Description |
---|---|
Optional[str] |
A flavor docs url. |
implementation_class: Type[SkypilotAWSOrchestrator]
property
readonly
Implementation class for this flavor.
Returns:
Type | Description |
---|---|
Type[SkypilotAWSOrchestrator] |
Implementation class for this flavor. |
logo_url: str
property
readonly
A url to represent the flavor in the dashboard.
Returns:
Type | Description |
---|---|
str |
The flavor logo. |
name: str
property
readonly
Name of the orchestrator flavor.
Returns:
Type | Description |
---|---|
str |
Name of the orchestrator flavor. |
sdk_docs_url: Optional[str]
property
readonly
A url to point at SDK docs explaining this flavor.
Returns:
Type | Description |
---|---|
Optional[str] |
A flavor SDK docs url. |
service_connector_requirements: Optional[zenml.models.v2.misc.service_connector_type.ServiceConnectorRequirements]
property
readonly
Service connector resource requirements for service connectors.
Specifies resource requirements that are used to filter the available service connector types that are compatible with this flavor.
Returns:
Type | Description |
---|---|
Optional[zenml.models.v2.misc.service_connector_type.ServiceConnectorRequirements] |
Requirements for compatible service connectors, if a service connector is required for this flavor. |
SkypilotAWSOrchestratorSettings (SkypilotBaseOrchestratorSettings)
Skypilot orchestrator settings.
Source code in zenml/integrations/skypilot_aws/flavors/skypilot_orchestrator_aws_vm_flavor.py
class SkypilotAWSOrchestratorSettings(SkypilotBaseOrchestratorSettings):
"""Skypilot orchestrator settings."""
orchestrators
special
Initialization of the Skypilot AWS ZenML orchestrator.
skypilot_aws_vm_orchestrator
Implementation of the a Skypilot based AWS VM orchestrator.
SkypilotAWSOrchestrator (SkypilotBaseOrchestrator)
Orchestrator responsible for running pipelines remotely in a VM on AWS.
This orchestrator does not support running on a schedule.
Source code in zenml/integrations/skypilot_aws/orchestrators/skypilot_aws_vm_orchestrator.py
class SkypilotAWSOrchestrator(SkypilotBaseOrchestrator):
"""Orchestrator responsible for running pipelines remotely in a VM on AWS.
This orchestrator does not support running on a schedule.
"""
DEFAULT_INSTANCE_TYPE: str = "t3.xlarge"
@property
def cloud(self) -> sky.clouds.Cloud:
"""The type of sky cloud to use.
Returns:
A `sky.clouds.Cloud` instance.
"""
return sky.clouds.AWS()
@property
def config(self) -> SkypilotAWSOrchestratorConfig:
"""Returns the `SkypilotAWSOrchestratorConfig` config.
Returns:
The configuration.
"""
return cast(SkypilotAWSOrchestratorConfig, self._config)
@property
def settings_class(self) -> Optional[Type["BaseSettings"]]:
"""Settings class for the Skypilot orchestrator.
Returns:
The settings class.
"""
return SkypilotAWSOrchestratorSettings
def prepare_environment_variable(self, set: bool = True) -> None:
"""Set up Environment variables that are required for the orchestrator.
Args:
set: Whether to set the environment variables or not.
Raises:
ValueError: If no service connector is found.
"""
connector = self.get_connector()
if connector is None:
raise ValueError(
"No service connector found. Please make sure to set up a connector "
"that is compatible with this orchestrator."
)
if set:
# The AWS connector creates a local configuration profile with the name computed from
# the first 8 digits of its UUID.
aws_profile = f"zenml-{str(connector.id)[:8]}"
os.environ[ENV_AWS_PROFILE] = aws_profile
else:
os.environ.pop(ENV_AWS_PROFILE, None)
cloud: sky.clouds.Cloud
property
readonly
The type of sky cloud to use.
Returns:
Type | Description |
---|---|
sky.clouds.Cloud |
A |
config: SkypilotAWSOrchestratorConfig
property
readonly
Returns the SkypilotAWSOrchestratorConfig
config.
Returns:
Type | Description |
---|---|
SkypilotAWSOrchestratorConfig |
The configuration. |
settings_class: Optional[Type[BaseSettings]]
property
readonly
Settings class for the Skypilot orchestrator.
Returns:
Type | Description |
---|---|
Optional[Type[BaseSettings]] |
The settings class. |
prepare_environment_variable(self, set=True)
Set up Environment variables that are required for the orchestrator.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
set |
bool |
Whether to set the environment variables or not. |
True |
Exceptions:
Type | Description |
---|---|
ValueError |
If no service connector is found. |
Source code in zenml/integrations/skypilot_aws/orchestrators/skypilot_aws_vm_orchestrator.py
def prepare_environment_variable(self, set: bool = True) -> None:
"""Set up Environment variables that are required for the orchestrator.
Args:
set: Whether to set the environment variables or not.
Raises:
ValueError: If no service connector is found.
"""
connector = self.get_connector()
if connector is None:
raise ValueError(
"No service connector found. Please make sure to set up a connector "
"that is compatible with this orchestrator."
)
if set:
# The AWS connector creates a local configuration profile with the name computed from
# the first 8 digits of its UUID.
aws_profile = f"zenml-{str(connector.id)[:8]}"
os.environ[ENV_AWS_PROFILE] = aws_profile
else:
os.environ.pop(ENV_AWS_PROFILE, None)