Image Builders
zenml.image_builders
special
Image builders allow you to build container images.
base_image_builder
Base class for all ZenML image builders.
BaseImageBuilder (StackComponent, ABC)
Base class for all ZenML image builders.
Source code in zenml/image_builders/base_image_builder.py
class BaseImageBuilder(StackComponent, ABC):
"""Base class for all ZenML image builders."""
@property
def config(self) -> BaseImageBuilderConfig:
"""The stack component configuration.
Returns:
The configuration.
"""
return cast(BaseImageBuilderConfig, self._config)
@property
def build_context_class(self) -> Type["BuildContext"]:
"""Build context class to use.
The default build context class creates a build context that works
for the Docker daemon. Override this method if your image builder
requires a custom context.
Returns:
The build context class.
"""
from zenml.image_builders import BuildContext
return BuildContext
@property
@abstractmethod
def is_building_locally(self) -> bool:
"""Whether the image builder builds the images on the client machine.
Returns:
True if the image builder builds locally, False otherwise.
"""
@abstractmethod
def build(
self,
image_name: str,
build_context: "BuildContext",
docker_build_options: Dict[str, Any],
container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
"""Builds a Docker image.
If a container registry is passed, the image will be pushed to that
registry.
Args:
image_name: Name of the image to build.
build_context: The build context to use for the image.
docker_build_options: Docker build options.
container_registry: Optional container registry to push to.
Returns:
The Docker image repo digest or name.
"""
@staticmethod
def _upload_build_context(
build_context: "BuildContext",
parent_path_directory_name: str,
) -> str:
"""Uploads a Docker image build context to a remote location.
Args:
build_context: The build context to upload.
parent_path_directory_name: The name of the directory to upload
the build context to. It will be appended to the artifact
store path to create the parent path where the build context
will be uploaded to.
Returns:
The path to the uploaded build context.
"""
artifact_store = Client().active_stack.artifact_store
parent_path = f"{artifact_store.path}/{parent_path_directory_name}"
fileio.makedirs(parent_path)
hash_ = hashlib.sha1() # nosec
with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as f:
build_context.write_archive(f, use_gzip=True)
while True:
data = f.read(64 * 1024)
if not data:
break
hash_.update(data)
filename = f"{hash_.hexdigest()}.tar.gz"
filepath = f"{parent_path}/{filename}"
if not fileio.exists(filepath):
logger.info("Uploading build context to `%s`.", filepath)
fileio.copy(f.name, filepath)
else:
logger.info("Build context already exists, not uploading.")
os.unlink(f.name)
return filepath
build_context_class: Type[BuildContext]
property
readonly
Build context class to use.
The default build context class creates a build context that works for the Docker daemon. Override this method if your image builder requires a custom context.
Returns:
Type | Description |
---|---|
Type[BuildContext] |
The build context class. |
config: BaseImageBuilderConfig
property
readonly
The stack component configuration.
Returns:
Type | Description |
---|---|
BaseImageBuilderConfig |
The configuration. |
is_building_locally: bool
property
readonly
Whether the image builder builds the images on the client machine.
Returns:
Type | Description |
---|---|
bool |
True if the image builder builds locally, False otherwise. |
build(self, image_name, build_context, docker_build_options, container_registry=None)
Builds a Docker image.
If a container registry is passed, the image will be pushed to that registry.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
image_name |
str |
Name of the image to build. |
required |
build_context |
BuildContext |
The build context to use for the image. |
required |
docker_build_options |
Dict[str, Any] |
Docker build options. |
required |
container_registry |
Optional[BaseContainerRegistry] |
Optional container registry to push to. |
None |
Returns:
Type | Description |
---|---|
str |
The Docker image repo digest or name. |
Source code in zenml/image_builders/base_image_builder.py
@abstractmethod
def build(
self,
image_name: str,
build_context: "BuildContext",
docker_build_options: Dict[str, Any],
container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
"""Builds a Docker image.
If a container registry is passed, the image will be pushed to that
registry.
Args:
image_name: Name of the image to build.
build_context: The build context to use for the image.
docker_build_options: Docker build options.
container_registry: Optional container registry to push to.
Returns:
The Docker image repo digest or name.
"""
BaseImageBuilderConfig (StackComponentConfig)
Base config for image builders.
Source code in zenml/image_builders/base_image_builder.py
class BaseImageBuilderConfig(StackComponentConfig):
"""Base config for image builders."""
BaseImageBuilderFlavor (Flavor, ABC)
Base class for all ZenML image builder flavors.
Source code in zenml/image_builders/base_image_builder.py
class BaseImageBuilderFlavor(Flavor, ABC):
"""Base class for all ZenML image builder flavors."""
@property
def type(self) -> StackComponentType:
"""Returns the flavor type.
Returns:
The flavor type.
"""
return StackComponentType.IMAGE_BUILDER
@property
def config_class(self) -> Type[BaseImageBuilderConfig]:
"""Config class.
Returns:
The config class.
"""
return BaseImageBuilderConfig
@property
def implementation_class(self) -> Type[BaseImageBuilder]:
"""Implementation class.
Returns:
The implementation class.
"""
return BaseImageBuilder
config_class: Type[zenml.image_builders.base_image_builder.BaseImageBuilderConfig]
property
readonly
Config class.
Returns:
Type | Description |
---|---|
Type[zenml.image_builders.base_image_builder.BaseImageBuilderConfig] |
The config class. |
implementation_class: Type[zenml.image_builders.base_image_builder.BaseImageBuilder]
property
readonly
Implementation class.
Returns:
Type | Description |
---|---|
Type[zenml.image_builders.base_image_builder.BaseImageBuilder] |
The implementation class. |
type: StackComponentType
property
readonly
Returns the flavor type.
Returns:
Type | Description |
---|---|
StackComponentType |
The flavor type. |
build_context
Image build context.
BuildContext (Archivable)
Image build context.
This class is responsible for creating an archive of the files needed to build a container image.
Source code in zenml/image_builders/build_context.py
class BuildContext(Archivable):
"""Image build context.
This class is responsible for creating an archive of the files needed to
build a container image.
"""
def __init__(
self,
root: Optional[str] = None,
dockerignore_file: Optional[str] = None,
) -> None:
"""Initializes a build context.
Args:
root: Optional root directory for the build context.
dockerignore_file: Optional path to a dockerignore file. If not
given, a file called `.dockerignore` in the build context root
directory will be used instead if it exists.
"""
super().__init__()
self._root = root
self._dockerignore_file = dockerignore_file
@property
def dockerignore_file(self) -> Optional[str]:
"""The dockerignore file to use.
Returns:
Path to the dockerignore file to use.
"""
if self._dockerignore_file:
return self._dockerignore_file
if self._root:
default_dockerignore_path = os.path.join(
self._root, ".dockerignore"
)
if fileio.exists(default_dockerignore_path):
return default_dockerignore_path
return None
def write_archive(
self, output_file: IO[bytes], use_gzip: bool = True
) -> None:
"""Writes an archive of the build context to the given file.
Args:
output_file: The file to write the archive to.
use_gzip: Whether to use `gzip` to compress the file.
"""
from docker.utils import build as docker_build_utils
files = self.get_files()
extra_files = self.get_extra_files()
context_archive = docker_build_utils.create_archive(
fileobj=output_file,
root=self._root,
files=sorted(files.keys()),
gzip=use_gzip,
extra_files=list(extra_files.items()),
)
build_context_size = os.path.getsize(context_archive.name)
if (
self._root
and build_context_size > 50 * 1024 * 1024
and not self.dockerignore_file
):
# The build context exceeds 50MiB and we didn't find any excludes
# in dockerignore files -> remind to specify a .dockerignore file
logger.warning(
"Build context size for docker image: `%s`. If you believe this is "
"unreasonably large, make sure to include a `.dockerignore` file "
"at the root of your build context `%s` or specify a custom file "
"in the Docker configuration when defining your pipeline.",
string_utils.get_human_readable_filesize(build_context_size),
os.path.join(self._root, ".dockerignore"),
)
def get_files(self) -> Dict[str, str]:
"""Gets all regular files that should be included in the archive.
Returns:
A dict {path_in_archive: path_on_filesystem} for all regular files
in the archive.
"""
if self._root:
from docker.utils import build as docker_build_utils
exclude_patterns = self._get_exclude_patterns()
archive_paths = cast(
Set[str],
docker_build_utils.exclude_paths(
self._root, patterns=exclude_patterns
),
)
return {
archive_path: os.path.join(self._root, archive_path)
for archive_path in archive_paths
}
else:
return {}
def _get_exclude_patterns(self) -> List[str]:
"""Gets all exclude patterns from the dockerignore file.
Returns:
The exclude patterns from the dockerignore file.
"""
dockerignore = self.dockerignore_file
if dockerignore:
patterns = self._parse_dockerignore(dockerignore)
# Always include the .zen directory
patterns.append(f"!/{REPOSITORY_DIRECTORY_NAME}")
return patterns
else:
logger.info(
"No `.dockerignore` found, including all files inside build "
"context.",
)
return []
@staticmethod
def _parse_dockerignore(dockerignore_path: str) -> List[str]:
"""Parses a dockerignore file and returns a list of patterns to ignore.
Args:
dockerignore_path: Path to the dockerignore file.
Returns:
List of patterns to ignore.
"""
try:
file_content = io_utils.read_file_contents_as_string(
dockerignore_path
)
except FileNotFoundError:
logger.warning(
"Unable to find dockerignore file at path '%s'.",
dockerignore_path,
)
return []
exclude_patterns = []
for line in file_content.split("\n"):
line = line.strip()
if line and not line.startswith("#"):
exclude_patterns.append(line)
return exclude_patterns
dockerignore_file: Optional[str]
property
readonly
The dockerignore file to use.
Returns:
Type | Description |
---|---|
Optional[str] |
Path to the dockerignore file to use. |
__init__(self, root=None, dockerignore_file=None)
special
Initializes a build context.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
root |
Optional[str] |
Optional root directory for the build context. |
None |
dockerignore_file |
Optional[str] |
Optional path to a dockerignore file. If not
given, a file called |
None |
Source code in zenml/image_builders/build_context.py
def __init__(
self,
root: Optional[str] = None,
dockerignore_file: Optional[str] = None,
) -> None:
"""Initializes a build context.
Args:
root: Optional root directory for the build context.
dockerignore_file: Optional path to a dockerignore file. If not
given, a file called `.dockerignore` in the build context root
directory will be used instead if it exists.
"""
super().__init__()
self._root = root
self._dockerignore_file = dockerignore_file
get_files(self)
Gets all regular files that should be included in the archive.
Returns:
Type | Description |
---|---|
A dict {path_in_archive |
path_on_filesystem} for all regular files in the archive. |
Source code in zenml/image_builders/build_context.py
def get_files(self) -> Dict[str, str]:
"""Gets all regular files that should be included in the archive.
Returns:
A dict {path_in_archive: path_on_filesystem} for all regular files
in the archive.
"""
if self._root:
from docker.utils import build as docker_build_utils
exclude_patterns = self._get_exclude_patterns()
archive_paths = cast(
Set[str],
docker_build_utils.exclude_paths(
self._root, patterns=exclude_patterns
),
)
return {
archive_path: os.path.join(self._root, archive_path)
for archive_path in archive_paths
}
else:
return {}
write_archive(self, output_file, use_gzip=True)
Writes an archive of the build context to the given file.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
output_file |
IO[bytes] |
The file to write the archive to. |
required |
use_gzip |
bool |
Whether to use |
True |
Source code in zenml/image_builders/build_context.py
def write_archive(
self, output_file: IO[bytes], use_gzip: bool = True
) -> None:
"""Writes an archive of the build context to the given file.
Args:
output_file: The file to write the archive to.
use_gzip: Whether to use `gzip` to compress the file.
"""
from docker.utils import build as docker_build_utils
files = self.get_files()
extra_files = self.get_extra_files()
context_archive = docker_build_utils.create_archive(
fileobj=output_file,
root=self._root,
files=sorted(files.keys()),
gzip=use_gzip,
extra_files=list(extra_files.items()),
)
build_context_size = os.path.getsize(context_archive.name)
if (
self._root
and build_context_size > 50 * 1024 * 1024
and not self.dockerignore_file
):
# The build context exceeds 50MiB and we didn't find any excludes
# in dockerignore files -> remind to specify a .dockerignore file
logger.warning(
"Build context size for docker image: `%s`. If you believe this is "
"unreasonably large, make sure to include a `.dockerignore` file "
"at the root of your build context `%s` or specify a custom file "
"in the Docker configuration when defining your pipeline.",
string_utils.get_human_readable_filesize(build_context_size),
os.path.join(self._root, ".dockerignore"),
)
local_image_builder
Local Docker image builder implementation.
LocalImageBuilder (BaseImageBuilder)
Local image builder implementation.
Source code in zenml/image_builders/local_image_builder.py
class LocalImageBuilder(BaseImageBuilder):
"""Local image builder implementation."""
@property
def config(self) -> LocalImageBuilderConfig:
"""The stack component configuration.
Returns:
The configuration.
"""
return cast(LocalImageBuilderConfig, self._config)
@property
def is_building_locally(self) -> bool:
"""Whether the image builder builds the images on the client machine.
Returns:
True if the image builder builds locally, False otherwise.
"""
return True
@staticmethod
def _check_prerequisites() -> None:
"""Checks that all prerequisites are installed.
Raises:
RuntimeError: If any of the prerequisites are not installed or
running.
"""
if not shutil.which("docker"):
raise RuntimeError(
"`docker` is required to run the local image builder."
)
if not docker_utils.check_docker():
raise RuntimeError(
"Unable to connect to the Docker daemon. There are two "
"common causes for this:\n"
"1) The Docker daemon isn't running.\n"
"2) The Docker client isn't configured correctly. The client "
"loads its configuration from the following file: "
"$HOME/.docker/config.json. If your configuration file is in a "
"different location, set it using the `DOCKER_CONFIG` "
"environment variable."
)
def build(
self,
image_name: str,
build_context: "BuildContext",
docker_build_options: Optional[Dict[str, Any]] = None,
container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
"""Builds and optionally pushes an image using the local Docker client.
Args:
image_name: Name of the image to build and push.
build_context: The build context to use for the image.
docker_build_options: Docker build options.
container_registry: Optional container registry to push to.
Returns:
The Docker image repo digest.
"""
self._check_prerequisites()
if container_registry:
# Use the container registry's docker client, which may be
# authenticated to access additional registries
docker_client = container_registry.docker_client
else:
docker_client = docker_utils._try_get_docker_client_from_env()
with tempfile.TemporaryFile(mode="w+b") as f:
build_context.write_archive(f)
# We use the client api directly here, so we can stream the logs
output_stream = docker_client.images.client.api.build(
fileobj=f,
custom_context=True,
tag=image_name,
**(docker_build_options or {}),
)
docker_utils._process_stream(output_stream)
if container_registry:
return container_registry.push_image(image_name)
else:
return image_name
config: LocalImageBuilderConfig
property
readonly
The stack component configuration.
Returns:
Type | Description |
---|---|
LocalImageBuilderConfig |
The configuration. |
is_building_locally: bool
property
readonly
Whether the image builder builds the images on the client machine.
Returns:
Type | Description |
---|---|
bool |
True if the image builder builds locally, False otherwise. |
build(self, image_name, build_context, docker_build_options=None, container_registry=None)
Builds and optionally pushes an image using the local Docker client.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
image_name |
str |
Name of the image to build and push. |
required |
build_context |
BuildContext |
The build context to use for the image. |
required |
docker_build_options |
Optional[Dict[str, Any]] |
Docker build options. |
None |
container_registry |
Optional[BaseContainerRegistry] |
Optional container registry to push to. |
None |
Returns:
Type | Description |
---|---|
str |
The Docker image repo digest. |
Source code in zenml/image_builders/local_image_builder.py
def build(
self,
image_name: str,
build_context: "BuildContext",
docker_build_options: Optional[Dict[str, Any]] = None,
container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
"""Builds and optionally pushes an image using the local Docker client.
Args:
image_name: Name of the image to build and push.
build_context: The build context to use for the image.
docker_build_options: Docker build options.
container_registry: Optional container registry to push to.
Returns:
The Docker image repo digest.
"""
self._check_prerequisites()
if container_registry:
# Use the container registry's docker client, which may be
# authenticated to access additional registries
docker_client = container_registry.docker_client
else:
docker_client = docker_utils._try_get_docker_client_from_env()
with tempfile.TemporaryFile(mode="w+b") as f:
build_context.write_archive(f)
# We use the client api directly here, so we can stream the logs
output_stream = docker_client.images.client.api.build(
fileobj=f,
custom_context=True,
tag=image_name,
**(docker_build_options or {}),
)
docker_utils._process_stream(output_stream)
if container_registry:
return container_registry.push_image(image_name)
else:
return image_name
LocalImageBuilderConfig (BaseImageBuilderConfig)
Local image builder configuration.
Source code in zenml/image_builders/local_image_builder.py
class LocalImageBuilderConfig(BaseImageBuilderConfig):
"""Local image builder configuration."""
LocalImageBuilderFlavor (BaseImageBuilderFlavor)
Local image builder flavor.
Source code in zenml/image_builders/local_image_builder.py
class LocalImageBuilderFlavor(BaseImageBuilderFlavor):
"""Local image builder flavor."""
@property
def name(self) -> str:
"""The flavor name.
Returns:
The flavor name.
"""
return "local"
@property
def docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_docs_url()
@property
def sdk_docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_sdk_docs_url()
@property
def logo_url(self) -> str:
"""A url to represent the flavor in the dashboard.
Returns:
The flavor logo.
"""
return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/image_builder/local.svg"
@property
def config_class(self) -> Type[LocalImageBuilderConfig]:
"""Config class.
Returns:
The config class.
"""
return LocalImageBuilderConfig
@property
def implementation_class(self) -> Type[LocalImageBuilder]:
"""Implementation class.
Returns:
The implementation class.
"""
return LocalImageBuilder
config_class: Type[zenml.image_builders.local_image_builder.LocalImageBuilderConfig]
property
readonly
Config class.
Returns:
Type | Description |
---|---|
Type[zenml.image_builders.local_image_builder.LocalImageBuilderConfig] |
The config class. |
docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
Type | Description |
---|---|
Optional[str] |
A flavor docs url. |
implementation_class: Type[zenml.image_builders.local_image_builder.LocalImageBuilder]
property
readonly
Implementation class.
Returns:
Type | Description |
---|---|
Type[zenml.image_builders.local_image_builder.LocalImageBuilder] |
The implementation class. |
logo_url: str
property
readonly
A url to represent the flavor in the dashboard.
Returns:
Type | Description |
---|---|
str |
The flavor logo. |
name: str
property
readonly
The flavor name.
Returns:
Type | Description |
---|---|
str |
The flavor name. |
sdk_docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
Type | Description |
---|---|
Optional[str] |
A flavor docs url. |