Image Builders
zenml.image_builders
special
Image builders allow you to build container images.
base_image_builder
Base class for all ZenML image builders.
BaseImageBuilder (StackComponent, ABC)
Base class for all ZenML image builders.
Source code in zenml/image_builders/base_image_builder.py
class BaseImageBuilder(StackComponent, ABC):
"""Base class for all ZenML image builders."""
@property
def config(self) -> BaseImageBuilderConfig:
"""The stack component configuration.
Returns:
The configuration.
"""
return cast(BaseImageBuilderConfig, self._config)
@property
def build_context_class(self) -> Type["BuildContext"]:
"""Build context class to use.
The default build context class creates a build context that works
for the Docker daemon. Override this method if your image builder
requires a custom context.
Returns:
The build context class.
"""
from zenml.image_builders import BuildContext
return BuildContext
@property
@abstractmethod
def is_building_locally(self) -> bool:
"""Whether the image builder builds the images on the client machine.
Returns:
True if the image builder builds locally, False otherwise.
"""
@abstractmethod
def build(
self,
image_name: str,
build_context: "BuildContext",
docker_build_options: Dict[str, Any],
container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
"""Builds a Docker image.
If a container registry is passed, the image will be pushed to that
registry.
Args:
image_name: Name of the image to build.
build_context: The build context to use for the image.
docker_build_options: Docker build options.
container_registry: Optional container registry to push to.
Returns:
The Docker image repo digest or name.
"""
@staticmethod
def _upload_build_context(
build_context: "BuildContext",
parent_path_directory_name: str,
) -> str:
"""Uploads a Docker image build context to a remote location.
Args:
build_context: The build context to upload.
parent_path_directory_name: The name of the directory to upload
the build context to. It will be appended to the artifact
store path to create the parent path where the build context
will be uploaded to.
Returns:
The path to the uploaded build context.
"""
artifact_store = Client().active_stack.artifact_store
parent_path = f"{artifact_store.path}/{parent_path_directory_name}"
fileio.makedirs(parent_path)
hash_ = hashlib.sha1() # nosec
with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as f:
build_context.write_archive(f, gzip=True)
while True:
data = f.read(64 * 1024)
if not data:
break
hash_.update(data)
filename = f"{hash_.hexdigest()}.tar.gz"
filepath = f"{parent_path}/{filename}"
if not fileio.exists(filepath):
logger.info("Uploading build context to `%s`.", filepath)
fileio.copy(f.name, filepath)
else:
logger.info("Build context already exists, not uploading.")
os.unlink(f.name)
return filepath
build_context_class: Type[BuildContext]
property
readonly
Build context class to use.
The default build context class creates a build context that works for the Docker daemon. Override this method if your image builder requires a custom context.
Returns:
Type | Description |
---|---|
Type[BuildContext] |
The build context class. |
config: BaseImageBuilderConfig
property
readonly
The stack component configuration.
Returns:
Type | Description |
---|---|
BaseImageBuilderConfig |
The configuration. |
is_building_locally: bool
property
readonly
Whether the image builder builds the images on the client machine.
Returns:
Type | Description |
---|---|
bool |
True if the image builder builds locally, False otherwise. |
build(self, image_name, build_context, docker_build_options, container_registry=None)
Builds a Docker image.
If a container registry is passed, the image will be pushed to that registry.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
image_name |
str |
Name of the image to build. |
required |
build_context |
BuildContext |
The build context to use for the image. |
required |
docker_build_options |
Dict[str, Any] |
Docker build options. |
required |
container_registry |
Optional[BaseContainerRegistry] |
Optional container registry to push to. |
None |
Returns:
Type | Description |
---|---|
str |
The Docker image repo digest or name. |
Source code in zenml/image_builders/base_image_builder.py
@abstractmethod
def build(
self,
image_name: str,
build_context: "BuildContext",
docker_build_options: Dict[str, Any],
container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
"""Builds a Docker image.
If a container registry is passed, the image will be pushed to that
registry.
Args:
image_name: Name of the image to build.
build_context: The build context to use for the image.
docker_build_options: Docker build options.
container_registry: Optional container registry to push to.
Returns:
The Docker image repo digest or name.
"""
BaseImageBuilderConfig (StackComponentConfig)
Base config for image builders.
Source code in zenml/image_builders/base_image_builder.py
class BaseImageBuilderConfig(StackComponentConfig):
"""Base config for image builders."""
BaseImageBuilderFlavor (Flavor, ABC)
Base class for all ZenML image builder flavors.
Source code in zenml/image_builders/base_image_builder.py
class BaseImageBuilderFlavor(Flavor, ABC):
"""Base class for all ZenML image builder flavors."""
@property
def type(self) -> StackComponentType:
"""Returns the flavor type.
Returns:
The flavor type.
"""
return StackComponentType.IMAGE_BUILDER
@property
def config_class(self) -> Type[BaseImageBuilderConfig]:
"""Config class.
Returns:
The config class.
"""
return BaseImageBuilderConfig
@property
def implementation_class(self) -> Type[BaseImageBuilder]:
"""Implementation class.
Returns:
The implementation class.
"""
return BaseImageBuilder
config_class: Type[zenml.image_builders.base_image_builder.BaseImageBuilderConfig]
property
readonly
Config class.
Returns:
Type | Description |
---|---|
Type[zenml.image_builders.base_image_builder.BaseImageBuilderConfig] |
The config class. |
implementation_class: Type[zenml.image_builders.base_image_builder.BaseImageBuilder]
property
readonly
Implementation class.
Returns:
Type | Description |
---|---|
Type[zenml.image_builders.base_image_builder.BaseImageBuilder] |
The implementation class. |
type: StackComponentType
property
readonly
Returns the flavor type.
Returns:
Type | Description |
---|---|
StackComponentType |
The flavor type. |
build_context
Image build context.
BuildContext
Image build context.
This class is responsible for creating an archive of the files needed to build a container image.
Source code in zenml/image_builders/build_context.py
class BuildContext:
"""Image build context.
This class is responsible for creating an archive of the files needed to
build a container image.
"""
def __init__(
self,
root: Optional[str] = None,
dockerignore_file: Optional[str] = None,
) -> None:
"""Initializes a build context.
Args:
root: Optional root directory for the build context.
dockerignore_file: Optional path to a dockerignore file. If not
given, a file called `.dockerignore` in the build context root
directory will be used instead if it exists.
"""
self._root = root
self._dockerignore_file = dockerignore_file
self._extra_files: Dict[str, str] = {}
@property
def dockerignore_file(self) -> Optional[str]:
"""The dockerignore file to use.
Returns:
Path to the dockerignore file to use.
"""
if self._dockerignore_file:
return self._dockerignore_file
if self._root:
default_dockerignore_path = os.path.join(
self._root, ".dockerignore"
)
if fileio.exists(default_dockerignore_path):
return default_dockerignore_path
return None
def add_file(self, source: str, destination: str) -> None:
"""Adds a file to the build context.
Args:
source: The source of the file to add. This can either be a path
or the file content.
destination: The path inside the build context where the file
should be added.
"""
if fileio.exists(source):
with fileio.open(source) as f:
self._extra_files[destination] = f.read()
else:
self._extra_files[destination] = source
def add_directory(self, source: str, destination: str) -> None:
"""Adds a directory to the build context.
Args:
source: Path to the directory.
destination: The path inside the build context where the directory
should be added.
Raises:
ValueError: If `source` does not point to a directory.
"""
if not fileio.isdir(source):
raise ValueError(
f"Can't add directory {source} to the build context as it "
"does not exist or is not a directory."
)
for dir, _, files in fileio.walk(source):
dir_path = Path(fileio.convert_to_str(dir))
for file_name in files:
file_name = fileio.convert_to_str(file_name)
file_source = dir_path / file_name
file_destination = (
Path(destination)
/ dir_path.relative_to(source)
/ file_name
)
with file_source.open("r") as f:
self._extra_files[file_destination.as_posix()] = f.read()
def write_archive(self, output_file: IO[bytes], gzip: bool = True) -> None:
"""Writes an archive of the build context to the given file.
Args:
output_file: The file to write the archive to.
gzip: Whether to use `gzip` to compress the file.
"""
from docker.utils import build as docker_build_utils
files = self._get_files()
extra_files = self._get_extra_files()
context_archive = docker_build_utils.create_archive(
fileobj=output_file,
root=self._root,
files=sorted(files),
gzip=gzip,
extra_files=extra_files,
)
build_context_size = os.path.getsize(context_archive.name)
if (
self._root
and build_context_size > 50 * 1024 * 1024
and not self.dockerignore_file
):
# The build context exceeds 50MiB and we didn't find any excludes
# in dockerignore files -> remind to specify a .dockerignore file
logger.warning(
"Build context size for docker image: `%s`. If you believe this is "
"unreasonably large, make sure to include a `.dockerignore` file "
"at the root of your build context `%s` or specify a custom file "
"in the Docker configuration when defining your pipeline.",
string_utils.get_human_readable_filesize(build_context_size),
os.path.join(self._root, ".dockerignore"),
)
def _get_files(self) -> Set[str]:
"""Gets all non-ignored files in the build context root directory.
Returns:
All build context files.
"""
if self._root:
exclude_patterns = self._get_exclude_patterns()
from docker.utils import build as docker_build_utils
return cast(
Set[str],
docker_build_utils.exclude_paths(
self._root, patterns=exclude_patterns
),
)
else:
return set()
def _get_extra_files(self) -> List[Tuple[str, str]]:
"""Gets all extra files of the build context.
Returns:
A tuple (path, file_content) for all extra files in the build
context.
"""
return list(self._extra_files.items())
def _get_exclude_patterns(self) -> List[str]:
"""Gets all exclude patterns from the dockerignore file.
Returns:
The exclude patterns from the dockerignore file.
"""
dockerignore = self.dockerignore_file
if dockerignore:
patterns = self._parse_dockerignore(dockerignore)
# Always include the .zen directory
patterns.append(f"!/{REPOSITORY_DIRECTORY_NAME}")
return patterns
else:
logger.info(
"No `.dockerignore` found, including all files inside build "
"context.",
)
return []
@staticmethod
def _parse_dockerignore(dockerignore_path: str) -> List[str]:
"""Parses a dockerignore file and returns a list of patterns to ignore.
Args:
dockerignore_path: Path to the dockerignore file.
Returns:
List of patterns to ignore.
"""
try:
file_content = io_utils.read_file_contents_as_string(
dockerignore_path
)
except FileNotFoundError:
logger.warning(
"Unable to find dockerignore file at path '%s'.",
dockerignore_path,
)
return []
exclude_patterns = []
for line in file_content.split("\n"):
line = line.strip()
if line and not line.startswith("#"):
exclude_patterns.append(line)
return exclude_patterns
dockerignore_file: Optional[str]
property
readonly
The dockerignore file to use.
Returns:
Type | Description |
---|---|
Optional[str] |
Path to the dockerignore file to use. |
__init__(self, root=None, dockerignore_file=None)
special
Initializes a build context.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
root |
Optional[str] |
Optional root directory for the build context. |
None |
dockerignore_file |
Optional[str] |
Optional path to a dockerignore file. If not
given, a file called |
None |
Source code in zenml/image_builders/build_context.py
def __init__(
self,
root: Optional[str] = None,
dockerignore_file: Optional[str] = None,
) -> None:
"""Initializes a build context.
Args:
root: Optional root directory for the build context.
dockerignore_file: Optional path to a dockerignore file. If not
given, a file called `.dockerignore` in the build context root
directory will be used instead if it exists.
"""
self._root = root
self._dockerignore_file = dockerignore_file
self._extra_files: Dict[str, str] = {}
add_directory(self, source, destination)
Adds a directory to the build context.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
source |
str |
Path to the directory. |
required |
destination |
str |
The path inside the build context where the directory should be added. |
required |
Exceptions:
Type | Description |
---|---|
ValueError |
If |
Source code in zenml/image_builders/build_context.py
def add_directory(self, source: str, destination: str) -> None:
"""Adds a directory to the build context.
Args:
source: Path to the directory.
destination: The path inside the build context where the directory
should be added.
Raises:
ValueError: If `source` does not point to a directory.
"""
if not fileio.isdir(source):
raise ValueError(
f"Can't add directory {source} to the build context as it "
"does not exist or is not a directory."
)
for dir, _, files in fileio.walk(source):
dir_path = Path(fileio.convert_to_str(dir))
for file_name in files:
file_name = fileio.convert_to_str(file_name)
file_source = dir_path / file_name
file_destination = (
Path(destination)
/ dir_path.relative_to(source)
/ file_name
)
with file_source.open("r") as f:
self._extra_files[file_destination.as_posix()] = f.read()
add_file(self, source, destination)
Adds a file to the build context.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
source |
str |
The source of the file to add. This can either be a path or the file content. |
required |
destination |
str |
The path inside the build context where the file should be added. |
required |
Source code in zenml/image_builders/build_context.py
def add_file(self, source: str, destination: str) -> None:
"""Adds a file to the build context.
Args:
source: The source of the file to add. This can either be a path
or the file content.
destination: The path inside the build context where the file
should be added.
"""
if fileio.exists(source):
with fileio.open(source) as f:
self._extra_files[destination] = f.read()
else:
self._extra_files[destination] = source
write_archive(self, output_file, gzip=True)
Writes an archive of the build context to the given file.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
output_file |
IO[bytes] |
The file to write the archive to. |
required |
gzip |
bool |
Whether to use |
True |
Source code in zenml/image_builders/build_context.py
def write_archive(self, output_file: IO[bytes], gzip: bool = True) -> None:
"""Writes an archive of the build context to the given file.
Args:
output_file: The file to write the archive to.
gzip: Whether to use `gzip` to compress the file.
"""
from docker.utils import build as docker_build_utils
files = self._get_files()
extra_files = self._get_extra_files()
context_archive = docker_build_utils.create_archive(
fileobj=output_file,
root=self._root,
files=sorted(files),
gzip=gzip,
extra_files=extra_files,
)
build_context_size = os.path.getsize(context_archive.name)
if (
self._root
and build_context_size > 50 * 1024 * 1024
and not self.dockerignore_file
):
# The build context exceeds 50MiB and we didn't find any excludes
# in dockerignore files -> remind to specify a .dockerignore file
logger.warning(
"Build context size for docker image: `%s`. If you believe this is "
"unreasonably large, make sure to include a `.dockerignore` file "
"at the root of your build context `%s` or specify a custom file "
"in the Docker configuration when defining your pipeline.",
string_utils.get_human_readable_filesize(build_context_size),
os.path.join(self._root, ".dockerignore"),
)
local_image_builder
Local Docker image builder implementation.
LocalImageBuilder (BaseImageBuilder)
Local image builder implementation.
Source code in zenml/image_builders/local_image_builder.py
class LocalImageBuilder(BaseImageBuilder):
"""Local image builder implementation."""
@property
def config(self) -> LocalImageBuilderConfig:
"""The stack component configuration.
Returns:
The configuration.
"""
return cast(LocalImageBuilderConfig, self._config)
@property
def is_building_locally(self) -> bool:
"""Whether the image builder builds the images on the client machine.
Returns:
True if the image builder builds locally, False otherwise.
"""
return True
@staticmethod
def _check_prerequisites() -> None:
"""Checks that all prerequisites are installed.
Raises:
RuntimeError: If any of the prerequisites are not installed or
running.
"""
if not shutil.which("docker"):
raise RuntimeError(
"`docker` is required to run the local image builder."
)
if not docker_utils.check_docker():
raise RuntimeError(
"Unable to connect to the Docker daemon. There are two "
"common causes for this:\n"
"1) The Docker daemon isn't running.\n"
"2) The Docker client isn't configured correctly. The client "
"loads its configuration from the following file: "
"$HOME/.docker/config.json. If your configuration file is in a "
"different location, set it using the `DOCKER_CONFIG` "
"environment variable."
)
def build(
self,
image_name: str,
build_context: "BuildContext",
docker_build_options: Optional[Dict[str, Any]] = None,
container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
"""Builds and optionally pushes an image using the local Docker client.
Args:
image_name: Name of the image to build and push.
build_context: The build context to use for the image.
docker_build_options: Docker build options.
container_registry: Optional container registry to push to.
Returns:
The Docker image repo digest.
"""
self._check_prerequisites()
if container_registry:
# Use the container registry's docker client, which may be
# authenticated to access additional registries
docker_client = container_registry.docker_client
else:
docker_client = docker_utils._try_get_docker_client_from_env()
with tempfile.TemporaryFile(mode="w+b") as f:
build_context.write_archive(f)
# We use the client api directly here, so we can stream the logs
output_stream = docker_client.images.client.api.build(
fileobj=f,
custom_context=True,
tag=image_name,
**(docker_build_options or {}),
)
docker_utils._process_stream(output_stream)
if container_registry:
return container_registry.push_image(image_name)
else:
return image_name
config: LocalImageBuilderConfig
property
readonly
The stack component configuration.
Returns:
Type | Description |
---|---|
LocalImageBuilderConfig |
The configuration. |
is_building_locally: bool
property
readonly
Whether the image builder builds the images on the client machine.
Returns:
Type | Description |
---|---|
bool |
True if the image builder builds locally, False otherwise. |
build(self, image_name, build_context, docker_build_options=None, container_registry=None)
Builds and optionally pushes an image using the local Docker client.
Parameters:
Name | Type | Description | Default |
---|---|---|---|
image_name |
str |
Name of the image to build and push. |
required |
build_context |
BuildContext |
The build context to use for the image. |
required |
docker_build_options |
Optional[Dict[str, Any]] |
Docker build options. |
None |
container_registry |
Optional[BaseContainerRegistry] |
Optional container registry to push to. |
None |
Returns:
Type | Description |
---|---|
str |
The Docker image repo digest. |
Source code in zenml/image_builders/local_image_builder.py
def build(
self,
image_name: str,
build_context: "BuildContext",
docker_build_options: Optional[Dict[str, Any]] = None,
container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
"""Builds and optionally pushes an image using the local Docker client.
Args:
image_name: Name of the image to build and push.
build_context: The build context to use for the image.
docker_build_options: Docker build options.
container_registry: Optional container registry to push to.
Returns:
The Docker image repo digest.
"""
self._check_prerequisites()
if container_registry:
# Use the container registry's docker client, which may be
# authenticated to access additional registries
docker_client = container_registry.docker_client
else:
docker_client = docker_utils._try_get_docker_client_from_env()
with tempfile.TemporaryFile(mode="w+b") as f:
build_context.write_archive(f)
# We use the client api directly here, so we can stream the logs
output_stream = docker_client.images.client.api.build(
fileobj=f,
custom_context=True,
tag=image_name,
**(docker_build_options or {}),
)
docker_utils._process_stream(output_stream)
if container_registry:
return container_registry.push_image(image_name)
else:
return image_name
LocalImageBuilderConfig (BaseImageBuilderConfig)
Local image builder configuration.
Source code in zenml/image_builders/local_image_builder.py
class LocalImageBuilderConfig(BaseImageBuilderConfig):
"""Local image builder configuration."""
LocalImageBuilderFlavor (BaseImageBuilderFlavor)
Local image builder flavor.
Source code in zenml/image_builders/local_image_builder.py
class LocalImageBuilderFlavor(BaseImageBuilderFlavor):
"""Local image builder flavor."""
@property
def name(self) -> str:
"""The flavor name.
Returns:
The flavor name.
"""
return "local"
@property
def docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_docs_url()
@property
def sdk_docs_url(self) -> Optional[str]:
"""A url to point at docs explaining this flavor.
Returns:
A flavor docs url.
"""
return self.generate_default_sdk_docs_url()
@property
def logo_url(self) -> str:
"""A url to represent the flavor in the dashboard.
Returns:
The flavor logo.
"""
return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/image_builder/local.svg"
@property
def config_class(self) -> Type[LocalImageBuilderConfig]:
"""Config class.
Returns:
The config class.
"""
return LocalImageBuilderConfig
@property
def implementation_class(self) -> Type[LocalImageBuilder]:
"""Implementation class.
Returns:
The implementation class.
"""
return LocalImageBuilder
config_class: Type[zenml.image_builders.local_image_builder.LocalImageBuilderConfig]
property
readonly
Config class.
Returns:
Type | Description |
---|---|
Type[zenml.image_builders.local_image_builder.LocalImageBuilderConfig] |
The config class. |
docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
Type | Description |
---|---|
Optional[str] |
A flavor docs url. |
implementation_class: Type[zenml.image_builders.local_image_builder.LocalImageBuilder]
property
readonly
Implementation class.
Returns:
Type | Description |
---|---|
Type[zenml.image_builders.local_image_builder.LocalImageBuilder] |
The implementation class. |
logo_url: str
property
readonly
A url to represent the flavor in the dashboard.
Returns:
Type | Description |
---|---|
str |
The flavor logo. |
name: str
property
readonly
The flavor name.
Returns:
Type | Description |
---|---|
str |
The flavor name. |
sdk_docs_url: Optional[str]
property
readonly
A url to point at docs explaining this flavor.
Returns:
Type | Description |
---|---|
Optional[str] |
A flavor docs url. |