Image Builders

zenml.image_builders special

Image builders allow you to build container images.

base_image_builder

Base class for all ZenML image builders.

BaseImageBuilder (StackComponent, ABC)

Base class for all ZenML image builders.

Source code in zenml/image_builders/base_image_builder.py

class BaseImageBuilder(StackComponent, ABC):
    """Base class for all ZenML image builders."""

    @property
    def config(self) -> BaseImageBuilderConfig:
        """The stack component configuration.

        Returns:
            The configuration.
        """
        return cast(BaseImageBuilderConfig, self._config)

    @property
    def build_context_class(self) -> Type["BuildContext"]:
        """Build context class to use.

        The default build context class creates a build context that works
        for the Docker daemon. Override this method if your image builder
        requires a custom context.

        Returns:
            The build context class.
        """
        from zenml.image_builders import BuildContext

        return BuildContext

    @property
    @abstractmethod
    def is_building_locally(self) -> bool:
        """Whether the image builder builds the images on the client machine.

        Returns:
            True if the image builder builds locally, False otherwise.
        """

    @abstractmethod
    def build(
        self,
        image_name: str,
        build_context: "BuildContext",
        docker_build_options: Dict[str, Any],
        container_registry: Optional["BaseContainerRegistry"] = None,
    ) -> str:
        """Builds a Docker image.

        If a container registry is passed, the image will be pushed to that
        registry.

        Args:
            image_name: Name of the image to build.
            build_context: The build context to use for the image.
            docker_build_options: Docker build options.
            container_registry: Optional container registry to push to.

        Returns:
            The Docker image repo digest or name.
        """

    @staticmethod
    def _upload_build_context(
        build_context: "BuildContext",
        parent_path_directory_name: str,
    ) -> str:
        """Uploads a Docker image build context to a remote location.

        Args:
            build_context: The build context to upload.
            parent_path_directory_name: The name of the directory to upload
                the build context to. It will be appended to the artifact
                store path to create the parent path where the build context
                will be uploaded to.

        Returns:
            The path to the uploaded build context.
        """
        artifact_store = Client().active_stack.artifact_store
        parent_path = f"{artifact_store.path}/{parent_path_directory_name}"
        fileio.makedirs(parent_path)

        hash_ = hashlib.sha1()
        with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as f:
            build_context.write_archive(f, gzip=True)

            while True:
                data = f.read(64 * 1024)
                if not data:
                    break
                hash_.update(data)

            filename = f"{hash_.hexdigest()}.tar.gz"
            filepath = f"{parent_path}/{filename}"
            if not fileio.exists(filepath):
                logger.info("Uploading build context to `%s`.", filepath)
                fileio.copy(f.name, filepath)
            else:
                logger.info("Build context already exists, not uploading.")

        os.unlink(f.name)
        return filepath

build_context_class: Type[BuildContext] property readonly

Build context class to use.

The default build context class creates a build context that works for the Docker daemon. Override this method if your image builder requires a custom context.

Returns:

Type	Description
Type[BuildContext]	The build context class.

config: BaseImageBuilderConfig property readonly

The stack component configuration.

Returns:

Type	Description
BaseImageBuilderConfig	The configuration.

is_building_locally: bool property readonly

Whether the image builder builds the images on the client machine.

Returns:

Type	Description
bool	True if the image builder builds locally, False otherwise.

build(self, image_name, build_context, docker_build_options, container_registry=None)

Builds a Docker image.

If a container registry is passed, the image will be pushed to that registry.

Parameters:

Name	Type	Description	Default
image_name	str	Name of the image to build.	required
build_context	BuildContext	The build context to use for the image.	required
docker_build_options	Dict[str, Any]	Docker build options.	required
container_registry	Optional[BaseContainerRegistry]	Optional container registry to push to.	None

Returns:

Type	Description
str	The Docker image repo digest or name.

Source code in zenml/image_builders/base_image_builder.py

@abstractmethod
def build(
    self,
    image_name: str,
    build_context: "BuildContext",
    docker_build_options: Dict[str, Any],
    container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
    """Builds a Docker image.

    If a container registry is passed, the image will be pushed to that
    registry.

    Args:
        image_name: Name of the image to build.
        build_context: The build context to use for the image.
        docker_build_options: Docker build options.
        container_registry: Optional container registry to push to.

    Returns:
        The Docker image repo digest or name.
    """

BaseImageBuilderConfig (StackComponentConfig) pydantic-model

Base config for image builders.

Source code in zenml/image_builders/base_image_builder.py

class BaseImageBuilderConfig(StackComponentConfig):
    """Base config for image builders."""

BaseImageBuilderFlavor (Flavor, ABC)

Base class for all ZenML image builder flavors.

Source code in zenml/image_builders/base_image_builder.py

class BaseImageBuilderFlavor(Flavor, ABC):
    """Base class for all ZenML image builder flavors."""

    @property
    def type(self) -> StackComponentType:
        """Returns the flavor type.

        Returns:
            The flavor type.
        """
        return StackComponentType.IMAGE_BUILDER

    @property
    def config_class(self) -> Type[BaseImageBuilderConfig]:
        """Config class.

        Returns:
            The config class.
        """
        return BaseImageBuilderConfig

    @property
    def implementation_class(self) -> Type[BaseImageBuilder]:
        """Implementation class.

        Returns:
            The implementation class.
        """
        return BaseImageBuilder

config_class: Type[zenml.image_builders.base_image_builder.BaseImageBuilderConfig] property readonly

Config class.

Returns:

Type	Description
Type[zenml.image_builders.base_image_builder.BaseImageBuilderConfig]	The config class.

implementation_class: Type[zenml.image_builders.base_image_builder.BaseImageBuilder] property readonly

Implementation class.

Returns:

Type	Description
Type[zenml.image_builders.base_image_builder.BaseImageBuilder]	The implementation class.

type: StackComponentType property readonly

Returns the flavor type.

Returns:

Type	Description
StackComponentType	The flavor type.

build_context

Image build context.

BuildContext

Image build context.

This class is responsible for creating an archive of the files needed to build a container image.

Source code in zenml/image_builders/build_context.py

class BuildContext:
    """Image build context.

    This class is responsible for creating an archive of the files needed to
    build a container image.
    """

    def __init__(
        self,
        root: Optional[str] = None,
        dockerignore_file: Optional[str] = None,
    ) -> None:
        """Initializes a build context.

        Args:
            root: Optional root directory for the build context.
            dockerignore_file: Optional path to a dockerignore file. If not
                given, a file called `.dockerignore` in the build context root
                directory will be used instead if it exists.
        """
        self._root = root
        self._dockerignore_file = dockerignore_file
        self._extra_files: Dict[str, str] = {}

    @property
    def dockerignore_file(self) -> Optional[str]:
        """The dockerignore file to use.

        Returns:
            Path to the dockerignore file to use.
        """
        if self._dockerignore_file:
            return self._dockerignore_file

        if self._root:
            default_dockerignore_path = os.path.join(
                self._root, ".dockerignore"
            )
            if fileio.exists(default_dockerignore_path):
                return default_dockerignore_path

        return None

    def add_file(self, source: str, destination: str) -> None:
        """Adds a file to the build context.

        Args:
            source: The source of the file to add. This can either be a path
                or the file content.
            destination: The path inside the build context where the file
                should be added.
        """
        if fileio.exists(source):
            with fileio.open(source) as f:
                self._extra_files[destination] = f.read()
        else:
            self._extra_files[destination] = source

    def add_directory(self, source: str, destination: str) -> None:
        """Adds a directory to the build context.

        Args:
            source: Path to the directory.
            destination: The path inside the build context where the directory
                should be added.

        Raises:
            ValueError: If `source` does not point to a directory.
        """
        if not fileio.isdir(source):
            raise ValueError(
                f"Can't add directory {source} to the build context as it "
                "does not exist or is not a directory."
            )

        for dir, _, files in fileio.walk(source):
            dir_path = Path(fileio.convert_to_str(dir))
            for file_name in files:
                file_name = fileio.convert_to_str(file_name)
                file_source = dir_path / file_name
                file_destination = (
                    Path(destination)
                    / dir_path.relative_to(source)
                    / file_name
                )

                with file_source.open("r") as f:
                    self._extra_files[file_destination.as_posix()] = f.read()

    def write_archive(self, output_file: IO[bytes], gzip: bool = True) -> None:
        """Writes an archive of the build context to the given file.

        Args:
            output_file: The file to write the archive to.
            gzip: Whether to use `gzip` to compress the file.
        """
        from docker.utils import build as docker_build_utils

        files = self._get_files()
        extra_files = self._get_extra_files()

        context_archive = docker_build_utils.create_archive(
            fileobj=output_file,
            root=self._root,
            files=sorted(files),
            gzip=gzip,
            extra_files=extra_files,
        )

        build_context_size = os.path.getsize(context_archive.name)
        if (
            self._root
            and build_context_size > 50 * 1024 * 1024
            and not self.dockerignore_file
        ):
            # The build context exceeds 50MiB and we didn't find any excludes
            # in dockerignore files -> remind to specify a .dockerignore file
            logger.warning(
                "Build context size for docker image: `%s`. If you believe this is "
                "unreasonably large, make sure to include a `.dockerignore` file "
                "at the root of your build context `%s` or specify a custom file "
                "in the Docker configuration when defining your pipeline.",
                string_utils.get_human_readable_filesize(build_context_size),
                os.path.join(self._root, ".dockerignore"),
            )

    def _get_files(self) -> Set[str]:
        """Gets all non-ignored files in the build context root directory.

        Returns:
            All build context files.
        """
        if self._root:
            exclude_patterns = self._get_exclude_patterns()
            from docker.utils import build as docker_build_utils

            return cast(
                Set[str],
                docker_build_utils.exclude_paths(
                    self._root, patterns=exclude_patterns
                ),
            )
        else:
            return set()

    def _get_extra_files(self) -> List[Tuple[str, str]]:
        """Gets all extra files of the build context.

        Returns:
            A tuple (path, file_content) for all extra files in the build
            context.
        """
        return list(self._extra_files.items())

    def _get_exclude_patterns(self) -> List[str]:
        """Gets all exclude patterns from the dockerignore file.

        Returns:
            The exclude patterns from the dockerignore file.
        """
        dockerignore = self.dockerignore_file
        if dockerignore:
            patterns = self._parse_dockerignore(dockerignore)
            # Always include the .zen directory
            patterns.append(f"!/{REPOSITORY_DIRECTORY_NAME}")
            return patterns
        else:
            logger.info(
                "No `.dockerignore` found, including all files inside build "
                "context.",
            )
            return []

    @staticmethod
    def _parse_dockerignore(dockerignore_path: str) -> List[str]:
        """Parses a dockerignore file and returns a list of patterns to ignore.

        Args:
            dockerignore_path: Path to the dockerignore file.

        Returns:
            List of patterns to ignore.
        """
        try:
            file_content = io_utils.read_file_contents_as_string(
                dockerignore_path
            )
        except FileNotFoundError:
            logger.warning(
                "Unable to find dockerignore file at path '%s'.",
                dockerignore_path,
            )
            return []

        exclude_patterns = []
        for line in file_content.split("\n"):
            line = line.strip()
            if line and not line.startswith("#"):
                exclude_patterns.append(line)

        return exclude_patterns

dockerignore_file: Optional[str] property readonly

The dockerignore file to use.

Returns:

Type	Description
Optional[str]	Path to the dockerignore file to use.

__init__(self, root=None, dockerignore_file=None) special

Initializes a build context.

Parameters:

Name	Type	Description	Default
root	Optional[str]	Optional root directory for the build context.	None
dockerignore_file	Optional[str]	Optional path to a dockerignore file. If not given, a file called .dockerignore in the build context root directory will be used instead if it exists.	None

Source code in zenml/image_builders/build_context.py

def __init__(
    self,
    root: Optional[str] = None,
    dockerignore_file: Optional[str] = None,
) -> None:
    """Initializes a build context.

    Args:
        root: Optional root directory for the build context.
        dockerignore_file: Optional path to a dockerignore file. If not
            given, a file called `.dockerignore` in the build context root
            directory will be used instead if it exists.
    """
    self._root = root
    self._dockerignore_file = dockerignore_file
    self._extra_files: Dict[str, str] = {}

add_directory(self, source, destination)

Adds a directory to the build context.

Parameters:

Name	Type	Description	Default
source	str	Path to the directory.	required
destination	str	The path inside the build context where the directory should be added.	required

Exceptions:

Type	Description
ValueError	If source does not point to a directory.

Source code in zenml/image_builders/build_context.py

def add_directory(self, source: str, destination: str) -> None:
    """Adds a directory to the build context.

    Args:
        source: Path to the directory.
        destination: The path inside the build context where the directory
            should be added.

    Raises:
        ValueError: If `source` does not point to a directory.
    """
    if not fileio.isdir(source):
        raise ValueError(
            f"Can't add directory {source} to the build context as it "
            "does not exist or is not a directory."
        )

    for dir, _, files in fileio.walk(source):
        dir_path = Path(fileio.convert_to_str(dir))
        for file_name in files:
            file_name = fileio.convert_to_str(file_name)
            file_source = dir_path / file_name
            file_destination = (
                Path(destination)
                / dir_path.relative_to(source)
                / file_name
            )

            with file_source.open("r") as f:
                self._extra_files[file_destination.as_posix()] = f.read()

add_file(self, source, destination)

Adds a file to the build context.

Parameters:

Name	Type	Description	Default
source	str	The source of the file to add. This can either be a path or the file content.	required
destination	str	The path inside the build context where the file should be added.	required

Source code in zenml/image_builders/build_context.py

def add_file(self, source: str, destination: str) -> None:
    """Adds a file to the build context.

    Args:
        source: The source of the file to add. This can either be a path
            or the file content.
        destination: The path inside the build context where the file
            should be added.
    """
    if fileio.exists(source):
        with fileio.open(source) as f:
            self._extra_files[destination] = f.read()
    else:
        self._extra_files[destination] = source

write_archive(self, output_file, gzip=True)

Writes an archive of the build context to the given file.

Parameters:

Name	Type	Description	Default
output_file	IO[bytes]	The file to write the archive to.	required
gzip	bool	Whether to use gzip to compress the file.	True

Source code in zenml/image_builders/build_context.py

def write_archive(self, output_file: IO[bytes], gzip: bool = True) -> None:
    """Writes an archive of the build context to the given file.

    Args:
        output_file: The file to write the archive to.
        gzip: Whether to use `gzip` to compress the file.
    """
    from docker.utils import build as docker_build_utils

    files = self._get_files()
    extra_files = self._get_extra_files()

    context_archive = docker_build_utils.create_archive(
        fileobj=output_file,
        root=self._root,
        files=sorted(files),
        gzip=gzip,
        extra_files=extra_files,
    )

    build_context_size = os.path.getsize(context_archive.name)
    if (
        self._root
        and build_context_size > 50 * 1024 * 1024
        and not self.dockerignore_file
    ):
        # The build context exceeds 50MiB and we didn't find any excludes
        # in dockerignore files -> remind to specify a .dockerignore file
        logger.warning(
            "Build context size for docker image: `%s`. If you believe this is "
            "unreasonably large, make sure to include a `.dockerignore` file "
            "at the root of your build context `%s` or specify a custom file "
            "in the Docker configuration when defining your pipeline.",
            string_utils.get_human_readable_filesize(build_context_size),
            os.path.join(self._root, ".dockerignore"),
        )

local_image_builder

Local Docker image builder implementation.

LocalImageBuilder (BaseImageBuilder)

Local image builder implementation.

Source code in zenml/image_builders/local_image_builder.py

class LocalImageBuilder(BaseImageBuilder):
    """Local image builder implementation."""

    @property
    def config(self) -> LocalImageBuilderConfig:
        """The stack component configuration.

        Returns:
            The configuration.
        """
        return cast(LocalImageBuilderConfig, self._config)

    @property
    def is_building_locally(self) -> bool:
        """Whether the image builder builds the images on the client machine.

        Returns:
            True if the image builder builds locally, False otherwise.
        """
        return True

    @staticmethod
    def _check_prerequisites() -> None:
        """Checks that all prerequisites are installed.

        Raises:
            RuntimeError: If any of the prerequisites are not installed or
                running.
        """
        if not shutil.which("docker"):
            raise RuntimeError(
                "`docker` is required to run the local image builder."
            )

        if not docker_utils.check_docker():
            raise RuntimeError(
                "Unable to connect to the Docker daemon. There are two "
                "common causes for this:\n"
                "1) The Docker daemon isn't running.\n"
                "2) The Docker client isn't configured correctly. The client "
                "loads its configuration from the following file: "
                "$HOME/.docker/config.json. If your configuration file is in a "
                "different location, set it using the `DOCKER_CONFIG` "
                "environment variable."
            )

    def build(
        self,
        image_name: str,
        build_context: "BuildContext",
        docker_build_options: Optional[Dict[str, Any]] = None,
        container_registry: Optional["BaseContainerRegistry"] = None,
    ) -> str:
        """Builds and optionally pushes an image using the local Docker client.

        Args:
            image_name: Name of the image to build and push.
            build_context: The build context to use for the image.
            docker_build_options: Docker build options.
            container_registry: Optional container registry to push to.

        Returns:
            The Docker image repo digest.
        """
        self._check_prerequisites()

        docker_client = DockerClient.from_env()

        with tempfile.TemporaryFile(mode="w+b") as f:
            build_context.write_archive(f)

            # We use the client api directly here, so we can stream the logs
            output_stream = docker_client.images.client.api.build(
                fileobj=f,
                custom_context=True,
                tag=image_name,
                **(docker_build_options or {}),
            )
        docker_utils._process_stream(output_stream)

        if container_registry:
            return container_registry.push_image(image_name)
        else:
            return image_name

config: LocalImageBuilderConfig property readonly

The stack component configuration.

Returns:

Type	Description
LocalImageBuilderConfig	The configuration.

is_building_locally: bool property readonly

Whether the image builder builds the images on the client machine.

Returns:

Type	Description
bool	True if the image builder builds locally, False otherwise.

build(self, image_name, build_context, docker_build_options=None, container_registry=None)

Builds and optionally pushes an image using the local Docker client.

Parameters:

Name	Type	Description	Default
image_name	str	Name of the image to build and push.	required
build_context	BuildContext	The build context to use for the image.	required
docker_build_options	Optional[Dict[str, Any]]	Docker build options.	None
container_registry	Optional[BaseContainerRegistry]	Optional container registry to push to.	None

Returns:

Type	Description
str	The Docker image repo digest.

Source code in zenml/image_builders/local_image_builder.py

def build(
    self,
    image_name: str,
    build_context: "BuildContext",
    docker_build_options: Optional[Dict[str, Any]] = None,
    container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
    """Builds and optionally pushes an image using the local Docker client.

    Args:
        image_name: Name of the image to build and push.
        build_context: The build context to use for the image.
        docker_build_options: Docker build options.
        container_registry: Optional container registry to push to.

    Returns:
        The Docker image repo digest.
    """
    self._check_prerequisites()

    docker_client = DockerClient.from_env()

    with tempfile.TemporaryFile(mode="w+b") as f:
        build_context.write_archive(f)

        # We use the client api directly here, so we can stream the logs
        output_stream = docker_client.images.client.api.build(
            fileobj=f,
            custom_context=True,
            tag=image_name,
            **(docker_build_options or {}),
        )
    docker_utils._process_stream(output_stream)

    if container_registry:
        return container_registry.push_image(image_name)
    else:
        return image_name

LocalImageBuilderConfig (BaseImageBuilderConfig) pydantic-model

Local image builder configuration.

Source code in zenml/image_builders/local_image_builder.py

class LocalImageBuilderConfig(BaseImageBuilderConfig):
    """Local image builder configuration."""

LocalImageBuilderFlavor (BaseImageBuilderFlavor)

Local image builder flavor.

Source code in zenml/image_builders/local_image_builder.py

class LocalImageBuilderFlavor(BaseImageBuilderFlavor):
    """Local image builder flavor."""

    @property
    def name(self) -> str:
        """The flavor name.

        Returns:
            The flavor name.
        """
        return "local"

    @property
    def docs_url(self) -> Optional[str]:
        """A url to point at docs explaining this flavor.

        Returns:
            A flavor docs url.
        """
        return self.generate_default_docs_url()

    @property
    def sdk_docs_url(self) -> Optional[str]:
        """A url to point at docs explaining this flavor.

        Returns:
            A flavor docs url.
        """
        return self.generate_default_sdk_docs_url()

    @property
    def logo_url(self) -> str:
        """A url to represent the flavor in the dashboard.

        Returns:
            The flavor logo.
        """
        return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/image_builder/local.svg"

    @property
    def config_class(self) -> Type[LocalImageBuilderConfig]:
        """Config class.

        Returns:
            The config class.
        """
        return LocalImageBuilderConfig

    @property
    def implementation_class(self) -> Type[LocalImageBuilder]:
        """Implementation class.

        Returns:
            The implementation class.
        """
        return LocalImageBuilder

config_class: Type[zenml.image_builders.local_image_builder.LocalImageBuilderConfig] property readonly

Config class.

Returns:

Type	Description
Type[zenml.image_builders.local_image_builder.LocalImageBuilderConfig]	The config class.

docs_url: Optional[str] property readonly

A url to point at docs explaining this flavor.

Returns:

Type	Description
Optional[str]	A flavor docs url.

implementation_class: Type[zenml.image_builders.local_image_builder.LocalImageBuilder] property readonly

Implementation class.

Returns:

Type	Description
Type[zenml.image_builders.local_image_builder.LocalImageBuilder]	The implementation class.

logo_url: str property readonly

A url to represent the flavor in the dashboard.

Returns:

Type	Description
str	The flavor logo.

name: str property readonly

The flavor name.

Returns:

Type	Description
str	The flavor name.

sdk_docs_url: Optional[str] property readonly

A url to point at docs explaining this flavor.

Returns:

Type	Description
Optional[str]	A flavor docs url.