Skip to content

Image Builders

Image builders allow you to build container images.

BaseImageBuilder

Bases: StackComponent, ABC

Base class for all ZenML image builders.

Source code in src/zenml/image_builders/base_image_builder.py 
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
class BaseImageBuilder(StackComponent, ABC):
    """Base class for all ZenML image builders."""

    @property
    def config(self) -> BaseImageBuilderConfig:
        """The stack component configuration.

        Returns:
            The configuration.
        """
        return cast(BaseImageBuilderConfig, self._config)

    @property
    def build_context_class(self) -> Type["BuildContext"]:
        """Build context class to use.

        The default build context class creates a build context that works
        for the Docker daemon. Override this method if your image builder
        requires a custom context.

        Returns:
            The build context class.
        """
        from zenml.image_builders import BuildContext

        return BuildContext

    @property
    @abstractmethod
    def is_building_locally(self) -> bool:
        """Whether the image builder builds the images on the client machine.

        Returns:
            True if the image builder builds locally, False otherwise.
        """

    @abstractmethod
    def build(
        self,
        image_name: str,
        build_context: "BuildContext",
        docker_build_options: Dict[str, Any],
        container_registry: Optional["BaseContainerRegistry"] = None,
    ) -> str:
        """Builds a Docker image.

        If a container registry is passed, the image will be pushed to that
        registry.

        Args:
            image_name: Name of the image to build.
            build_context: The build context to use for the image.
            docker_build_options: Docker build options.
            container_registry: Optional container registry to push to.

        Returns:
            The Docker image repo digest or name.
        """

    @staticmethod
    def _upload_build_context(
        build_context: "BuildContext",
        parent_path_directory_name: str,
        archive_type: ArchiveType = ArchiveType.TAR_GZ,
    ) -> str:
        """Uploads a Docker image build context to a remote location.

        Args:
            build_context: The build context to upload.
            parent_path_directory_name: The name of the directory to upload
                the build context to. It will be appended to the artifact
                store path to create the parent path where the build context
                will be uploaded to.
            archive_type: The type of archive to create.

        Returns:
            The path to the uploaded build context.
        """
        artifact_store = Client().active_stack.artifact_store
        parent_path = f"{artifact_store.path}/{parent_path_directory_name}"
        fileio.makedirs(parent_path)

        hash_ = hashlib.sha1()  # nosec
        with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as f:
            build_context.write_archive(f, archive_type)

            while True:
                data = f.read(64 * 1024)
                if not data:
                    break
                hash_.update(data)

            filename = f"{hash_.hexdigest()}.{archive_type.value}"
            filepath = f"{parent_path}/{filename}"
            if not fileio.exists(filepath):
                logger.info("Uploading build context to `%s`.", filepath)
                fileio.copy(f.name, filepath)
            else:
                logger.info("Build context already exists, not uploading.")

        os.unlink(f.name)
        return filepath

build_context_class property

Build context class to use.

The default build context class creates a build context that works for the Docker daemon. Override this method if your image builder requires a custom context.

Returns:

Type Description
Type[BuildContext]

The build context class.

config property

The stack component configuration.

Returns:

Type Description
BaseImageBuilderConfig

The configuration.

is_building_locally abstractmethod property

Whether the image builder builds the images on the client machine.

Returns:

Type Description
bool

True if the image builder builds locally, False otherwise.

build(image_name, build_context, docker_build_options, container_registry=None) abstractmethod

Builds a Docker image.

If a container registry is passed, the image will be pushed to that registry.

Parameters:

Name Type Description Default
image_name str

Name of the image to build.

 required
build_context BuildContext

The build context to use for the image.

 required
docker_build_options Dict[str, Any]

Docker build options.

 required
container_registry Optional[BaseContainerRegistry]

Optional container registry to push to.

 None

Returns:

Type Description
str

The Docker image repo digest or name.
Source code in src/zenml/image_builders/base_image_builder.py 
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
@abstractmethod
def build(
    self,
    image_name: str,
    build_context: "BuildContext",
    docker_build_options: Dict[str, Any],
    container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
    """Builds a Docker image.

    If a container registry is passed, the image will be pushed to that
    registry.

    Args:
        image_name: Name of the image to build.
        build_context: The build context to use for the image.
        docker_build_options: Docker build options.
        container_registry: Optional container registry to push to.

    Returns:
        The Docker image repo digest or name.
    """

BaseImageBuilderConfig

Bases: StackComponentConfig

Base config for image builders.

Source code in src/zenml/image_builders/base_image_builder.py 
37
38
class BaseImageBuilderConfig(StackComponentConfig):
    """Base config for image builders."""

BaseImageBuilderFlavor

Bases: Flavor, ABC

Base class for all ZenML image builder flavors.

Source code in src/zenml/image_builders/base_image_builder.py 
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
class BaseImageBuilderFlavor(Flavor, ABC):
    """Base class for all ZenML image builder flavors."""

    @property
    def type(self) -> StackComponentType:
        """Returns the flavor type.

        Returns:
            The flavor type.
        """
        return StackComponentType.IMAGE_BUILDER

    @property
    def config_class(self) -> Type[BaseImageBuilderConfig]:
        """Config class.

        Returns:
            The config class.
        """
        return BaseImageBuilderConfig

    @property
    def implementation_class(self) -> Type[BaseImageBuilder]:
        """Implementation class.

        Returns:
            The implementation class.
        """
        return BaseImageBuilder

config_class property

Config class.

Returns:

Type Description
Type[BaseImageBuilderConfig]

The config class.

implementation_class property

Implementation class.

Returns:

Type Description
Type[BaseImageBuilder]

The implementation class.

type property

Returns the flavor type.

Returns:

Type Description
StackComponentType

The flavor type.

BuildContext

Bases: Archivable

Image build context.

This class is responsible for creating an archive of the files needed to build a container image.

Source code in src/zenml/image_builders/build_context.py 
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
class BuildContext(Archivable):
    """Image build context.

    This class is responsible for creating an archive of the files needed to
    build a container image.
    """

    def __init__(
        self,
        root: Optional[str] = None,
        dockerignore_file: Optional[str] = None,
    ) -> None:
        """Initializes a build context.

        Args:
            root: Optional root directory for the build context.
            dockerignore_file: Optional path to a dockerignore file. If not
                given, a file called `.dockerignore` in the build context root
                directory will be used instead if it exists.
        """
        super().__init__()
        self._root = root
        self._dockerignore_file = dockerignore_file

    @property
    def dockerignore_file(self) -> Optional[str]:
        """The dockerignore file to use.

        Returns:
            Path to the dockerignore file to use.
        """
        if self._dockerignore_file:
            return self._dockerignore_file

        if self._root:
            default_dockerignore_path = os.path.join(
                self._root, ".dockerignore"
            )
            if fileio.exists(default_dockerignore_path):
                return default_dockerignore_path

        return None

    def write_archive(
        self,
        output_file: IO[bytes],
        archive_type: ArchiveType = ArchiveType.TAR_GZ,
    ) -> None:
        """Writes an archive of the build context to the given file.

        Args:
            output_file: The file to write the archive to.
            archive_type: The type of archive to create.
        """
        super().write_archive(output_file, archive_type)

        build_context_size = os.path.getsize(output_file.name)
        if (
            self._root
            and build_context_size > 50 * 1024 * 1024
            and not self.dockerignore_file
        ):
            # The build context exceeds 50MiB and we didn't find any excludes
            # in dockerignore files -> remind to specify a .dockerignore file
            logger.warning(
                "Build context size for docker image: `%s`. If you believe this is "
                "unreasonably large, make sure to include a `.dockerignore` file "
                "at the root of your build context `%s` or specify a custom file "
                "in the Docker configuration when defining your pipeline.",
                string_utils.get_human_readable_filesize(build_context_size),
                os.path.join(self._root, ".dockerignore"),
            )

    def get_files(self) -> Dict[str, str]:
        """Gets all regular files that should be included in the archive.

        Returns:
            A dict {path_in_archive: path_on_filesystem} for all regular files
            in the archive.
        """
        if self._root:
            from docker.utils import build as docker_build_utils

            exclude_patterns = self._get_exclude_patterns()

            archive_paths = cast(
                Set[str],
                docker_build_utils.exclude_paths(
                    self._root, patterns=exclude_patterns
                ),
            )
            return {
                archive_path: os.path.join(self._root, archive_path)
                for archive_path in archive_paths
            }
        else:
            return {}

    def _get_exclude_patterns(self) -> List[str]:
        """Gets all exclude patterns from the dockerignore file.

        Returns:
            The exclude patterns from the dockerignore file.
        """
        dockerignore = self.dockerignore_file
        if dockerignore:
            patterns = self._parse_dockerignore(dockerignore)
            # Always include the .zen directory
            patterns.append(f"!/{REPOSITORY_DIRECTORY_NAME}")
            return patterns
        else:
            logger.info(
                "No `.dockerignore` found, including all files inside build "
                "context.",
            )
            return []

    @staticmethod
    def _parse_dockerignore(dockerignore_path: str) -> List[str]:
        """Parses a dockerignore file and returns a list of patterns to ignore.

        Args:
            dockerignore_path: Path to the dockerignore file.

        Returns:
            List of patterns to ignore.
        """
        try:
            file_content = io_utils.read_file_contents_as_string(
                dockerignore_path
            )
        except FileNotFoundError:
            logger.warning(
                "Unable to find dockerignore file at path '%s'.",
                dockerignore_path,
            )
            return []

        exclude_patterns = []
        for line in file_content.split("\n"):
            line = line.strip()
            if line and not line.startswith("#"):
                exclude_patterns.append(line)

        return exclude_patterns

dockerignore_file property

The dockerignore file to use.

Returns:

Type Description
Optional[str]

Path to the dockerignore file to use.

__init__(root=None, dockerignore_file=None)

Initializes a build context.

Parameters:

Name Type Description Default
root Optional[str]

Optional root directory for the build context.

 None
dockerignore_file Optional[str]

Optional path to a dockerignore file. If not given, a file called .dockerignore in the build context root directory will be used instead if it exists.

 None
Source code in src/zenml/image_builders/build_context.py 
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def __init__(
    self,
    root: Optional[str] = None,
    dockerignore_file: Optional[str] = None,
) -> None:
    """Initializes a build context.

    Args:
        root: Optional root directory for the build context.
        dockerignore_file: Optional path to a dockerignore file. If not
            given, a file called `.dockerignore` in the build context root
            directory will be used instead if it exists.
    """
    super().__init__()
    self._root = root
    self._dockerignore_file = dockerignore_file

get_files()

Gets all regular files that should be included in the archive.

Returns:

Type Description
Dict[str, str]

A dict {path_in_archive: path_on_filesystem} for all regular files
Dict[str, str]

in the archive.
Source code in src/zenml/image_builders/build_context.py 
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
def get_files(self) -> Dict[str, str]:
    """Gets all regular files that should be included in the archive.

    Returns:
        A dict {path_in_archive: path_on_filesystem} for all regular files
        in the archive.
    """
    if self._root:
        from docker.utils import build as docker_build_utils

        exclude_patterns = self._get_exclude_patterns()

        archive_paths = cast(
            Set[str],
            docker_build_utils.exclude_paths(
                self._root, patterns=exclude_patterns
            ),
        )
        return {
            archive_path: os.path.join(self._root, archive_path)
            for archive_path in archive_paths
        }
    else:
        return {}

write_archive(output_file, archive_type=ArchiveType.TAR_GZ)

Writes an archive of the build context to the given file.

Parameters:

Name Type Description Default
output_file IO[bytes]

The file to write the archive to.

 required
archive_type ArchiveType

The type of archive to create.

 TAR_GZ
Source code in src/zenml/image_builders/build_context.py 
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
def write_archive(
    self,
    output_file: IO[bytes],
    archive_type: ArchiveType = ArchiveType.TAR_GZ,
) -> None:
    """Writes an archive of the build context to the given file.

    Args:
        output_file: The file to write the archive to.
        archive_type: The type of archive to create.
    """
    super().write_archive(output_file, archive_type)

    build_context_size = os.path.getsize(output_file.name)
    if (
        self._root
        and build_context_size > 50 * 1024 * 1024
        and not self.dockerignore_file
    ):
        # The build context exceeds 50MiB and we didn't find any excludes
        # in dockerignore files -> remind to specify a .dockerignore file
        logger.warning(
            "Build context size for docker image: `%s`. If you believe this is "
            "unreasonably large, make sure to include a `.dockerignore` file "
            "at the root of your build context `%s` or specify a custom file "
            "in the Docker configuration when defining your pipeline.",
            string_utils.get_human_readable_filesize(build_context_size),
            os.path.join(self._root, ".dockerignore"),
        )

LocalImageBuilder

Bases: BaseImageBuilder

Local image builder implementation.

Source code in src/zenml/image_builders/local_image_builder.py 
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
class LocalImageBuilder(BaseImageBuilder):
    """Local image builder implementation."""

    @property
    def config(self) -> LocalImageBuilderConfig:
        """The stack component configuration.

        Returns:
            The configuration.
        """
        return cast(LocalImageBuilderConfig, self._config)

    @property
    def is_building_locally(self) -> bool:
        """Whether the image builder builds the images on the client machine.

        Returns:
            True if the image builder builds locally, False otherwise.
        """
        return True

    @staticmethod
    def _check_prerequisites() -> None:
        """Checks that all prerequisites are installed.

        Raises:
            RuntimeError: If any of the prerequisites are not installed or
                running.
        """
        if not shutil.which("docker"):
            raise RuntimeError(
                "`docker` is required to run the local image builder."
            )

        if not docker_utils.check_docker():
            # For 3., this is not supported by the python docker library
            # https://github.com/docker/docker-py/issues/3146
            raise RuntimeError(
                "Unable to connect to the Docker daemon. There are three "
                "common causes for this:\n"
                "1) The Docker daemon isn't running.\n"
                "2) The Docker client isn't configured correctly. The client "
                "loads its configuration from the following file: "
                "$HOME/.docker/config.json. If your configuration file is in a "
                "different location, set the `DOCKER_CONFIG` environment "
                "variable to the directory that contains your `config.json` "
                "file.\n"
                "3) If your Docker CLI is working fine but you ran into this "
                "issue, you might be using a non-default Docker context which "
                "is not supported by the Docker python library. To verify "
                "this, run `docker context ls` and check which context has a "
                "`*` next to it. If this is not the `default` context, copy "
                "the `DOCKER ENDPOINT` value of that context and set the "
                "`DOCKER_HOST` environment variable to that value."
            )

    def build(
        self,
        image_name: str,
        build_context: "BuildContext",
        docker_build_options: Optional[Dict[str, Any]] = None,
        container_registry: Optional["BaseContainerRegistry"] = None,
    ) -> str:
        """Builds and optionally pushes an image using the local Docker client.

        Args:
            image_name: Name of the image to build and push.
            build_context: The build context to use for the image.
            docker_build_options: Docker build options.
            container_registry: Optional container registry to push to.

        Returns:
            The Docker image repo digest.
        """
        self._check_prerequisites()

        if container_registry:
            # Use the container registry's docker client, which may be
            # authenticated to access additional registries
            docker_client = container_registry.docker_client
        else:
            docker_client = docker_utils._try_get_docker_client_from_env()

        with tempfile.TemporaryFile(mode="w+b") as f:
            build_context.write_archive(f)

            # We use the client api directly here, so we can stream the logs
            output_stream = docker_client.images.client.api.build(
                fileobj=f,
                custom_context=True,
                tag=image_name,
                **(docker_build_options or {}),
            )
        docker_utils._process_stream(output_stream)

        if container_registry:
            return container_registry.push_image(image_name)
        else:
            return image_name

config property

The stack component configuration.

Returns:

Type Description
LocalImageBuilderConfig

The configuration.

is_building_locally property

Whether the image builder builds the images on the client machine.

Returns:

Type Description
bool

True if the image builder builds locally, False otherwise.

build(image_name, build_context, docker_build_options=None, container_registry=None)

Builds and optionally pushes an image using the local Docker client.

Parameters:

Name Type Description Default
image_name str

Name of the image to build and push.

 required
build_context BuildContext

The build context to use for the image.

 required
docker_build_options Optional[Dict[str, Any]]

Docker build options.

 None
container_registry Optional[BaseContainerRegistry]

Optional container registry to push to.

 None

Returns:

Type Description
str

The Docker image repo digest.
Source code in src/zenml/image_builders/local_image_builder.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
def build(
    self,
    image_name: str,
    build_context: "BuildContext",
    docker_build_options: Optional[Dict[str, Any]] = None,
    container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
    """Builds and optionally pushes an image using the local Docker client.

    Args:
        image_name: Name of the image to build and push.
        build_context: The build context to use for the image.
        docker_build_options: Docker build options.
        container_registry: Optional container registry to push to.

    Returns:
        The Docker image repo digest.
    """
    self._check_prerequisites()

    if container_registry:
        # Use the container registry's docker client, which may be
        # authenticated to access additional registries
        docker_client = container_registry.docker_client
    else:
        docker_client = docker_utils._try_get_docker_client_from_env()

    with tempfile.TemporaryFile(mode="w+b") as f:
        build_context.write_archive(f)

        # We use the client api directly here, so we can stream the logs
        output_stream = docker_client.images.client.api.build(
            fileobj=f,
            custom_context=True,
            tag=image_name,
            **(docker_build_options or {}),
        )
    docker_utils._process_stream(output_stream)

    if container_registry:
        return container_registry.push_image(image_name)
    else:
        return image_name

LocalImageBuilderConfig

Bases: BaseImageBuilderConfig

Local image builder configuration.

Source code in src/zenml/image_builders/local_image_builder.py 
32
33
class LocalImageBuilderConfig(BaseImageBuilderConfig):
    """Local image builder configuration."""

LocalImageBuilderFlavor

Bases: BaseImageBuilderFlavor

Local image builder flavor.

Source code in src/zenml/image_builders/local_image_builder.py 
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
class LocalImageBuilderFlavor(BaseImageBuilderFlavor):
    """Local image builder flavor."""

    @property
    def name(self) -> str:
        """The flavor name.

        Returns:
            The flavor name.
        """
        return "local"

    @property
    def docs_url(self) -> Optional[str]:
        """A url to point at docs explaining this flavor.

        Returns:
            A flavor docs url.
        """
        return self.generate_default_docs_url()

    @property
    def sdk_docs_url(self) -> Optional[str]:
        """A url to point at docs explaining this flavor.

        Returns:
            A flavor docs url.
        """
        return self.generate_default_sdk_docs_url()

    @property
    def logo_url(self) -> str:
        """A url to represent the flavor in the dashboard.

        Returns:
            The flavor logo.
        """
        return "https://public-flavor-logos.s3.eu-central-1.amazonaws.com/image_builder/local.svg"

    @property
    def config_class(self) -> Type[LocalImageBuilderConfig]:
        """Config class.

        Returns:
            The config class.
        """
        return LocalImageBuilderConfig

    @property
    def implementation_class(self) -> Type[LocalImageBuilder]:
        """Implementation class.

        Returns:
            The implementation class.
        """
        return LocalImageBuilder

config_class property

Config class.

Returns:

Type Description
Type[LocalImageBuilderConfig]

The config class.

docs_url property

A url to point at docs explaining this flavor.

Returns:

Type Description
Optional[str]

A flavor docs url.

implementation_class property

Implementation class.

Returns:

Type Description
Type[LocalImageBuilder]

The implementation class.

logo_url property

A url to represent the flavor in the dashboard.

Returns:

Type Description
str

The flavor logo.

name property

The flavor name.

Returns:

Type Description
str

The flavor name.

sdk_docs_url property

A url to point at docs explaining this flavor.

Returns:

Type Description
Optional[str]

A flavor docs url.