Skip to content

Gitlab

zenml.integrations.gitlab

Initialization of the GitLab ZenML integration.

Attributes

GITLAB = 'gitlab' module-attribute

Classes

GitLabIntegration

Bases: Integration

Definition of GitLab integration for ZenML.

Integration

Base class for integration in ZenML.

Functions
activate() -> None classmethod

Abstract method to activate the integration.

Source code in src/zenml/integrations/integration.py 
175
176
177
@classmethod
def activate(cls) -> None:
    """Abstract method to activate the integration."""
check_installation() -> bool classmethod

Method to check whether the required packages are installed.

Returns:

Type Description
bool

True if all required packages are installed, False otherwise.
Source code in src/zenml/integrations/integration.py 
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
@classmethod
def check_installation(cls) -> bool:
    """Method to check whether the required packages are installed.

    Returns:
        True if all required packages are installed, False otherwise.
    """
    for r in cls.get_requirements():
        try:
            # First check if the base package is installed
            dist = pkg_resources.get_distribution(r)

            # Next, check if the dependencies (including extras) are
            # installed
            deps: List[Requirement] = []

            _, extras = parse_requirement(r)
            if extras:
                extra_list = extras[1:-1].split(",")
                for extra in extra_list:
                    try:
                        requirements = dist.requires(extras=[extra])  # type: ignore[arg-type]
                    except pkg_resources.UnknownExtra as e:
                        logger.debug(f"Unknown extra: {str(e)}")
                        return False
                    deps.extend(requirements)
            else:
                deps = dist.requires()

            for ri in deps:
                try:
                    # Remove the "extra == ..." part from the requirement string
                    cleaned_req = re.sub(
                        r"; extra == \"\w+\"", "", str(ri)
                    )
                    pkg_resources.get_distribution(cleaned_req)
                except pkg_resources.DistributionNotFound as e:
                    logger.debug(
                        f"Unable to find required dependency "
                        f"'{e.req}' for requirement '{r}' "
                        f"necessary for integration '{cls.NAME}'."
                    )
                    return False
                except pkg_resources.VersionConflict as e:
                    logger.debug(
                        f"Package version '{e.dist}' does not match "
                        f"version '{e.req}' required by '{r}' "
                        f"necessary for integration '{cls.NAME}'."
                    )
                    return False

        except pkg_resources.DistributionNotFound as e:
            logger.debug(
                f"Unable to find required package '{e.req}' for "
                f"integration {cls.NAME}."
            )
            return False
        except pkg_resources.VersionConflict as e:
            logger.debug(
                f"Package version '{e.dist}' does not match version "
                f"'{e.req}' necessary for integration {cls.NAME}."
            )
            return False

    logger.debug(
        f"Integration {cls.NAME} is installed correctly with "
        f"requirements {cls.get_requirements()}."
    )
    return True
flavors() -> List[Type[Flavor]] classmethod

Abstract method to declare new stack component flavors.

Returns:

Type Description
List[Type[Flavor]]

A list of new stack component flavors.
Source code in src/zenml/integrations/integration.py 
179
180
181
182
183
184
185
186
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
    """Abstract method to declare new stack component flavors.

    Returns:
        A list of new stack component flavors.
    """
    return []
get_requirements(target_os: Optional[str] = None, python_version: Optional[str] = None) -> List[str] classmethod

Method to get the requirements for the integration.

Parameters:

Name Type Description Default
target_os Optional[str]

The target operating system to get the requirements for.

 None
python_version Optional[str]

The Python version to use for the requirements.

 None

Returns:

Type Description
List[str]

A list of requirements.
Source code in src/zenml/integrations/integration.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
@classmethod
def get_requirements(
    cls,
    target_os: Optional[str] = None,
    python_version: Optional[str] = None,
) -> List[str]:
    """Method to get the requirements for the integration.

    Args:
        target_os: The target operating system to get the requirements for.
        python_version: The Python version to use for the requirements.

    Returns:
        A list of requirements.
    """
    return cls.REQUIREMENTS
get_uninstall_requirements(target_os: Optional[str] = None) -> List[str] classmethod

Method to get the uninstall requirements for the integration.

Parameters:

Name Type Description Default
target_os Optional[str]

The target operating system to get the requirements for.

 None

Returns:

Type Description
List[str]

A list of requirements.
Source code in src/zenml/integrations/integration.py 
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
@classmethod
def get_uninstall_requirements(
    cls, target_os: Optional[str] = None
) -> List[str]:
    """Method to get the uninstall requirements for the integration.

    Args:
        target_os: The target operating system to get the requirements for.

    Returns:
        A list of requirements.
    """
    ret = []
    for each in cls.get_requirements(target_os=target_os):
        is_ignored = False
        for ignored in cls.REQUIREMENTS_IGNORED_ON_UNINSTALL:
            if each.startswith(ignored):
                is_ignored = True
                break
        if not is_ignored:
            ret.append(each)
    return ret
plugin_flavors() -> List[Type[BasePluginFlavor]] classmethod

Abstract method to declare new plugin flavors.

Returns:

Type Description
List[Type[BasePluginFlavor]]

A list of new plugin flavors.
Source code in src/zenml/integrations/integration.py 
188
189
190
191
192
193
194
195
@classmethod
def plugin_flavors(cls) -> List[Type["BasePluginFlavor"]]:
    """Abstract method to declare new plugin flavors.

    Returns:
        A list of new plugin flavors.
    """
    return []

Modules

code_repositories

Initialization of the ZenML GitLab code repository.

Classes
GitLabCodeRepository(id: UUID, name: str, config: Dict[str, Any])

Bases: BaseCodeRepository

GitLab code repository.

Source code in src/zenml/code_repositories/base_code_repository.py 
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def __init__(
    self,
    id: UUID,
    name: str,
    config: Dict[str, Any],
) -> None:
    """Initializes a code repository.

    Args:
        id: The ID of the code repository.
        name: The name of the code repository.
        config: The config of the code repository.
    """
    self._id = id
    self._name = name
    self._config = config
    self.login()
Attributes
config: GitLabCodeRepositoryConfig property

Returns the GitLabCodeRepositoryConfig config.

Returns:

Type Description
GitLabCodeRepositoryConfig

The configuration.
gitlab_project: Project property

The GitLab project object from the GitLab API.

Returns:

Type Description
Project

The GitLab project object.
Functions
check_remote_url(url: str) -> bool

Checks whether the remote url matches the code repository.

Parameters:

Name Type Description Default
url str

The remote url.

 required

Returns:

Type Description
bool

Whether the remote url is correct.
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def check_remote_url(self, url: str) -> bool:
    """Checks whether the remote url matches the code repository.

    Args:
        url: The remote url.

    Returns:
        Whether the remote url is correct.
    """
    host = self.config.host or "gitlab.com"
    host = host.rstrip("/")

    parsed_url = urlparse(url)
    if (
        parsed_url.scheme == "https"
        and parsed_url.hostname == host
        and parsed_url.path
        == f"/{self.config.group}/{self.config.project}.git"
    ):
        return True

    ssh_regex = re.compile(
        r"^(?P<scheme_with_delimiter>ssh://)?"
        r"(?P<userinfo>git)"
        f"@{host}:"
        r"(?P<port>\d+)?"
        r"(?(scheme_with_delimiter)/|/?)"
        f"{self.config.group}/{self.config.project}.git$",
    )
    if ssh_regex.fullmatch(url):
        return True

    return False
download_files(commit: str, directory: str, repo_sub_directory: Optional[str]) -> None

Downloads files from a commit to a local directory.

Parameters:

Name Type Description Default
commit str

The commit to download.

 required
directory str

The directory to download to.

 required
repo_sub_directory Optional[str]

The sub directory to download from.

 required
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
def download_files(
    self, commit: str, directory: str, repo_sub_directory: Optional[str]
) -> None:
    """Downloads files from a commit to a local directory.

    Args:
        commit: The commit to download.
        directory: The directory to download to.
        repo_sub_directory: The sub directory to download from.
    """
    contents = self.gitlab_project.repository_tree(
        ref=commit, path=repo_sub_directory or "", iterator=True
    )
    for content in contents:
        logger.debug(f"Processing {content['path']}")
        if content["type"] == "tree":
            path = os.path.join(directory, content["name"])
            os.makedirs(path, exist_ok=True)
            self.download_files(
                commit=commit,
                directory=path,
                repo_sub_directory=content["path"],
            )
        else:
            try:
                path = content["path"]
                file_content = self.gitlab_project.files.get(
                    file_path=path, ref=commit
                ).decode()
                path = os.path.join(directory, content["name"])
                with open(path, "wb") as file:
                    file.write(file_content)
            except Exception as e:
                logger.error("Error processing %s: %s", content["path"], e)
get_local_context(path: str) -> Optional[LocalRepositoryContext]

Gets the local repository context.

Parameters:

Name Type Description Default
path str

The path to the local repository.

 required

Returns:

Type Description
Optional[LocalRepositoryContext]

The local repository context.
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def get_local_context(self, path: str) -> Optional[LocalRepositoryContext]:
    """Gets the local repository context.

    Args:
        path: The path to the local repository.

    Returns:
        The local repository context.
    """
    return LocalGitRepositoryContext.at(
        path=path,
        code_repository=self,
        remote_url_validation_callback=self.check_remote_url,
    )
login() -> None

Logs in to GitLab.

Raises:

Type Description
RuntimeError

If the login fails.
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def login(self) -> None:
    """Logs in to GitLab.

    Raises:
        RuntimeError: If the login fails.
    """
    try:
        self._gitlab_session = Gitlab(
            url=self.config.instance_url, private_token=self.config.token
        )

        if self.config.token:
            self._gitlab_session.auth()
            if user := self._gitlab_session.user:
                logger.debug(f"Logged into GitLab as {user.username}")
    except Exception as e:
        raise RuntimeError(f"An error occurred while logging in: {str(e)}")
validate_config(config: Dict[str, Any]) -> None classmethod

Validate the code repository config.

This method should check that the config/credentials are valid and the configured repository exists.

Parameters:

Name Type Description Default
config Dict[str, Any]

The configuration.

 required
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
68
69
70
71
72
73
74
75
76
77
78
79
80
@classmethod
def validate_config(cls, config: Dict[str, Any]) -> None:
    """Validate the code repository config.

    This method should check that the config/credentials are valid and
    the configured repository exists.

    Args:
        config: The configuration.
    """
    code_repo = cls(id=uuid4(), name="", config=config)
    # Try to access the project to make sure it exists
    _ = code_repo.gitlab_project
Modules
gitlab_code_repository

GitLab code repository.

Classes
GitLabCodeRepository(id: UUID, name: str, config: Dict[str, Any])

Bases: BaseCodeRepository

GitLab code repository.

Source code in src/zenml/code_repositories/base_code_repository.py 
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def __init__(
    self,
    id: UUID,
    name: str,
    config: Dict[str, Any],
) -> None:
    """Initializes a code repository.

    Args:
        id: The ID of the code repository.
        name: The name of the code repository.
        config: The config of the code repository.
    """
    self._id = id
    self._name = name
    self._config = config
    self.login()
Attributes
config: GitLabCodeRepositoryConfig property

Returns the GitLabCodeRepositoryConfig config.

Returns:

Type Description
GitLabCodeRepositoryConfig

The configuration.
gitlab_project: Project property

The GitLab project object from the GitLab API.

Returns:

Type Description
Project

The GitLab project object.
Functions
check_remote_url(url: str) -> bool

Checks whether the remote url matches the code repository.

Parameters:

Name Type Description Default
url str

The remote url.

 required

Returns:

Type Description
bool

Whether the remote url is correct.
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def check_remote_url(self, url: str) -> bool:
    """Checks whether the remote url matches the code repository.

    Args:
        url: The remote url.

    Returns:
        Whether the remote url is correct.
    """
    host = self.config.host or "gitlab.com"
    host = host.rstrip("/")

    parsed_url = urlparse(url)
    if (
        parsed_url.scheme == "https"
        and parsed_url.hostname == host
        and parsed_url.path
        == f"/{self.config.group}/{self.config.project}.git"
    ):
        return True

    ssh_regex = re.compile(
        r"^(?P<scheme_with_delimiter>ssh://)?"
        r"(?P<userinfo>git)"
        f"@{host}:"
        r"(?P<port>\d+)?"
        r"(?(scheme_with_delimiter)/|/?)"
        f"{self.config.group}/{self.config.project}.git$",
    )
    if ssh_regex.fullmatch(url):
        return True

    return False
download_files(commit: str, directory: str, repo_sub_directory: Optional[str]) -> None

Downloads files from a commit to a local directory.

Parameters:

Name Type Description Default
commit str

The commit to download.

 required
directory str

The directory to download to.

 required
repo_sub_directory Optional[str]

The sub directory to download from.

 required
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
def download_files(
    self, commit: str, directory: str, repo_sub_directory: Optional[str]
) -> None:
    """Downloads files from a commit to a local directory.

    Args:
        commit: The commit to download.
        directory: The directory to download to.
        repo_sub_directory: The sub directory to download from.
    """
    contents = self.gitlab_project.repository_tree(
        ref=commit, path=repo_sub_directory or "", iterator=True
    )
    for content in contents:
        logger.debug(f"Processing {content['path']}")
        if content["type"] == "tree":
            path = os.path.join(directory, content["name"])
            os.makedirs(path, exist_ok=True)
            self.download_files(
                commit=commit,
                directory=path,
                repo_sub_directory=content["path"],
            )
        else:
            try:
                path = content["path"]
                file_content = self.gitlab_project.files.get(
                    file_path=path, ref=commit
                ).decode()
                path = os.path.join(directory, content["name"])
                with open(path, "wb") as file:
                    file.write(file_content)
            except Exception as e:
                logger.error("Error processing %s: %s", content["path"], e)
get_local_context(path: str) -> Optional[LocalRepositoryContext]

Gets the local repository context.

Parameters:

Name Type Description Default
path str

The path to the local repository.

 required

Returns:

Type Description
Optional[LocalRepositoryContext]

The local repository context.
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
155
156
157
158
159
160
161
162
163
164
165
166
167
168
def get_local_context(self, path: str) -> Optional[LocalRepositoryContext]:
    """Gets the local repository context.

    Args:
        path: The path to the local repository.

    Returns:
        The local repository context.
    """
    return LocalGitRepositoryContext.at(
        path=path,
        code_repository=self,
        remote_url_validation_callback=self.check_remote_url,
    )
login() -> None

Logs in to GitLab.

Raises:

Type Description
RuntimeError

If the login fails.
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def login(self) -> None:
    """Logs in to GitLab.

    Raises:
        RuntimeError: If the login fails.
    """
    try:
        self._gitlab_session = Gitlab(
            url=self.config.instance_url, private_token=self.config.token
        )

        if self.config.token:
            self._gitlab_session.auth()
            if user := self._gitlab_session.user:
                logger.debug(f"Logged into GitLab as {user.username}")
    except Exception as e:
        raise RuntimeError(f"An error occurred while logging in: {str(e)}")
validate_config(config: Dict[str, Any]) -> None classmethod

Validate the code repository config.

This method should check that the config/credentials are valid and the configured repository exists.

Parameters:

Name Type Description Default
config Dict[str, Any]

The configuration.

 required
Source code in src/zenml/integrations/gitlab/code_repositories/gitlab_code_repository.py 
68
69
70
71
72
73
74
75
76
77
78
79
80
@classmethod
def validate_config(cls, config: Dict[str, Any]) -> None:
    """Validate the code repository config.

    This method should check that the config/credentials are valid and
    the configured repository exists.

    Args:
        config: The configuration.
    """
    code_repo = cls(id=uuid4(), name="", config=config)
    # Try to access the project to make sure it exists
    _ = code_repo.gitlab_project
GitLabCodeRepositoryConfig(warn_about_plain_text_secrets: bool = False, **kwargs: Any)

Bases: BaseCodeRepositoryConfig

Config for GitLab code repositories.

Parameters:

Name Type Description Default
instance_url

The URL of the GitLab instance.

 required
group

The group of the project.

 required
project

The name of the GitLab project.

 required
host

The host of GitLab in case it is self-hosted instance.

 required
token

The token to access the repository.

 required
Source code in src/zenml/config/secret_reference_mixin.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def __init__(
    self, warn_about_plain_text_secrets: bool = False, **kwargs: Any
) -> None:
    """Ensures that secret references are only passed for valid fields.

    This method ensures that secret references are not passed for fields
    that explicitly prevent them or require pydantic validation.

    Args:
        warn_about_plain_text_secrets: If true, then warns about using plain-text secrets.
        **kwargs: Arguments to initialize this object.

    Raises:
        ValueError: If an attribute that requires custom pydantic validation
            or an attribute which explicitly disallows secret references
            is passed as a secret reference.
    """
    for key, value in kwargs.items():
        try:
            field = self.__class__.model_fields[key]
        except KeyError:
            # Value for a private attribute or non-existing field, this
            # will fail during the upcoming pydantic validation
            continue

        if value is None:
            continue

        if not secret_utils.is_secret_reference(value):
            if (
                secret_utils.is_secret_field(field)
                and warn_about_plain_text_secrets
            ):
                logger.warning(
                    "You specified a plain-text value for the sensitive "
                    f"attribute `{key}`. This is currently only a warning, "
                    "but future versions of ZenML will require you to pass "
                    "in sensitive information as secrets. Check out the "
                    "documentation on how to configure values with secrets "
                    "here: https://docs.zenml.io/getting-started/deploying-zenml/secret-management"
                )
            continue

        if secret_utils.is_clear_text_field(field):
            raise ValueError(
                f"Passing the `{key}` attribute as a secret reference is "
                "not allowed."
            )

        requires_validation = has_validators(
            pydantic_class=self.__class__, field_name=key
        )
        if requires_validation:
            raise ValueError(
                f"Passing the attribute `{key}` as a secret reference is "
                "not allowed as additional validation is required for "
                "this attribute."
            )

    super().__init__(**kwargs)
Functions Modules