Skip to content

Spark

The Spark integration module to enable distributed processing for steps.

SparkIntegration

Bases: Integration

Definition of Spark integration for ZenML.

Source code in src/zenml/integrations/spark/__init__.py 
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
class SparkIntegration(Integration):
    """Definition of Spark integration for ZenML."""

    NAME = SPARK
    REQUIREMENTS = ["pyspark==3.2.1"]

    @classmethod
    def activate(cls) -> None:
        """Activating the corresponding Spark materializers."""
        from zenml.integrations.spark import materializers  # noqa

    @classmethod
    def flavors(cls) -> List[Type[Flavor]]:
        """Declare the stack component flavors for the Spark integration.

        Returns:
            The flavor wrapper for the step operator flavor
        """
        from zenml.integrations.spark.flavors import (
            KubernetesSparkStepOperatorFlavor,
        )

        return [KubernetesSparkStepOperatorFlavor]

activate() classmethod

Activating the corresponding Spark materializers.

Source code in src/zenml/integrations/spark/__init__.py 
33
34
35
36
@classmethod
def activate(cls) -> None:
    """Activating the corresponding Spark materializers."""
    from zenml.integrations.spark import materializers  # noqa

flavors() classmethod

Declare the stack component flavors for the Spark integration.

Returns:

Type Description
List[Type[Flavor]]

The flavor wrapper for the step operator flavor
Source code in src/zenml/integrations/spark/__init__.py 
38
39
40
41
42
43
44
45
46
47
48
49
@classmethod
def flavors(cls) -> List[Type[Flavor]]:
    """Declare the stack component flavors for the Spark integration.

    Returns:
        The flavor wrapper for the step operator flavor
    """
    from zenml.integrations.spark.flavors import (
        KubernetesSparkStepOperatorFlavor,
    )

    return [KubernetesSparkStepOperatorFlavor]