Skip to content

Package Management

fusion_bench.utils.packages

compare_versions(v1, v2)

Compare two version strings. Returns -1 if v1 < v2, 0 if v1 == v2, 1 if v1 > v2

Source code in fusion_bench/utils/packages.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def compare_versions(v1: str, v2: str) -> int:
    """Compare two version strings.
    Returns -1 if v1 < v2, 0 if v1 == v2, 1 if v1 > v2"""

    v1 = version.parse(v1)
    v2 = version.parse(v2)
    if v1 < v2:
        return -1
    elif v1 > v2:
        return 1
    else:
        return 0

import_object(abs_obj_name)

Imports a class from a module given the absolute class name.

Parameters:

  • abs_obj_name (str) –

    The absolute name of the object to import.

Returns:

  • Any –

    The imported class.

Source code in fusion_bench/utils/packages.py 
76
77
78
79
80
81
82
83
84
85
86
87
88
def import_object(abs_obj_name: str) -> Any:
    """
    Imports a class from a module given the absolute class name.

    Args:
        abs_obj_name (str): The absolute name of the object to import.

    Returns:
        The imported class.
    """
    module_name, obj_name = abs_obj_name.rsplit(".", 1)
    module = importlib.import_module(module_name)
    return getattr(module, obj_name)

fusion_bench.utils.lazy_imports

Lazy-Imports module.

This is code taken from the HuggingFace team <https://huggingface.co/>. Many thanks to HuggingFace for your consent <https://github.com/huggingface/transformers/issues/12861#issuecomment-886712209> to publish it as a standalone package.

LazyImporter

Bases: ModuleType

Lazy importer for modules and their components.

This class allows for lazy importing of modules, meaning modules are only imported when they are actually accessed. This can help reduce startup time and memory usage for large packages with many optional dependencies.

Attributes:

  • _modules (Set[str]) –

    Set of module names available for import.

  • _class_to_module (Dict[str, str]) –

    Mapping from class/function names to their module names.

  • _objects (Dict[str, Any]) –

    Dictionary of extra objects to include in the module.

  • _name –

    Name of the module.

  • _import_structure –

    Dictionary mapping module names to lists of their exports.

Source code in fusion_bench/utils/lazy_imports.py 
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
class LazyImporter(ModuleType):
    """Lazy importer for modules and their components.

    This class allows for lazy importing of modules, meaning modules are only
    imported when they are actually accessed. This can help reduce startup
    time and memory usage for large packages with many optional dependencies.

    Attributes:
        _modules: Set of module names available for import.
        _class_to_module: Mapping from class/function names to their module names.
        _objects: Dictionary of extra objects to include in the module.
        _name: Name of the module.
        _import_structure: Dictionary mapping module names to lists of their exports.
    """

    # Very heavily inspired by optuna.integration._IntegrationModule
    # https://github.com/optuna/optuna/blob/master/optuna/integration/__init__.py
    def __init__(
        self,
        name: str,
        module_file: str,
        import_structure: Dict[str, List[str]],
        extra_objects: Optional[Dict[str, Any]] = None,
    ) -> None:
        """Initialize the LazyImporter.

        Args:
            name: The name of the module.
            module_file: Path to the module file.
            import_structure: Dictionary mapping module names to lists of their exports.
            extra_objects: Optional dictionary of extra objects to include.
        """
        super().__init__(name)
        self._modules: Set[str] = set(import_structure.keys())
        self._class_to_module: Dict[str, str] = {}
        for key, values in import_structure.items():
            for value in values:
                self._class_to_module[value] = key
        # Needed for autocompletion in an IDE
        self.__all__: List[str] = list(import_structure.keys()) + sum(
            import_structure.values(), []
        )
        self.__file__ = module_file
        self.__path__ = [os.path.dirname(module_file)]
        self._objects: Dict[str, Any] = {} if extra_objects is None else extra_objects
        self._name = name
        self._import_structure = import_structure

    # Needed for autocompletion in an IDE
    def __dir__(self) -> List[str]:
        """Return list of available attributes for autocompletion.

        Returns:
            List of all available attribute names.
        """
        return super().__dir__() + self.__all__

    def __getattr__(self, name: str) -> Any:
        """Get attribute lazily, importing the module if necessary.

        Args:
            name: The name of the attribute to retrieve.

        Returns:
            The requested attribute.

        Raises:
            AttributeError: If the attribute is not found in any module.
        """
        if name in self._objects:
            return self._objects[name]
        if name in self._modules:
            value = self._get_module(name)
        elif name in self._class_to_module:
            module = self._get_module(self._class_to_module[name])
            value = getattr(module, name)
        else:
            raise AttributeError(f"module {self.__name__} has no attribute {name}")

        setattr(self, name, value)
        return value

    def _get_module(self, module_name: str) -> ModuleType:
        """Import and return the specified module.

        Args:
            module_name: Name of the module to import.

        Returns:
            The imported module.
        """
        return importlib.import_module("." + module_name, self.__name__)

    def __reduce__(self) -> tuple:
        """Support for pickling the LazyImporter.

        Returns:
            Tuple containing the class and arguments needed to reconstruct the object.
        """
        return (self.__class__, (self._name, self.__file__, self._import_structure))
__dir__()

Return list of available attributes for autocompletion.

Returns:

  • List[str] –

    List of all available attribute names.

Source code in fusion_bench/utils/lazy_imports.py 
79
80
81
82
83
84
85
def __dir__(self) -> List[str]:
    """Return list of available attributes for autocompletion.

    Returns:
        List of all available attribute names.
    """
    return super().__dir__() + self.__all__
__getattr__(name)

Get attribute lazily, importing the module if necessary.

Parameters:

  • name (str) –

    The name of the attribute to retrieve.

Returns:

  • Any –

    The requested attribute.

Raises:

  • AttributeError –

    If the attribute is not found in any module.

Source code in fusion_bench/utils/lazy_imports.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def __getattr__(self, name: str) -> Any:
    """Get attribute lazily, importing the module if necessary.

    Args:
        name: The name of the attribute to retrieve.

    Returns:
        The requested attribute.

    Raises:
        AttributeError: If the attribute is not found in any module.
    """
    if name in self._objects:
        return self._objects[name]
    if name in self._modules:
        value = self._get_module(name)
    elif name in self._class_to_module:
        module = self._get_module(self._class_to_module[name])
        value = getattr(module, name)
    else:
        raise AttributeError(f"module {self.__name__} has no attribute {name}")

    setattr(self, name, value)
    return value
__init__(name, module_file, import_structure, extra_objects=None)

Initialize the LazyImporter.

Parameters:

  • name (str) –

    The name of the module.

  • module_file (str) –

    Path to the module file.

  • import_structure (Dict[str, List[str]]) –

    Dictionary mapping module names to lists of their exports.

  • extra_objects (Optional[Dict[str, Any]], default: None ) –

    Optional dictionary of extra objects to include.

Source code in fusion_bench/utils/lazy_imports.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
def __init__(
    self,
    name: str,
    module_file: str,
    import_structure: Dict[str, List[str]],
    extra_objects: Optional[Dict[str, Any]] = None,
) -> None:
    """Initialize the LazyImporter.

    Args:
        name: The name of the module.
        module_file: Path to the module file.
        import_structure: Dictionary mapping module names to lists of their exports.
        extra_objects: Optional dictionary of extra objects to include.
    """
    super().__init__(name)
    self._modules: Set[str] = set(import_structure.keys())
    self._class_to_module: Dict[str, str] = {}
    for key, values in import_structure.items():
        for value in values:
            self._class_to_module[value] = key
    # Needed for autocompletion in an IDE
    self.__all__: List[str] = list(import_structure.keys()) + sum(
        import_structure.values(), []
    )
    self.__file__ = module_file
    self.__path__ = [os.path.dirname(module_file)]
    self._objects: Dict[str, Any] = {} if extra_objects is None else extra_objects
    self._name = name
    self._import_structure = import_structure
__reduce__()

Support for pickling the LazyImporter.

Returns:

  • tuple –

    Tuple containing the class and arguments needed to reconstruct the object.

Source code in fusion_bench/utils/lazy_imports.py 
123
124
125
126
127
128
129
def __reduce__(self) -> tuple:
    """Support for pickling the LazyImporter.

    Returns:
        Tuple containing the class and arguments needed to reconstruct the object.
    """
    return (self.__class__, (self._name, self.__file__, self._import_structure))

LazyPyModule

Bases: ModuleType

Module wrapper for lazy import.

Adapted from Optuna: https://github.com/optuna/optuna/blob/1f92d496b0c4656645384e31539e4ee74992ff55/optuna/init.py

This class wraps specified module and lazily import it when they are actually accessed. This can help reduce startup time and memory usage by deferring module imports until they are needed.

Parameters:

  • name (str) –

    Name of module to apply lazy import.

Attributes:

  • _name (str) –

    The name of the module to be lazily imported.

Source code in fusion_bench/utils/lazy_imports.py 
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
class LazyPyModule(ModuleType):
    """Module wrapper for lazy import.

    Adapted from Optuna: https://github.com/optuna/optuna/blob/1f92d496b0c4656645384e31539e4ee74992ff55/optuna/__init__.py

    This class wraps specified module and lazily import it when they are actually accessed.
    This can help reduce startup time and memory usage by deferring module imports
    until they are needed.

    Args:
        name: Name of module to apply lazy import.

    Attributes:
        _name: The name of the module to be lazily imported.
    """

    def __init__(self, name: str) -> None:
        """Initialize the LazyPyModule.

        Args:
            name: The name of the module to be lazily imported.
        """
        super().__init__(name)
        self._name: str = name

    def _load(self) -> ModuleType:
        """Load the actual module and update this object's dictionary.

        Returns:
            The loaded module.
        """
        module = importlib.import_module(self._name)
        self.__dict__.update(module.__dict__)
        return module

    def __getattr__(self, item: str) -> Any:
        """Get attribute from the lazily loaded module.

        Args:
            item: The name of the attribute to retrieve.

        Returns:
            The requested attribute from the loaded module.
        """
        return getattr(self._load(), item)
__getattr__(item)

Get attribute from the lazily loaded module.

Parameters:

  • item (str) –

    The name of the attribute to retrieve.

Returns:

  • Any –

    The requested attribute from the loaded module.

Source code in fusion_bench/utils/lazy_imports.py 
167
168
169
170
171
172
173
174
175
176
def __getattr__(self, item: str) -> Any:
    """Get attribute from the lazily loaded module.

    Args:
        item: The name of the attribute to retrieve.

    Returns:
        The requested attribute from the loaded module.
    """
    return getattr(self._load(), item)
__init__(name)

Initialize the LazyPyModule.

Parameters:

  • name (str) –

    The name of the module to be lazily imported.

Source code in fusion_bench/utils/lazy_imports.py 
148
149
150
151
152
153
154
155
def __init__(self, name: str) -> None:
    """Initialize the LazyPyModule.

    Args:
        name: The name of the module to be lazily imported.
    """
    super().__init__(name)
    self._name: str = name