diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py new file mode 100644 index 0000000..1c886a8 --- /dev/null +++ b/_python_utils_tests/test_lazy_imports.py @@ -0,0 +1,76 @@ +"""Tests for the lazy-import machinery added to keep `import python_utils` +light (PEP 562 `__getattr__`/`__dir__` in the package). +""" + +import os +import subprocess +import sys + +import pytest + +import python_utils +from python_utils import types + + +def test_package_lazy_attribute_access() -> None: + # Submodule access and exported-name access both resolve via __getattr__. + assert python_utils.aio is python_utils.aio + assert callable(python_utils.acount) + missing = 'definitely_not_a_real_attribute' + with pytest.raises(AttributeError): + getattr(python_utils, missing) + + +def test_bare_import_stays_light() -> None: + # Importing the package must not eagerly pull in heavy/optional deps. This + # runs in a clean interpreter because the test session itself has long + # since imported asyncio/typing_extensions. + code = ( + 'import sys, python_utils\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + "assert 'typing_extensions' not in sys.modules\n" + ) + env = {**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)} + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 0, result.stderr + + +def test_first_access_caches_into_module_dict() -> None: + # PEP 562 __getattr__ runs once: the resolved object is cached in the + # module namespace so subsequent lookups skip __getattr__ entirely. + module = python_utils.time + assert python_utils.__dict__['time'] is module + + func = python_utils.format_time + assert python_utils.__dict__['format_time'] is func + + +def test_dir_lists_lazy_submodules() -> None: + # Lazy submodules that are not in __all__ (e.g. ``containers`` and + # ``exceptions``) must still be discoverable via ``dir``; tools such as + # ``import_global`` intersect requested names with ``dir(module)``. + names = set(dir(python_utils)) + assert {'containers', 'exceptions'} <= names + assert set(python_utils.__all__) <= names + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_default_iterable() -> None: + # With no iterable the generator defaults to ``aio.acount`` -- exercising + # the lazy ``aio``/``asyncio`` import and the None-resolution branch. + count = 0 + generator: types.AsyncGenerator[object, None] = ( + python_utils.aio_timeout_generator(timeout=0.05, interval=0.0) + ) + async for _ in generator: + count += 1 + if count >= 2: + break + + assert count == 2 diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 7c4242c..0e14731 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -2,6 +2,12 @@ This module initializes the `python_utils` package by importing various submodules and functions. +Imports are performed lazily (PEP 562): nothing is imported when you ``import +python_utils``; each submodule/function is loaded on first access. This keeps +``import python_utils`` cheap and, in particular, avoids eagerly importing +``asyncio`` (via the async helpers) for consumers that only need the +synchronous utilities. + Submodules: aio converters @@ -49,39 +55,130 @@ LoggerBase """ -from . import ( - aio, - converters, - decorators, - formatters, - generators, - import_, - logger, - terminal, - time, - types, -) -from .aio import acount -from .containers import CastedDict, LazyCastedDict, UniqueList -from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode -from .decorators import listify, set_attributes -from .exceptions import raise_exception, reraise -from .formatters import camel_to_underscore, timesince -from .generators import abatcher, batcher -from .import_ import import_global -from .logger import Logged, LoggerBase -from .terminal import get_terminal_size -from .time import ( - aio_generator_timeout_detector, - aio_generator_timeout_detector_decorator, - aio_timeout_generator, - delta_to_seconds, - delta_to_seconds_or_none, - format_time, - timedelta_to_seconds, - timeout_generator, +import importlib as _importlib +import typing as _typing + +if _typing.TYPE_CHECKING: # pragma: no cover + # Eager imports for type checkers only; the runtime equivalents are loaded + # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are + # treated as re-exports (not unused imports). + from . import ( + aio, + converters, + decorators, + formatters, + generators, + import_, + logger, + terminal, + time, + types, + ) + from .aio import acount + from .containers import CastedDict, LazyCastedDict, UniqueList + from .converters import ( + remap, + scale_1024, + to_float, + to_int, + to_str, + to_unicode, + ) + from .decorators import listify, set_attributes + from .exceptions import raise_exception, reraise + from .formatters import camel_to_underscore, timesince + from .generators import abatcher, batcher + from .import_ import import_global + from .logger import Logged, LoggerBase + from .terminal import get_terminal_size + from .time import ( + aio_generator_timeout_detector, + aio_generator_timeout_detector_decorator, + aio_timeout_generator, + delta_to_seconds, + delta_to_seconds_or_none, + format_time, + timedelta_to_seconds, + timeout_generator, + ) + +#: Submodules that can be accessed as ``python_utils.``. +_SUBMODULES: frozenset[str] = frozenset( + { + 'aio', + 'containers', + 'converters', + 'decorators', + 'exceptions', + 'formatters', + 'generators', + 'import_', + 'logger', + 'terminal', + 'time', + 'types', + } ) +#: Exported name -> submodule it lives in. +_NAME_TO_MODULE: dict[str, str] = { + 'acount': 'aio', + 'CastedDict': 'containers', + 'LazyCastedDict': 'containers', + 'UniqueList': 'containers', + 'remap': 'converters', + 'scale_1024': 'converters', + 'to_float': 'converters', + 'to_int': 'converters', + 'to_str': 'converters', + 'to_unicode': 'converters', + 'listify': 'decorators', + 'set_attributes': 'decorators', + 'raise_exception': 'exceptions', + 'reraise': 'exceptions', + 'camel_to_underscore': 'formatters', + 'timesince': 'formatters', + 'abatcher': 'generators', + 'batcher': 'generators', + 'import_global': 'import_', + 'Logged': 'logger', + 'LoggerBase': 'logger', + 'get_terminal_size': 'terminal', + 'aio_generator_timeout_detector': 'time', + 'aio_generator_timeout_detector_decorator': 'time', + 'aio_timeout_generator': 'time', + 'delta_to_seconds': 'time', + 'delta_to_seconds_or_none': 'time', + 'format_time': 'time', + 'timedelta_to_seconds': 'time', + 'timeout_generator': 'time', +} + + +def __getattr__(name: str) -> _typing.Any: + """Lazily import submodules and their exported names on first access.""" + if name in _SUBMODULES: + module = _importlib.import_module(f'.{name}', __name__) + elif name in _NAME_TO_MODULE: + module = _importlib.import_module( + f'.{_NAME_TO_MODULE[name]}', __name__ + ) + value = getattr(module, name) + globals()[name] = value # cache so __getattr__ runs only once + return value + else: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + globals()[name] = module + return module + + +def __dir__() -> list[str]: + return sorted( + set(globals()) | set(__all__) | _SUBMODULES | set(_NAME_TO_MODULE) + ) + + __all__ = [ 'CastedDict', 'LazyCastedDict', diff --git a/python_utils/converters.py b/python_utils/converters.py index c4240f3..d4b4d41 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -353,7 +353,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] new_max: _TN, ) -> _TN: """ - remap a value from one range into another. + Remap a value from one range into another. >>> remap(500, 0, 1000, 0, 100) 50 diff --git a/python_utils/time.py b/python_utils/time.py index 224d8e1..338ef18 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -19,14 +19,13 @@ """ # pyright: reportUnnecessaryIsInstance=false -import asyncio import datetime import functools import itertools import time import python_utils -from python_utils import aio, exceptions, types +from python_utils import exceptions, types _T = types.TypeVar('_T') _P = types.ParamSpec('_P') @@ -260,9 +259,12 @@ def timeout_generator( async def aio_timeout_generator( timeout: types.delta_type, # noqa: ASYNC109 interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[ - types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]] - ] = aio.acount, + iterable: types.Optional[ + types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ] + ] = None, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ) -> types.AsyncGenerator[_T, None]: @@ -280,6 +282,18 @@ async def aio_timeout_generator( effectively the same as the `timeout_generator` but it uses `async for` instead. """ + # Imported lazily so that importing `python_utils.time` for its + # synchronous helpers (e.g. ``format_time``) does not pull in ``asyncio``. + import asyncio + + from python_utils import aio + + if iterable is None: + iterable = types.cast( + types.Callable[[], types.AsyncIterable[_T]], + aio.acount, + ) + float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( maximum_interval @@ -328,6 +342,9 @@ async def aio_generator_timeout_detector( If `on_timeout` is `None`, the exception is silently ignored and the generator will finish as normal. """ + # Imported lazily so importing `python_utils.time` stays asyncio-free. + import asyncio + if total_timeout is None: total_timeout_end = None else: