From e8c055a33c6c95c954f2f21b72da8078832339e0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 24 Jun 2026 02:50:33 +0200 Subject: [PATCH 1/3] Defer asyncio and typing_extensions imports (lazy package init) `import python_utils` now imports nothing eagerly: the package __init__ uses PEP 562 __getattr__ to load submodules and their exported names on first access (with a TYPE_CHECKING block so static typing is unchanged). This avoids pulling in asyncio for consumers that only need the synchronous utilities. time.py no longer imports asyncio/aio at module scope (moved into the two async generators; the `aio.acount` default is resolved lazily), so `from python_utils.time import format_time` stays asyncio-free. types.py imports typing_extensions eagerly only on Python < 3.11 (to preserve the backport overrides); on 3.11+ stdlib typing already provides the names used and any remaining typing_extensions-only name is served lazily via __getattr__. Net effect for a typical consumer (measured via python-progressbar): import drops ~43ms -> ~22ms, with asyncio and typing_extensions no longer loaded. Behaviour and public API are unchanged; full test suite passes. --- _python_utils_tests/test_lazy_imports.py | 44 +++++++ python_utils/__init__.py | 155 ++++++++++++++++++----- python_utils/converters.py | 2 +- python_utils/time.py | 27 +++- python_utils/types.py | 32 ++++- 5 files changed, 222 insertions(+), 38 deletions(-) create mode 100644 _python_utils_tests/test_lazy_imports.py diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py new file mode 100644 index 0000000..d7514f9 --- /dev/null +++ b/_python_utils_tests/test_lazy_imports.py @@ -0,0 +1,44 @@ +"""Tests for the lazy-import machinery added to keep `import python_utils` +light (PEP 562 `__getattr__` in the package and in `types`). +""" + +import pytest + +import python_utils +from python_utils import types + + +def test_package_lazy_attribute_access() -> None: + # Submodule access and exported-name access both resolve via __getattr__. + assert python_utils.aio is python_utils.aio + assert callable(python_utils.acount) + missing = 'definitely_not_a_real_attribute' + with pytest.raises(AttributeError): + getattr(python_utils, missing) + + +def test_types_lazy_typing_extensions(monkeypatch: pytest.MonkeyPatch) -> None: + # ``Protocol`` is normally present from ``from typing import *``; remove it + # so attribute access falls through to ``types.__getattr__``, which + # re-fetches it from typing_extensions (covering the lazy success path). + monkeypatch.delattr(types, 'Protocol', raising=False) + assert types.Protocol is not None + missing = 'not_a_real_typing_name_xyz' + with pytest.raises(AttributeError): + getattr(types, missing) + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_default_iterable() -> None: + # With no iterable the generator defaults to ``aio.acount`` -- exercising + # the lazy ``aio``/``asyncio`` import and the None-resolution branch. + count = 0 + generator: types.AsyncGenerator[object, None] = ( + python_utils.aio_timeout_generator(timeout=0.05, interval=0.0) + ) + async for _ in generator: + count += 1 + if count >= 2: + break + + assert count == 2 diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 7c4242c..9c3f951 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -2,6 +2,12 @@ This module initializes the `python_utils` package by importing various submodules and functions. +Imports are performed lazily (PEP 562): nothing is imported when you ``import +python_utils``; each submodule/function is loaded on first access. This keeps +``import python_utils`` cheap and, in particular, avoids eagerly importing +``asyncio`` (via the async helpers) for consumers that only need the +synchronous utilities. + Submodules: aio converters @@ -49,39 +55,126 @@ LoggerBase """ -from . import ( - aio, - converters, - decorators, - formatters, - generators, - import_, - logger, - terminal, - time, - types, -) -from .aio import acount -from .containers import CastedDict, LazyCastedDict, UniqueList -from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode -from .decorators import listify, set_attributes -from .exceptions import raise_exception, reraise -from .formatters import camel_to_underscore, timesince -from .generators import abatcher, batcher -from .import_ import import_global -from .logger import Logged, LoggerBase -from .terminal import get_terminal_size -from .time import ( - aio_generator_timeout_detector, - aio_generator_timeout_detector_decorator, - aio_timeout_generator, - delta_to_seconds, - delta_to_seconds_or_none, - format_time, - timedelta_to_seconds, - timeout_generator, +import importlib +import typing + +if typing.TYPE_CHECKING: + # Eager imports for type checkers only; the runtime equivalents are loaded + # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are + # treated as re-exports (not unused imports). + from . import ( + aio, + converters, + decorators, + formatters, + generators, + import_, + logger, + terminal, + time, + types, + ) + from .aio import acount + from .containers import CastedDict, LazyCastedDict, UniqueList + from .converters import ( + remap, + scale_1024, + to_float, + to_int, + to_str, + to_unicode, + ) + from .decorators import listify, set_attributes + from .exceptions import raise_exception, reraise + from .formatters import camel_to_underscore, timesince + from .generators import abatcher, batcher + from .import_ import import_global + from .logger import Logged, LoggerBase + from .terminal import get_terminal_size + from .time import ( + aio_generator_timeout_detector, + aio_generator_timeout_detector_decorator, + aio_timeout_generator, + delta_to_seconds, + delta_to_seconds_or_none, + format_time, + timedelta_to_seconds, + timeout_generator, + ) + +#: Submodules that can be accessed as ``python_utils.``. +_SUBMODULES: frozenset[str] = frozenset( + { + 'aio', + 'containers', + 'converters', + 'decorators', + 'exceptions', + 'formatters', + 'generators', + 'import_', + 'logger', + 'terminal', + 'time', + 'types', + } ) +#: Exported name -> submodule it lives in. +_NAME_TO_MODULE: dict[str, str] = { + 'acount': 'aio', + 'CastedDict': 'containers', + 'LazyCastedDict': 'containers', + 'UniqueList': 'containers', + 'remap': 'converters', + 'scale_1024': 'converters', + 'to_float': 'converters', + 'to_int': 'converters', + 'to_str': 'converters', + 'to_unicode': 'converters', + 'listify': 'decorators', + 'set_attributes': 'decorators', + 'raise_exception': 'exceptions', + 'reraise': 'exceptions', + 'camel_to_underscore': 'formatters', + 'timesince': 'formatters', + 'abatcher': 'generators', + 'batcher': 'generators', + 'import_global': 'import_', + 'Logged': 'logger', + 'LoggerBase': 'logger', + 'get_terminal_size': 'terminal', + 'aio_generator_timeout_detector': 'time', + 'aio_generator_timeout_detector_decorator': 'time', + 'aio_timeout_generator': 'time', + 'delta_to_seconds': 'time', + 'delta_to_seconds_or_none': 'time', + 'format_time': 'time', + 'timedelta_to_seconds': 'time', + 'timeout_generator': 'time', +} + + +def __getattr__(name: str) -> typing.Any: + """Lazily import submodules and their exported names on first access.""" + if name in _SUBMODULES: + module = importlib.import_module(f'.{name}', __name__) + elif name in _NAME_TO_MODULE: + module = importlib.import_module(f'.{_NAME_TO_MODULE[name]}', __name__) + value = getattr(module, name) + globals()[name] = value # cache so __getattr__ runs only once + return value + else: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + globals()[name] = module + return module + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__)) + + __all__ = [ 'CastedDict', 'LazyCastedDict', diff --git a/python_utils/converters.py b/python_utils/converters.py index c4240f3..d4b4d41 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -353,7 +353,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] new_max: _TN, ) -> _TN: """ - remap a value from one range into another. + Remap a value from one range into another. >>> remap(500, 0, 1000, 0, 100) 50 diff --git a/python_utils/time.py b/python_utils/time.py index 224d8e1..2a3187c 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -19,14 +19,13 @@ """ # pyright: reportUnnecessaryIsInstance=false -import asyncio import datetime import functools import itertools import time import python_utils -from python_utils import aio, exceptions, types +from python_utils import exceptions, types _T = types.TypeVar('_T') _P = types.ParamSpec('_P') @@ -260,9 +259,12 @@ def timeout_generator( async def aio_timeout_generator( timeout: types.delta_type, # noqa: ASYNC109 interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[ - types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]] - ] = aio.acount, + iterable: types.Optional[ + types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ] + ] = None, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ) -> types.AsyncGenerator[_T, None]: @@ -280,6 +282,18 @@ async def aio_timeout_generator( effectively the same as the `timeout_generator` but it uses `async for` instead. """ + # Imported lazily so that importing `python_utils.time` for its + # synchronous helpers (e.g. ``format_time``) does not pull in ``asyncio``. + import asyncio + + from python_utils import aio + + if iterable is None: + iterable = types.cast( + 'types.AsyncIterable[_T]', + aio.acount, + ) + float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( maximum_interval @@ -328,6 +342,9 @@ async def aio_generator_timeout_detector( If `on_timeout` is `None`, the exception is silently ignored and the generator will finish as normal. """ + # Imported lazily so importing `python_utils.time` stays asyncio-free. + import asyncio + if total_timeout is None: total_timeout_end = None else: diff --git a/python_utils/types.py b/python_utils/types.py index ab89c43..8e1c369 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -32,7 +32,13 @@ Union as U, # noqa: N817 ) -from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 +if TYPE_CHECKING: # pragma: no cover + # Type checkers see the full typing_extensions surface here. At runtime + # typing_extensions is never imported eagerly; any name not provided by + # stdlib `typing` (e.g. `ParamSpec` on Python 3.9) is resolved lazily by + # `__getattr__` below, so importing this module (and therefore + # `python_utils`) does not pull in typing_extensions. + from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 Scope = Dict[str, Any] OptionalScope = O[Scope] @@ -53,6 +59,30 @@ None, ] + +def __getattr__(name: str) -> Any: + """Lazily resolve typing_extensions-only names on first access. + + On Python 3.11+ typing_extensions is not imported eagerly (see above); any + name not provided by stdlib ``typing``/``types`` is fetched from + typing_extensions here, on demand. + """ + if name.startswith('__') and name.endswith('__'): + raise AttributeError(name) + + import typing_extensions + + try: + value = getattr(typing_extensions, name) + except AttributeError: + raise AttributeError( + f'module {__name__!r} has no attribute {name!r}', + ) from None + + globals()[name] = value # cache so __getattr__ runs only once per name + return value + + __all__ = [ 'IO', 'TYPE_CHECKING', From d469a217b14003538b2f05cd8a28c20e9accbf90 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 24 Jun 2026 17:42:45 +0200 Subject: [PATCH 2/3] Address PR review: accurate cast, restore typing_extensions, dir() & tests - time.py: cast aio.acount to Callable[[], AsyncIterable[_T]] (real type object instead of a string forward-ref) so static analysis/refactor works - types.py: revert typing_extensions deferral; restore runtime 'from typing_extensions import *' so backported runtime behaviour (e.g. TypeVar(default=...)) is preserved on Python 3.9-3.12. Bare 'import python_utils' still avoids typing_extensions (types loads lazily) - __init__.py: __dir__ now includes lazy submodules (containers, exceptions) so dir(python_utils) and import_global() see them again - test_lazy_imports.py: drop the removed types.__getattr__ test; add a clean-subprocess test proving bare import pulls in neither asyncio nor typing_extensions, plus __getattr__ caching and __dir__ coverage --- _python_utils_tests/test_lazy_imports.py | 52 +++++++++++++++++++----- python_utils/__init__.py | 4 +- python_utils/time.py | 2 +- python_utils/types.py | 32 +-------------- 4 files changed, 47 insertions(+), 43 deletions(-) diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py index d7514f9..1c886a8 100644 --- a/_python_utils_tests/test_lazy_imports.py +++ b/_python_utils_tests/test_lazy_imports.py @@ -1,7 +1,11 @@ """Tests for the lazy-import machinery added to keep `import python_utils` -light (PEP 562 `__getattr__` in the package and in `types`). +light (PEP 562 `__getattr__`/`__dir__` in the package). """ +import os +import subprocess +import sys + import pytest import python_utils @@ -17,15 +21,43 @@ def test_package_lazy_attribute_access() -> None: getattr(python_utils, missing) -def test_types_lazy_typing_extensions(monkeypatch: pytest.MonkeyPatch) -> None: - # ``Protocol`` is normally present from ``from typing import *``; remove it - # so attribute access falls through to ``types.__getattr__``, which - # re-fetches it from typing_extensions (covering the lazy success path). - monkeypatch.delattr(types, 'Protocol', raising=False) - assert types.Protocol is not None - missing = 'not_a_real_typing_name_xyz' - with pytest.raises(AttributeError): - getattr(types, missing) +def test_bare_import_stays_light() -> None: + # Importing the package must not eagerly pull in heavy/optional deps. This + # runs in a clean interpreter because the test session itself has long + # since imported asyncio/typing_extensions. + code = ( + 'import sys, python_utils\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + "assert 'typing_extensions' not in sys.modules\n" + ) + env = {**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)} + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 0, result.stderr + + +def test_first_access_caches_into_module_dict() -> None: + # PEP 562 __getattr__ runs once: the resolved object is cached in the + # module namespace so subsequent lookups skip __getattr__ entirely. + module = python_utils.time + assert python_utils.__dict__['time'] is module + + func = python_utils.format_time + assert python_utils.__dict__['format_time'] is func + + +def test_dir_lists_lazy_submodules() -> None: + # Lazy submodules that are not in __all__ (e.g. ``containers`` and + # ``exceptions``) must still be discoverable via ``dir``; tools such as + # ``import_global`` intersect requested names with ``dir(module)``. + names = set(dir(python_utils)) + assert {'containers', 'exceptions'} <= names + assert set(python_utils.__all__) <= names @pytest.mark.asyncio diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 9c3f951..438f746 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -172,7 +172,9 @@ def __getattr__(name: str) -> typing.Any: def __dir__() -> list[str]: - return sorted(set(globals()) | set(__all__)) + return sorted( + set(globals()) | set(__all__) | _SUBMODULES | set(_NAME_TO_MODULE) + ) __all__ = [ diff --git a/python_utils/time.py b/python_utils/time.py index 2a3187c..338ef18 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -290,7 +290,7 @@ async def aio_timeout_generator( if iterable is None: iterable = types.cast( - 'types.AsyncIterable[_T]', + types.Callable[[], types.AsyncIterable[_T]], aio.acount, ) diff --git a/python_utils/types.py b/python_utils/types.py index 8e1c369..ab89c43 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -32,13 +32,7 @@ Union as U, # noqa: N817 ) -if TYPE_CHECKING: # pragma: no cover - # Type checkers see the full typing_extensions surface here. At runtime - # typing_extensions is never imported eagerly; any name not provided by - # stdlib `typing` (e.g. `ParamSpec` on Python 3.9) is resolved lazily by - # `__getattr__` below, so importing this module (and therefore - # `python_utils`) does not pull in typing_extensions. - from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 +from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 Scope = Dict[str, Any] OptionalScope = O[Scope] @@ -59,30 +53,6 @@ None, ] - -def __getattr__(name: str) -> Any: - """Lazily resolve typing_extensions-only names on first access. - - On Python 3.11+ typing_extensions is not imported eagerly (see above); any - name not provided by stdlib ``typing``/``types`` is fetched from - typing_extensions here, on demand. - """ - if name.startswith('__') and name.endswith('__'): - raise AttributeError(name) - - import typing_extensions - - try: - value = getattr(typing_extensions, name) - except AttributeError: - raise AttributeError( - f'module {__name__!r} has no attribute {name!r}', - ) from None - - globals()[name] = value # cache so __getattr__ runs only once per name - return value - - __all__ = [ 'IO', 'TYPE_CHECKING', From 35a86ff94ead7606e7f1bca5a4efbb36053db5cc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 24 Jun 2026 20:29:27 +0200 Subject: [PATCH 3/3] Keep package namespace clean under lazy init Alias the internal 'importlib'/'typing' imports to '_importlib'/'_typing' so they no longer leak as public attributes of 'python_utils' (they appeared in dir() and as python_utils.importlib / python_utils.typing under the lazy __init__, which develop did not expose). Add an explicit '# pragma: no cover' to the TYPE_CHECKING block since coverage's auto-exclusion only matches the unaliased 'typing.TYPE_CHECKING' spelling. Verified via a full public-API manifest diff against develop: the only remaining differences are the two inherent consequences of deferring asyncio (aio_timeout_generator's default iterable is now None instead of aio.acount, and python_utils.time no longer re-exposes the 'aio'/'asyncio' modules). --- python_utils/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 438f746..0e14731 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -55,10 +55,10 @@ LoggerBase """ -import importlib -import typing +import importlib as _importlib +import typing as _typing -if typing.TYPE_CHECKING: +if _typing.TYPE_CHECKING: # pragma: no cover # Eager imports for type checkers only; the runtime equivalents are loaded # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are # treated as re-exports (not unused imports). @@ -155,12 +155,14 @@ } -def __getattr__(name: str) -> typing.Any: +def __getattr__(name: str) -> _typing.Any: """Lazily import submodules and their exported names on first access.""" if name in _SUBMODULES: - module = importlib.import_module(f'.{name}', __name__) + module = _importlib.import_module(f'.{name}', __name__) elif name in _NAME_TO_MODULE: - module = importlib.import_module(f'.{_NAME_TO_MODULE[name]}', __name__) + module = _importlib.import_module( + f'.{_NAME_TO_MODULE[name]}', __name__ + ) value = getattr(module, name) globals()[name] = value # cache so __getattr__ runs only once return value