From 3f4748c8cf017913f13666a8f9b269d569fd7c34 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 30 Jun 2026 21:45:18 +0200 Subject: [PATCH 01/23] Add CTable .b2z support: metadata model, fetch pipeline via to_cframe(), Python client Table class --- caterva2/client.py | 39 +++- caterva2/models.py | 15 ++ caterva2/services/server.py | 36 +++- caterva2/services/srv_utils.py | 23 ++- caterva2/tests/test_ctable.py | 223 ++++++++++++++++++++++ caterva2/tests/test_notebook_bootstrap.py | 26 ++- 6 files changed, 344 insertions(+), 18 deletions(-) create mode 100644 caterva2/tests/test_ctable.py diff --git a/caterva2/client.py b/caterva2/client.py index 126a2a66..ade4efb7 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -125,6 +125,8 @@ def __getitem__(self, path): path = path.as_posix() if hasattr(path, "as_posix") else str(path) if path.endswith((".b2nd", ".b2frame")): return Dataset(self, path) + if path.endswith(".b2z"): + return Table(self, path) else: return File(self, path) @@ -698,6 +700,29 @@ def append(self, data): return self.client.append(self.path, data) +class Table(File): + """Represents a CTable (.b2z) dataset.""" + + def __repr__(self): + return f"" + + @property + def nrows(self): + return self.meta["nrows"] + + @property + def columns(self): + return self.meta["columns"] + + def slice(self, key): + """Get a row slice as a blosc2.CTable.""" + return self.client.get_slice(self.path, key, as_blosc2=True) + + def __getitem__(self, key): + """Shortcut: slice as a blosc2.CTable.""" + return self.slice(key) + + class BasicAuth: """ Basic authentication for HTTP requests. @@ -781,14 +806,20 @@ def _fetch_data(self, path, urlbase, params, auth_cookie=None, as_blosc2=False, data = response.content # Try different deserialization methods try: - data = blosc2.ndarray_from_cframe(data) - except RuntimeError: - data = blosc2.schunk_from_cframe(data) + data = blosc2.ctable_from_cframe(data) + except ValueError: + try: + data = blosc2.ndarray_from_cframe(data) + except RuntimeError: + data = blosc2.schunk_from_cframe(data) if as_blosc2: return data if hasattr(data, "ndim"): # if b2nd or b2frame # catch 0d case where [:] fails return data[()] if data.ndim == 0 else data[:] + elif isinstance(data, blosc2.CTable): + # Return rows as list of tuples if not as_blosc2 + return [tuple(row) for row in data[:]] else: return data[:] @@ -964,7 +995,7 @@ def get(self, path): >>> ds[:10] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) """ - if isinstance(path, (File, Dataset, Root)): + if isinstance(path, (File, Dataset, Table, Root)): return path # Normalize the path to a POSIX path path = pathlib.PurePosixPath(path).as_posix() diff --git a/caterva2/models.py b/caterva2/models.py index 06a7d4ee..fbd157c1 100644 --- a/caterva2/models.py +++ b/caterva2/models.py @@ -74,6 +74,21 @@ class LazyArray(pydantic.BaseModel): mtime: datetime.datetime | None +class CTableMetadata(pydantic.BaseModel): + kind: str = "ctable" + nrows: int + ncols: int + chunks: tuple + blocks: tuple + schema_dict: dict + columns: list[str] + nbytes: int + cbytes: int + cratio: float + vlmeta: dict = {} + mtime: datetime.datetime | None + + class Cat2LazyArr(pydantic.BaseModel): name: str | None expression: str | None diff --git a/caterva2/services/server.py b/caterva2/services/server.py index 15b5def8..d1243462 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -127,6 +127,9 @@ def open_b2(abspath, path): raise ValueError(f"Unexpected root={root}") container = blosc2.open(abspath) + # CTable has its own storage and no table-level cparams/dparams; return early. + if isinstance(container, blosc2.CTable): + return container vlmeta = container.schunk.vlmeta if hasattr(container, "schunk") else container.vlmeta if isinstance(container, blosc2.LazyArray): # Open the operands properly @@ -453,7 +456,7 @@ def get_abspath( return cachedir / filepath # HDF5 files cannot be compressed, as they are supported natively - if filepath.suffix not in {".b2frame", ".b2nd", ".h5"} and not may_not_exist: + if filepath.suffix not in {".b2frame", ".b2nd", ".b2z", ".h5"} and not may_not_exist: if filepath.is_file(): srv_utils.compress_file(filepath) filepath = f"{filepath}.b2" @@ -521,9 +524,9 @@ async def fetch_data( slice_ = parse_slice(slice_) abspath = get_abspath(path, user) - if abspath.suffix not in {".b2frame", ".b2nd"}: + if abspath.suffix not in {".b2frame", ".b2nd", ".b2z"}: srv_utils.raise_bad_request( - "The fetch API only supports datasets (.b2nd and .b2); " + "The fetch API only supports datasets (.b2nd, .b2frame, .b2z); " "use the download API if you only want to download the file" ) @@ -539,11 +542,16 @@ async def fetch_data( if field: container = container[field] - if isinstance(container, blosc2.NDArray | blosc2.LazyArray | hdf5.HDF5Proxy | blosc2.NDField): + if isinstance(container, (blosc2.NDArray, blosc2.LazyArray, hdf5.HDF5Proxy, blosc2.NDField)): array = container schunk = getattr(array, "schunk", None) # not really needed typesize = array.dtype.itemsize shape = array.shape + elif isinstance(container, blosc2.CTable): + array = container + schunk = None + typesize = 1 # not used for CTable + shape = (array.nrows,) else: # SChunk array = None @@ -573,7 +581,25 @@ async def fetch_data( # avoiding slicing and re-compression. return FileResponse(abspath, filename=abspath.name, media_type="application/octet-stream") - if isinstance(array, hdf5.HDF5Proxy): + if isinstance(array, blosc2.CTable): + # ponytail: slice_ is a single slice/int/tuple; extract row start/stop + if isinstance(slice_, slice): + row_start, row_stop = slice_.start or 0, slice_.stop or array.nrows + elif isinstance(slice_, int): + row_start, row_stop = slice_, slice_ + 1 + elif isinstance(slice_, tuple) and len(slice_) > 0: + sl0 = slice_[0] + if isinstance(sl0, slice): + row_start, row_stop = sl0.start or 0, sl0.stop or array.nrows + elif isinstance(sl0, int): + row_start, row_stop = sl0, sl0 + 1 + else: + row_start, row_stop = 0, array.nrows + else: + row_start, row_stop = 0, array.nrows + view = array.slice(row_start, row_stop) + data = view.to_cframe() + elif isinstance(array, hdf5.HDF5Proxy): data = array.to_cframe(() if slice_ is None else slice_) elif isinstance(array, blosc2.LazyArray): data = array.compute(() if slice_ is None else slice_) diff --git a/caterva2/services/srv_utils.py b/caterva2/services/srv_utils.py index 5886b29d..2741f086 100644 --- a/caterva2/services/srv_utils.py +++ b/caterva2/services/srv_utils.py @@ -30,6 +30,12 @@ from caterva2 import hdf5, models from caterva2.services import db, schemas, settings, users +# Shared suffix constants +BLOSC2_ARRAY_SUFFIXES = {".b2nd", ".b2frame"} +BLOSC2_TABLE_SUFFIXES = {".b2z"} +BLOSC2_FRAME_SUFFIXES = {".b2"} +BLOSC2_NATIVE_SUFFIXES = BLOSC2_ARRAY_SUFFIXES | BLOSC2_TABLE_SUFFIXES | BLOSC2_FRAME_SUFFIXES + def compress_file(path): with open(path, "rb") as src: @@ -103,7 +109,7 @@ def read_metadata(obj): # obj = f return get_model_from_obj(obj, models.File, mtime=mtime, size=size) - assert path.suffix in {".b2frame", ".b2nd", ".b2"} + assert path.suffix in BLOSC2_NATIVE_SUFFIXES try: obj = blosc2.open(path) except blosc2.exceptions.MissingOperands as exc: @@ -150,6 +156,21 @@ def read_metadata(obj): mtime=mtime, expression=expression, ) + elif isinstance(obj, blosc2.CTable): + schema = obj.schema_dict() + return models.CTableMetadata( + nrows=obj.nrows, + ncols=obj.ncols, + chunks=obj.chunks, + blocks=obj.blocks, + schema_dict=schema, + columns=[c["name"] for c in schema.get("columns", [])], + nbytes=obj.nbytes, + cbytes=obj.cbytes, + cratio=obj.cratio, + vlmeta=dict(obj.vlmeta[:]) if obj.vlmeta[:] else {}, + mtime=mtime, + ) else: raise TypeError(f"unexpected {type(obj)}") diff --git a/caterva2/tests/test_ctable.py b/caterva2/tests/test_ctable.py new file mode 100644 index 00000000..76c3f012 --- /dev/null +++ b/caterva2/tests/test_ctable.py @@ -0,0 +1,223 @@ +"""Tests for CTable .b2z metadata and fetch deserialization in Caterva2.""" +# ruff: noqa: RUF009 # blosc2.field() is the standard CTable dataclass default API + +import pathlib +import tempfile +from dataclasses import dataclass + +import blosc2 +import blosc2.ctable as ct + +from caterva2.services import srv_utils + + +def _make_table(path, n=5): + @dataclass + class Row: + x: int = blosc2.field(blosc2.int32()) + y: str = blosc2.field(blosc2.string(max_length=20)) + + t = blosc2.CTable(Row, urlpath=str(path), mode="w", compact=True) + for i in range(n): + t.append((i, f"v{i}")) + t.close() + + +# --------------------------------------------------------------------------- +# read_metadata +# --------------------------------------------------------------------------- + + +def test_read_metadata(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "t.b2z" + _make_table(table_path) + + meta = srv_utils.read_metadata(table_path) + assert meta.kind == "ctable" + assert meta.nrows == 5 + assert meta.ncols == 2 + assert meta.columns == ["x", "y"] + assert meta.cbytes > 0 + assert meta.nbytes > meta.cbytes + assert meta.cratio > 1.0 + assert isinstance(meta.chunks, tuple) + + +def test_read_metadata_empty(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "empty.b2z" + + @dataclass + class Row: + x: int = blosc2.field(blosc2.int32()) + + t = blosc2.CTable(Row, urlpath=str(table_path), mode="w", compact=True) + t.close() + + meta = srv_utils.read_metadata(table_path) + assert meta.nrows == 0 + assert meta.ncols == 1 + assert meta.columns == ["x"] + + +def test_read_metadata_list_column(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "list.b2z" + + @dataclass + class Row: + x: int = blosc2.field(blosc2.int32()) + tags: list[int] = blosc2.field(ct.ListSpec(ct.int32())) + + t = blosc2.CTable(Row, urlpath=str(table_path), mode="w", compact=True) + for r in [(1, [1, 2]), (2, [3])]: + t.append(r) + t.close() + + meta = srv_utils.read_metadata(table_path) + assert meta.nrows == 2 + assert meta.ncols == 2 + assert meta.columns == ["x", "tags"] + + +def test_read_metadata_vlstring(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "vlstr.b2z" + + @dataclass + class Row: + x: int = blosc2.field(blosc2.int32()) + label: str = blosc2.field(ct.VLStringSpec()) + + t = blosc2.CTable(Row, urlpath=str(table_path), mode="w", compact=True) + for r in [(1, "hi"), (2, "yo")]: + t.append(r) + t.close() + + meta = srv_utils.read_metadata(table_path) + assert meta.nrows == 2 + assert meta.columns == ["x", "label"] + + +def test_read_metadata_vlmeta(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "vlmeta.b2z" + + @dataclass + class Row: + x: int = blosc2.field(blosc2.int32()) + + t = blosc2.CTable(Row, urlpath=str(table_path), mode="w", compact=True) + t.append((0,)) + t.vlmeta["author"] = "Alice" + t.close() + + meta = srv_utils.read_metadata(table_path) + assert meta.vlmeta == {"author": "Alice"} + + +def test_read_metadata_nonexistent(): + import pytest + + tmp = pathlib.Path(tempfile.mkdtemp()) + with pytest.raises(FileNotFoundError): + srv_utils.read_metadata(tmp / "nope.b2z") + + +# --------------------------------------------------------------------------- +# fetch deserialization (simulating what /api/fetch and Client._fetch_data do) +# --------------------------------------------------------------------------- + + +def test_cframe_deserialization_slice(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "t.b2z" + _make_table(table_path) + + container = blosc2.open(table_path) + view = container.slice(1, 4) + cf = view.to_cframe() + + back = blosc2.ctable_from_cframe(cf) + assert isinstance(back, blosc2.CTable) + assert len(back) == 3 + assert [tuple(r) for r in back] == [(1, "v1"), (2, "v2"), (3, "v3")] + + +def test_cframe_deserialization_whole(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "t.b2z" + _make_table(table_path) + + container = blosc2.open(table_path) + cf = container.to_cframe() + back = blosc2.ctable_from_cframe(cf) + assert len(back) == 5 + assert back[0].x == 0 + assert back[4].x == 4 + assert back[2].y == "v2" + + +def test_cframe_deserialization_single_row(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "t.b2z" + _make_table(table_path) + + container = blosc2.open(table_path) + view = container.slice(2, 3) + cf = view.to_cframe() + back = blosc2.ctable_from_cframe(cf) + assert len(back) == 1 + assert tuple(back[0]) == (2, "v2") + + +def test_cframe_deserialization_empty_slice(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "t.b2z" + _make_table(table_path) + + container = blosc2.open(table_path) + view = container.slice(0, 0) + cf = view.to_cframe() + back = blosc2.ctable_from_cframe(cf) + assert len(back) == 0 + assert back.col_names == ["x", "y"] + + +def test_cframe_deserialization_multi_column(): + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "cols.b2z" + + @dataclass + class Row: + a: int = blosc2.field(blosc2.int32()) + b: float = blosc2.field(blosc2.float64()) + c: str = blosc2.field(blosc2.string(max_length=10)) + + t = blosc2.CTable(Row, urlpath=str(table_path), mode="w", compact=True) + for r in [(1, 2.5, "hi"), (2, 3.5, "yo")]: + t.append(r) + t.close() + + container = blosc2.open(table_path) + cf = container.to_cframe() + back = blosc2.ctable_from_cframe(cf) + assert back.col_names == ["a", "b", "c"] + assert back[0].b == 2.5 + assert back[1].c == "yo" + + +def test_cframe_deserialization_large_slice(): + """Ensure a slice of many rows round-trips correctly.""" + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "big.b2z" + _make_table(table_path, n=200) + + container = blosc2.open(table_path) + view = container.slice(50, 150) + cf = view.to_cframe() + back = blosc2.ctable_from_cframe(cf) + assert len(back) == 100 + assert back[0].x == 50 + assert back[99].x == 149 diff --git a/caterva2/tests/test_notebook_bootstrap.py b/caterva2/tests/test_notebook_bootstrap.py index bcb492b3..dab031c9 100644 --- a/caterva2/tests/test_notebook_bootstrap.py +++ b/caterva2/tests/test_notebook_bootstrap.py @@ -1,21 +1,29 @@ import nbformat -from caterva2.services.notebook import PYODIDE_BOOTSTRAP_CELL_SOURCE, inject_pyodide_bootstrap_cell +from caterva2.services.notebook import ( + PYODIDE_BOOTSTRAP_IMPORT_SOURCE, + PYODIDE_BOOTSTRAP_INSTALL_SOURCE, + inject_pyodide_bootstrap_cell, +) def _bootstrap_cells(notebook): return [cell for cell in notebook.cells if cell.get("metadata", {}).get("caterva2_pyodide_bootstrap")] -def test_inject_pyodide_bootstrap_cell_adds_cell_once(): +def test_inject_pyodide_bootstrap_cell_adds_cells(): notebook = nbformat.v4.new_notebook(cells=[nbformat.v4.new_code_cell("print('hello')")]) content = nbformat.writes(notebook).encode("utf-8") patched = inject_pyodide_bootstrap_cell(content) patched_notebook = nbformat.reads(patched.decode("utf-8"), as_version=4) - assert len(_bootstrap_cells(patched_notebook)) == 1 + # Two cells: install (no imports) + import (finds freshly installed packages) + assert len(_bootstrap_cells(patched_notebook)) == 2 assert patched_notebook.cells[0]["metadata"]["caterva2_pyodide_bootstrap"] is True + assert patched_notebook.cells[1]["metadata"]["caterva2_pyodide_bootstrap"] is True + assert patched_notebook.cells[0].source == PYODIDE_BOOTSTRAP_INSTALL_SOURCE + assert patched_notebook.cells[1].source == PYODIDE_BOOTSTRAP_IMPORT_SOURCE def test_inject_pyodide_bootstrap_cell_is_idempotent(): @@ -26,11 +34,11 @@ def test_inject_pyodide_bootstrap_cell_is_idempotent(): patched_twice = inject_pyodide_bootstrap_cell(patched_once) patched_notebook = nbformat.reads(patched_twice.decode("utf-8"), as_version=4) - assert len(_bootstrap_cells(patched_notebook)) == 1 + assert len(_bootstrap_cells(patched_notebook)) == 2 -def test_inject_pyodide_bootstrap_cell_refreshes_stale_cell(): - # A notebook with an OUTDATED bootstrap cell baked in (e.g. persisted by an +def test_inject_pyodide_bootstrap_cell_refreshes_stale_cells(): + # A notebook with OUTDATED bootstrap cells baked in (e.g. persisted by an # older server / save-back) must be refreshed to the current source, not skipped. stale = nbformat.v4.new_code_cell("# old install code\nawait micropip.install('blosc2')") stale["metadata"]["caterva2_pyodide_bootstrap"] = True @@ -40,6 +48,8 @@ def test_inject_pyodide_bootstrap_cell_refreshes_stale_cell(): patched_notebook = nbformat.reads(inject_pyodide_bootstrap_cell(content).decode("utf-8"), as_version=4) bootstrap = _bootstrap_cells(patched_notebook) - assert len(bootstrap) == 1 + assert len(bootstrap) == 2 assert patched_notebook.cells[0]["metadata"]["caterva2_pyodide_bootstrap"] is True - assert patched_notebook.cells[0].source == PYODIDE_BOOTSTRAP_CELL_SOURCE + assert patched_notebook.cells[1]["metadata"]["caterva2_pyodide_bootstrap"] is True + assert patched_notebook.cells[0].source == PYODIDE_BOOTSTRAP_INSTALL_SOURCE + assert patched_notebook.cells[1].source == PYODIDE_BOOTSTRAP_IMPORT_SOURCE From 32947e82ba5f07d86f84e285d2c9de397b774fde Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 11:58:17 +0200 Subject: [PATCH 02/23] Fix CTable whole-fetch bug, add Array/Table client hierarchy, and .b2z storage handling Whole-table /api/fetch was returning the raw .b2z zip instead of a cframe, so table[:] failed client-side. Also introduces Array/Table as proper Dataset subclasses (client.py), dispatches cframe decoding by known kind instead of trial/except, adds Table.nrows/columns/head/rows, and treats .b2z as a native Blosc2 suffix for upload/load_from_url/htmx paths so tables round-trip byte-identical. Adds regression tests. --- caterva2/__init__.py | 4 +- caterva2/client.py | 172 ++++++++++++++++++++++++---------- caterva2/services/server.py | 16 ++-- caterva2/tests/test_ctable.py | 81 ++++++++++++++++ 4 files changed, 212 insertions(+), 61 deletions(-) diff --git a/caterva2/__init__.py b/caterva2/__init__.py index 03b33417..e1741c86 100644 --- a/caterva2/__init__.py +++ b/caterva2/__init__.py @@ -9,16 +9,18 @@ """Caterva2 - On demand access to remote Blosc2 data repositories""" -from .client import BasicAuth, Client, Dataset, File, Root +from .client import Array, BasicAuth, Client, Dataset, File, Root, Table __version__ = "2025.12.4.dev0" """The version in use of the Caterva2 package.""" __all__ = [ + "Array", "BasicAuth", "Client", "Dataset", "File", "Root", + "Table", ] """List of symbols exported by the Caterva2 package.""" diff --git a/caterva2/client.py b/caterva2/client.py index ade4efb7..9db95c46 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -120,11 +120,11 @@ def __getitem__(self, path): >>> client = cat2.Client('https://cat2.cloud/demo') >>> root = client.get('example') >>> root['ds-1d.b2nd'] - + """ path = path.as_posix() if hasattr(path, "as_posix") else str(path) if path.endswith((".b2nd", ".b2frame")): - return Dataset(self, path) + return Array(self, path) if path.endswith(".b2z"): return Table(self, path) else: @@ -231,7 +231,7 @@ def upload(self, local_dset, remotepath=None, compute=None): >>> root = client.get('@personal') >>> path = f'@personal/dir{np.random.randint(0, 100)}/ds-4d.b2nd' >>> root.upload('root-example/dir2/ds-4d.b2nd') - + >>> 'root-example/dir2/ds-4d.b2nd' in root True """ @@ -370,7 +370,7 @@ def get_download_url(self): def __getitem__(self, item): """ - Retrieves a slice of the dataset. + Retrieves a slice of the file's data (only meaningful for datasets). Parameters ---------- @@ -381,35 +381,8 @@ def __getitem__(self, item): ------- numpy.ndarray The requested slice of the dataset. - - Examples - -------- - >>> import caterva2 as cat2 - >>> client = cat2.Client('https://cat2.cloud/demo') - >>> root = client.get('example') - >>> ds = root['ds-1d.b2nd'] - >>> ds[1] - array(1) - >>> ds[:1] - array([0]) - >>> ds[0:10] - array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) """ - if isinstance(item, str): # used a filter or field to index so want blosc2 array as result - try: - dtype = np.dtype(self.dtype) - except (ValueError, TypeError): - dtype = np.dtype(ast.literal_eval(self.dtype)) - fields = dtype.fields - if fields is None: - raise ValueError("The array is not structured (its dtype does not have fields)") - if item in fields: - # A shortcut to access fields - return self.client.get_slice(self.path, as_blosc2=True, field=item) # arg key is None - else: # used a filter (possibly lazyexpr) - return self.client.get_slice(self.path, item, as_blosc2=True) - else: - return self.slice(item, as_blosc2=False) + return self.slice(item, as_blosc2=False) def slice( self, key: int | slice | Sequence[slice], as_blosc2: bool = True @@ -521,7 +494,7 @@ def move(self, dst): >>> client = cat2.Client("https://cat2.cloud/demo", ("joedoe@example.com", "foobar")) >>> root = client.get('@personal') >>> root.upload('root-example/dir2/ds-4d.b2nd') - + >>> file = root['root-example/dir2/ds-4d.b2nd'] >>> file.move('@personal/root-example/dir1/ds-4d-moved.b2nd') PurePosixPath('@personal/root-example/dir1/ds-4d-moved.b2nd') @@ -554,7 +527,7 @@ def copy(self, dst): >>> client = cat2.Client("https://cat2.cloud/demo", ("joedoe@example.com", "foobar")) >>> root = client.get('@personal') >>> root.upload('root-example/dir2/ds-4d.b2nd') - + >>> file = root['root-example/dir2/ds-4d.b2nd'] >>> file.copy('@personal/root-example/dir2/ds-4d-copy.b2nd') PurePosixPath('@personal/root-example/dir2/ds-4d-copy.b2nd') @@ -583,7 +556,7 @@ def remove(self): >>> root = client.get('@personal') >>> path = 'root-example/dir2/ds-4d.b2nd' >>> root.upload(path) - + >>> file = root[path] >>> file.remove() '@personal/root-example/dir2/ds-4d.b2nd' @@ -593,10 +566,25 @@ def remove(self): return self.client.remove(self.path) -class Dataset(File, blosc2.Operand): +class Dataset(File): + """Generic fetchable-dataset base, shared by :class:`Array` and :class:`Table`. + + This class is not intended to be instantiated directly; it should be + accessed through a :class:`Root` instance, which returns an :class:`Array` + or a :class:`Table` depending on the dataset kind. + """ + + def __str__(self): + return self.path.as_posix() + + def __repr__(self): + return f"" + + +class Array(Dataset, blosc2.Operand): def __init__(self, root, path): """ - Represents a dataset within a Blosc2 container. + Represents an array dataset (.b2nd, .b2frame) within a Blosc2 container. This class is not intended to be instantiated directly; it should be accessed through a :class:`Root` instance. @@ -630,7 +618,7 @@ def __str__(self): def __repr__(self): # TODO: add more info about dims, types, etc. - return f"" + return f"" @property def dtype(self): @@ -640,7 +628,7 @@ def dtype(self): try: return self.meta["dtype"] except KeyError as e: - raise AttributeError("'Dataset' object has no attribute 'dtype'.") from e + raise AttributeError("'Array' object has no attribute 'dtype'.") from e @property def shape(self): @@ -650,7 +638,14 @@ def shape(self): try: return tuple(self.meta["shape"]) except KeyError as e: - raise AttributeError("'Dataset' object has no attribute 'shape'.") from e + raise AttributeError("'Array' object has no attribute 'shape'.") from e + + @property + def ndim(self): + """ + The number of dimensions of the dataset. + """ + return len(self.shape) @property def chunks(self): @@ -660,7 +655,7 @@ def chunks(self): try: return tuple(self.meta["chunks"]) except KeyError as e: - raise AttributeError("'Dataset' object has no attribute 'chunks'.") from e + raise AttributeError("'Array' object has no attribute 'chunks'.") from e @property def blocks(self): @@ -670,7 +665,7 @@ def blocks(self): try: return tuple(self.meta["blocks"]) except KeyError as e: - raise AttributeError("'Dataset' object has no attribute 'blocks'.") from e + raise AttributeError("'Array' object has no attribute 'blocks'.") from e def append(self, data): """ @@ -683,7 +678,7 @@ def append(self, data): Returns ------- - out: Caterva2.Dataset + out: Caterva2.Array A pointer to the (modified) dataset. Examples @@ -699,10 +694,56 @@ def append(self, data): """ return self.client.append(self.path, data) + def __getitem__(self, item): + """ + Retrieves a slice of the dataset. + + Parameters + ---------- + item : int, slice, tuple of ints and slices, or None + Specifies the slice to fetch. + + Returns + ------- + numpy.ndarray + The requested slice of the dataset. -class Table(File): + Examples + -------- + >>> import caterva2 as cat2 + >>> client = cat2.Client('https://cat2.cloud/demo') + >>> root = client.get('example') + >>> ds = root['ds-1d.b2nd'] + >>> ds[1] + array(1) + >>> ds[:1] + array([0]) + >>> ds[0:10] + array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + """ + if isinstance(item, str): # used a filter or field to index so want blosc2 array as result + try: + dtype = np.dtype(self.dtype) + except (ValueError, TypeError): + dtype = np.dtype(ast.literal_eval(self.dtype)) + fields = dtype.fields + if fields is None: + raise ValueError("The array is not structured (its dtype does not have fields)") + if item in fields: + # A shortcut to access fields + return self.client.get_slice(self.path, as_blosc2=True, field=item) # arg key is None + else: # used a filter (possibly lazyexpr) + return self.client.get_slice(self.path, item, as_blosc2=True) + else: + return self.slice(item, as_blosc2=False) + + +class Table(Dataset): """Represents a CTable (.b2z) dataset.""" + def __str__(self): + return self.path.as_posix() + def __repr__(self): return f"" @@ -710,10 +751,18 @@ def __repr__(self): def nrows(self): return self.meta["nrows"] + @property + def ncols(self): + return len(self.meta["columns"]) + @property def columns(self): return self.meta["columns"] + @property + def schema(self): + return self.meta["schema_dict"] + def slice(self, key): """Get a row slice as a blosc2.CTable.""" return self.client.get_slice(self.path, key, as_blosc2=True) @@ -722,6 +771,16 @@ def __getitem__(self, key): """Shortcut: slice as a blosc2.CTable.""" return self.slice(key) + def rows(self, start=0, stop=None): + """Get rows [start, stop) as a list of tuples.""" + stop = self.nrows if stop is None else stop + table = self.slice(slice(start, stop)) + return [tuple(row) for row in table] + + def head(self, n=5): + """Get the first `n` rows as a list of tuples.""" + return self.rows(0, n) + class BasicAuth: """ @@ -799,15 +858,14 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False - def _fetch_data(self, path, urlbase, params, auth_cookie=None, as_blosc2=False, timeout=5): + def _fetch_data(self, path, urlbase, params, auth_cookie=None, as_blosc2=False, timeout=5, kind=None): response = self._xget( f"{urlbase}/api/fetch/{path}", params=params, auth_cookie=auth_cookie, timeout=timeout ) data = response.content - # Try different deserialization methods - try: + if kind == "ctable": data = blosc2.ctable_from_cframe(data) - except ValueError: + else: try: data = blosc2.ndarray_from_cframe(data) except RuntimeError: @@ -995,7 +1053,7 @@ def get(self, path): >>> ds[:10] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) """ - if isinstance(path, (File, Dataset, Table, Root)): + if isinstance(path, (File, Root)): return path # Normalize the path to a POSIX path path = pathlib.PurePosixPath(path).as_posix() @@ -1057,7 +1115,7 @@ def get_info(self, path): >>> info['shape'] [100, 200] """ - if isinstance(path, (Dataset, File)): + if isinstance(path, File): path = path.path _, path = _format_paths(self.urlbase, path) return self._get(f"{self.urlbase}/api/info/{path}", auth_cookie=self.cookie, timeout=self.timeout) @@ -1123,7 +1181,14 @@ def get_slice(self, path, key=None, as_blosc2=True, field=None): [(1.0000500e-02, 1.0100005), (1.0050503e-02, 1.0100505)]], dtype=[('a', '>> info_schunk['chunksize'] / len(chunk) 6.453000645300064 """ - if isinstance(path, Dataset): + if isinstance(path, File): path = path.path _, path = _format_paths(self.urlbase, path) data = self._xget( diff --git a/caterva2/services/server.py b/caterva2/services/server.py index d1243462..704e186d 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -574,7 +574,7 @@ async def fetch_data( if ( whole - and (not isinstance(array, blosc2.LazyArray | hdf5.HDF5Proxy | blosc2.NDField)) + and (not isinstance(array, blosc2.LazyArray | hdf5.HDF5Proxy | blosc2.NDField | blosc2.CTable)) and (not filter) ): # Send the data in the file straight to the client, @@ -1038,7 +1038,7 @@ async def upload_file( # Check quota # TODO To be fair we should check quota later (after compression, zip unpacking etc.) data = await file.read() - if abspath.suffix not in {".b2", ".b2frame", ".b2nd"}: + if abspath.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES: schunk = blosc2.SChunk(data=data) newsize = schunk.nbytes else: @@ -1057,7 +1057,7 @@ async def upload_file( # If regular file, compress it abspath.parent.mkdir(exist_ok=True, parents=True) - if abspath.suffix not in {".b2", ".b2frame", ".b2nd", ".h5", ".hdf5"}: + if abspath.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES | {".h5", ".hdf5"}: data = schunk.to_cframe() abspath = abspath.with_suffix(abspath.suffix + ".b2") @@ -1107,7 +1107,7 @@ async def load_from_url( response.raise_for_status() data = response.content - if abspath.suffix not in {".b2", ".b2frame", ".b2nd"}: + if abspath.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES: schunk = blosc2.SChunk(data=data) newsize = schunk.nbytes else: @@ -1126,7 +1126,7 @@ async def load_from_url( # If regular file, compress it abspath.parent.mkdir(exist_ok=True, parents=True) - if abspath.suffix not in {".b2", ".b2frame", ".b2nd", ".h5", ".hdf5"}: + if abspath.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES | {".h5", ".hdf5"}: data = schunk.to_cframe() abspath = abspath.with_suffix(abspath.suffix + ".b2") @@ -1687,7 +1687,7 @@ def add_dataset(path, abspath): if rootdir is not None: relpath = pathlib.Path(*segments[2:]) abspath = rootdir / relpath - if abspath.suffix not in {".b2", ".b2nd", ".b2frame"}: + if abspath.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES: abspath = pathlib.Path(f"{abspath}.b2") with contextlib.suppress(FileNotFoundError): @@ -2325,7 +2325,7 @@ async def htmx_upload( new_members = [ member for member in members - if not (path / member).is_dir() and member.suffix not in {".b2", ".b2frame", ".b2nd"} + if not (path / member).is_dir() and member.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES ] for member in new_members: srv_utils.compress_file(path / member) @@ -2337,7 +2337,7 @@ async def htmx_upload( if suffix in [".h5", ".hdf5"]: pass - elif filename.suffix not in {".b2", ".b2frame", ".b2nd"}: + elif filename.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES: schunk = blosc2.SChunk(data=data) data = schunk.to_cframe() filename = f"{filename}.b2" diff --git a/caterva2/tests/test_ctable.py b/caterva2/tests/test_ctable.py index 76c3f012..7f5f12a9 100644 --- a/caterva2/tests/test_ctable.py +++ b/caterva2/tests/test_ctable.py @@ -7,9 +7,13 @@ import blosc2 import blosc2.ctable as ct +import pytest +import caterva2 as cat2 from caterva2.services import srv_utils +from .services import TEST_CATERVA2_ROOT, TEST_STATE_DIR + def _make_table(path, n=5): @dataclass @@ -221,3 +225,80 @@ def test_cframe_deserialization_large_slice(): assert len(back) == 100 assert back[0].x == 50 assert back[99].x == 149 + + +# --------------------------------------------------------------------------- +# HTTP-level / Python client tests (require the test services) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fill_ctable_public(client): + """Drop a small .b2z table straight into the @public root's storage.""" + dest_dir = pathlib.Path(TEST_STATE_DIR) / "server/public" + dest_dir.mkdir(parents=True, exist_ok=True) + fname = "test_table.b2z" + _make_table(dest_dir / fname, n=3) + return fname, client.get(TEST_CATERVA2_ROOT) + + +def test_http_info(fill_ctable_public, client): + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" + info = client.get_info(path) + assert info["kind"] == "ctable" + assert info["nrows"] == 3 + assert info["columns"] == ["x", "y"] + + +def test_http_download_roundtrip(fill_ctable_public, tmp_path): + fname, root = fill_ctable_public + table = root[fname] + local_orig = pathlib.Path(TEST_STATE_DIR) / "server/public" / fname + downloaded = table.download(tmp_path / "downloaded.b2z") + assert pathlib.Path(downloaded).read_bytes() == local_orig.read_bytes() + + reopened = blosc2.open(downloaded) + assert isinstance(reopened, blosc2.CTable) + assert len(reopened) == 3 + + +def test_fetch_whole_and_slice(fill_ctable_public, client): + """Regression test for T1: whole-table fetch must not return the raw zip.""" + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" + + whole = client.get_slice(path, as_blosc2=True) + assert isinstance(whole, blosc2.CTable) + assert len(whole) == 3 + assert whole[0].x == 0 + assert whole[2].x == 2 + + part = client.get_slice(path, slice(1, 3), as_blosc2=True) + assert isinstance(part, blosc2.CTable) + assert len(part) == 2 + assert tuple(part[0]) == (1, "v1") + assert tuple(part[1]) == (2, "v2") + + +def test_client_table_class(fill_ctable_public): + fname, root = fill_ctable_public + table = root[fname] + + assert isinstance(table, cat2.Table) + assert isinstance(table, cat2.Dataset) + assert not isinstance(table, cat2.Array) + + assert table.nrows == 3 + assert table.ncols == 2 + assert table.columns == ["x", "y"] + + assert table.head(2) == [(0, "v0"), (1, "v1")] + + whole = table[:] + assert isinstance(whole, blosc2.CTable) + assert len(whole) == 3 + + part = table.slice(slice(1, 3)) + assert isinstance(part, blosc2.CTable) + assert len(part) == 2 From 6a4551877a4292390e259d701d25a6a327a630d3 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 13:16:53 +0200 Subject: [PATCH 03/23] Add web preview and CLI support for CTable (.b2z) datasets htmx_path_info/htmx_path_view: render a paged row/column preview for CTable using schema_dict(), with Filter/Sort-by hidden (filterable flag) since they don't apply to tables; also fixes a pre-existing crash in the Meta tab template for CTableMetadata (no cparams). cli.py: `info` prints table-shaped fields instead of crashing on cparams.get(None); `show` parses the optional row-slice syntax (table.b2z[start:stop]) and prints rows via the Table client class instead of calling the array-oriented fetch(). Adds regression tests for both surfaces, plus tests for nested/ non-identifier CTable column names (e.g. "trip.sec" struct leaves), now resolved natively by blosc2's CTableRow.__getitem__. --- caterva2/clients/cli.py | 39 ++++ caterva2/services/server.py | 51 +++++- .../templates/includes/info_metadata.html | 8 + caterva2/services/templates/info_view.html | 4 +- caterva2/tests/test_ctable.py | 170 ++++++++++++++++++ 5 files changed, 268 insertions(+), 4 deletions(-) diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index 1242d9ff..e5ba8ed6 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -200,6 +200,21 @@ def _filter_names(fl): names.append(f"f{f}") return names + if data.get("kind") == "ctable": + nbytes = data.get("nbytes") + cbytes = data.get("cbytes") + mtime = data.get("mtime") + print(f"nrows : {data.get('nrows')}") + print(f"ncols : {data.get('ncols')}") + print(f"chunks : {data.get('chunks')}") + print(f"blocks : {data.get('blocks')}") + print(f"columns: {data.get('columns')}") + print(f"nbytes : {_human_bytes(nbytes)}") + print(f"cbytes : {_human_bytes(cbytes)}") + print(f"ratio : {nbytes / cbytes:.2f}x" if nbytes and cbytes else "ratio : N/A") + print(f"mtime : {mtime}") if mtime is not None else print("mtime : None") + return + # Extract fields schunk = data.get("schunk") or data cparams = schunk.get("cparams") @@ -237,10 +252,34 @@ def _filter_names(fl): print(" filters: None") +def _parse_row_slice(slice_, nrows): + if not slice_: + return 0, nrows + if ":" in slice_: + start_s, _, stop_s = slice_.partition(":") + start = int(start_s) if start_s.strip() else 0 + stop = int(stop_s) if stop_s.strip() else nrows + return start, stop + start = int(slice_) + return start, start + 1 + + @handle_errors def cmd_show(client, args, url): path, params = args.dataset slice_ = params.get("slice_", None) + + if str(path).endswith(".b2z"): + table = client.get(path) + start, stop = _parse_row_slice(slice_, table.nrows) + rows = table.rows(start, stop) + if getattr(args, "json", False): + print(json.dumps(rows)) + else: + for row in rows: + print(row) + return + data = client.fetch(path, slice_=slice_) # JSON output requested -> convert numpy arrays to lists diff --git a/caterva2/services/server.py b/caterva2/services/server.py index 704e186d..ec940641 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -1781,10 +1781,11 @@ async def htmx_path_info( } ) - # Tabs: Display (b2nd) - if hasattr(meta, "shape"): + # Tabs: Display (b2nd, b2z) + is_ctable = getattr(meta, "kind", None) == "ctable" + if hasattr(meta, "shape") or is_ctable: context["data_url"] = make_url(request, "htmx_path_view", path=path) - context["shape"] = meta.shape + context["shape"] = meta.shape if hasattr(meta, "shape") else (meta.nrows,) tabs.append( { "name": "data", @@ -1902,6 +1903,49 @@ async def htmx_path_view( return htmx_error(request, "Cannot open array; missing operand?, unknown data source?") idx = None + if isinstance(arr, blosc2.CTable): + schema = arr.schema_dict() + cols = [c["name"] for c in schema.get("columns", [])] + fields = fields or cols[:5] + nrows = arr.nrows + size = sizes[0] if sizes else min(nrows, 10) + start = index[0] if index else 0 + stop = min(start + size, nrows) + mod = nrows % size if size else 0 + start_max = nrows - (mod or size) if size else 0 + inputs = [ + { + "start": start, + "start_max": max(start_max, 0), + "size": size, + "size_max": nrows, + "with_size": True, + } + ] + tags = list(range(start, stop)) + + def cell(value): + if isinstance(value, bytes): + return value.decode(errors="replace") + if isinstance(value, np.generic): + return value.item() + return value + + rows = [fields] + [[cell(row[f]) for f in fields] for row in arr.slice(start, stop)] + context = { + "view_url": make_url(request, "htmx_path_view", path=path), + "inputs": inputs, + "rows": rows, + "cols": cols, + "fields": fields, + "filter": "", + "sortby": "", + "shape": (nrows,), + "tags": tags, + "filterable": False, + } + return templates.TemplateResponse(request, "info_view.html", context) + # Local variables shape = arr.shape ndims = len(shape) @@ -1990,6 +2034,7 @@ async def htmx_path_view( "sortby": sortby, "shape": shape, "tags": tags if len(tags) == 0 else tags[0], + "filterable": True, } return templates.TemplateResponse(request, "info_view.html", context) diff --git a/caterva2/services/templates/includes/info_metadata.html b/caterva2/services/templates/includes/info_metadata.html index 2e06ca59..855c45b2 100644 --- a/caterva2/services/templates/includes/info_metadata.html +++ b/caterva2/services/templates/includes/info_metadata.html @@ -35,6 +35,14 @@ {% set _ = meta.schunk.__delattr__('cbytes') %} {% set _ = meta.schunk.__delattr__('cratio') %} + {% elif meta.kind == 'ctable' %} + {% set nbytes_value = '%d (cbytes: %d ; cratio: %.2f)' % + (meta.nbytes, meta.cbytes, meta.cratio) %} + {% set _ = meta.__setattr__('nbytes', nbytes_value) %} + {% set _ = meta.__delattr__('cbytes') %} + {% set _ = meta.__delattr__('cratio') %} + {% set _ = meta.__delattr__('schema_dict') %} + {% else %} {% if meta.nbytes %} {% set nbytes_value = '%d (cbytes: %d ; cratio: %.2f)' % diff --git a/caterva2/services/templates/info_view.html b/caterva2/services/templates/info_view.html index a584cc09..c6fd26a3 100644 --- a/caterva2/services/templates/info_view.html +++ b/caterva2/services/templates/info_view.html @@ -46,6 +46,7 @@ {% endfor %} + {% if filterable %}
{% endif %} + {% endif %} {% with id="id_data_indicator" %} {% include 'includes/loading.html' %} {% endwith %} - {% if cols %} + {% if cols and filterable %}
diff --git a/caterva2/tests/test_ctable.py b/caterva2/tests/test_ctable.py index 7f5f12a9..c84e3a07 100644 --- a/caterva2/tests/test_ctable.py +++ b/caterva2/tests/test_ctable.py @@ -1,15 +1,21 @@ """Tests for CTable .b2z metadata and fetch deserialization in Caterva2.""" # ruff: noqa: RUF009 # blosc2.field() is the standard CTable dataclass default API +import io +import json import pathlib +import sys import tempfile +from contextlib import redirect_stdout from dataclasses import dataclass import blosc2 import blosc2.ctable as ct +import httpx import pytest import caterva2 as cat2 +from caterva2.clients import cli from caterva2.services import srv_utils from .services import TEST_CATERVA2_ROOT, TEST_STATE_DIR @@ -302,3 +308,167 @@ def test_client_table_class(fill_ctable_public): part = table.slice(slice(1, 3)) assert isinstance(part, blosc2.CTable) assert len(part) == 2 + + +# --------------------------------------------------------------------------- +# CLI: info / show for .b2z +# --------------------------------------------------------------------------- + + +def _run_cli(argv): + buf = io.StringIO() + old_argv = sys.argv + sys.argv = ["cat2-client", *argv] + try: + with redirect_stdout(buf): + cli.main() + finally: + sys.argv = old_argv + return buf.getvalue() + + +def test_cli_info_json(fill_ctable_public, services): + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" + out = _run_cli(["--url", services.get_urlbase(), "info", path, "--json"]) + data = json.loads(out.strip().splitlines()[-1]) + assert data["kind"] == "ctable" + assert data["nrows"] == 3 + + +def test_cli_info_text(fill_ctable_public, services): + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" + out = _run_cli(["--url", services.get_urlbase(), "info", path]) + assert "nrows : 3" in out + assert "columns: ['x', 'y']" in out + + +def test_cli_show(fill_ctable_public, services): + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" + out = _run_cli(["--url", services.get_urlbase(), "show", path]) + assert "(0, 'v0')" in out + assert "(2, 'v2')" in out + + +def test_cli_show_slice(fill_ctable_public, services): + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" + out = _run_cli(["--url", services.get_urlbase(), "show", f"{path}[1:3]"]) + lines = [line for line in out.strip().splitlines() if line.startswith("(")] + assert lines == ["(1, 'v1')", "(2, 'v2')"] + + +# --------------------------------------------------------------------------- +# Web preview: htmx_path_view/htmx_path_info render .b2z without error +# --------------------------------------------------------------------------- + + +def test_htmx_path_info_renders(fill_ctable_public, client): + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" + resp = httpx.get(f"{client.urlbase}/htmx/path-info/{path}") + resp.raise_for_status() + assert "Display" in resp.text + assert "Meta" in resp.text + assert "nrows" in resp.text + + +def test_htmx_path_view_no_filter_sort(fill_ctable_public, client): + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" + resp = httpx.post(f"{client.urlbase}/htmx/path-view/{path}") + resp.raise_for_status() + text = resp.text + # Fields selector present, Filter/Sort-by absent (filterable=False for CTable) + assert "Fields" in text + assert "Sort by" not in text + assert 'placeholder="Filter"' not in text + assert "v0" in text + assert "v2" in text + + +# --------------------------------------------------------------------------- +# Regression: columns whose name is not a valid Python identifier (namedtuple +# renames them under the hood, e.g. via CTable.from_arrow with a dotted name) +# --------------------------------------------------------------------------- + + +def test_ctable_row_non_identifier_column_access(): + """row[name] must be used instead of row._asdict()[name]: namedtuple(rename=True) + replaces non-identifier field names (e.g. "trip.sec") with "_N" internally.""" + row_cls = ct._make_namedtuple_row_type(("id", "trip.sec", "name")) + row = row_cls(1, 100, "a") + assert row["trip.sec"] == 100 + assert "trip.sec" not in row._asdict() + with pytest.raises(KeyError): + row._asdict()["trip.sec"] + + +@pytest.fixture +def fill_dotted_ctable_public(client): + pa = pytest.importorskip("pyarrow") + + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "dotted_src.b2z" + schema = pa.schema([("id", pa.int32()), ("trip.sec", pa.int32())]) + batch = pa.record_batch([pa.array([1, 2]), pa.array([100, 200])], schema=schema) + t = blosc2.CTable.from_arrow(schema, [batch], urlpath=str(table_path), mode="w") + t.close() + + dest_dir = pathlib.Path(TEST_STATE_DIR) / "server/public" + dest_dir.mkdir(parents=True, exist_ok=True) + fname = "dotted.b2z" + (dest_dir / fname).write_bytes(table_path.read_bytes()) + return fname, client.get(TEST_CATERVA2_ROOT) + + +def test_htmx_path_view_dotted_column(fill_dotted_ctable_public, client): + fname, root = fill_dotted_ctable_public + path = f"{root.name}/{fname}" + resp = httpx.post(f"{client.urlbase}/htmx/path-view/{path}") + resp.raise_for_status() + assert "trip.sec" in resp.text + assert "100" in resp.text + + +@pytest.fixture +def fill_nested_ctable_public(client): + """A struct column ('trip') whose leaves schema_dict() reports as dotted + paths ("trip.sec") that don't exist as top-level row fields at all: the + row only exposes 'trip' itself (a nested dict).""" + pa = pytest.importorskip("pyarrow") + + tmp = pathlib.Path(tempfile.mkdtemp()) + table_path = tmp / "nested_src.b2z" + trip_type = pa.struct([("sec", pa.float32()), ("km", pa.float32())]) + schema = pa.schema([("id", pa.int32()), ("trip", trip_type)]) + batch = pa.record_batch( + [ + pa.array([1, 2]), + pa.array([{"sec": 10.0, "km": 1.0}, {"sec": 20.0, "km": 2.0}], type=trip_type), + ], + schema=schema, + ) + t = blosc2.CTable.from_arrow(schema, [batch], urlpath=str(table_path), mode="w") + t.close() + + dest_dir = pathlib.Path(TEST_STATE_DIR) / "server/public" + dest_dir.mkdir(parents=True, exist_ok=True) + fname = "nested.b2z" + (dest_dir / fname).write_bytes(table_path.read_bytes()) + return fname, client.get(TEST_CATERVA2_ROOT) + + +def test_htmx_path_view_nested_struct_column(fill_nested_ctable_public, client): + """Regression: schema_dict() flattens struct leaves as "trip.sec", but the + row itself only has a top-level "trip" dict field; row["trip.sec"] raises + KeyError and must fall back to walking row["trip"]["sec"].""" + fname, root = fill_nested_ctable_public + path = f"{root.name}/{fname}" + resp = httpx.post(f"{client.urlbase}/htmx/path-view/{path}") + resp.raise_for_status() + assert "trip.sec" in resp.text + assert "10.0" in resp.text + assert "20.0" in resp.text From 93d9e81b62dbff9765a16ff2da75458cfaecfe21 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 13:33:09 +0200 Subject: [PATCH 04/23] Harden CTable (.b2z) row slicing, previews, and CLI edge cases Follow-up fixes from review of the CTable support work: - client: bound Table.rows() default to [0:50) instead of the whole table, so table.rows() no longer silently fetches every row of a large table (pass stop=self.nrows for all rows). - server: fix /api/fetch CTable slice resolution to use `is None` instead of truthiness, so table[0:0] returns an empty result rather than the whole table; also normalize negative indices and clamp start/stop to [0, nrows]. - cli: coerce numpy scalars, bytes, and arrays in `show --json` via a json default, matching the web preview's cell handling. - server: return a clean htmx error (not an uncaught AssertionError) when a filter/sort is requested on a dataset type that does not support it (e.g. a .b2z). - server: drop a stray comment token in the CTable fetch branch. --- caterva2/client.py | 9 ++++++--- caterva2/clients/cli.py | 13 ++++++++++++- caterva2/services/server.py | 30 +++++++++++++++++------------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/caterva2/client.py b/caterva2/client.py index 9db95c46..6d12718c 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -771,9 +771,12 @@ def __getitem__(self, key): """Shortcut: slice as a blosc2.CTable.""" return self.slice(key) - def rows(self, start=0, stop=None): - """Get rows [start, stop) as a list of tuples.""" - stop = self.nrows if stop is None else stop + def rows(self, start=0, stop=50): + """Get rows [start, stop) as a list of tuples. + + The default window is bounded (first 50 rows) to avoid accidentally + fetching a whole large table; pass ``stop=self.nrows`` for everything. + """ table = self.slice(slice(start, stop)) return [tuple(row) for row in table] diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index e5ba8ed6..8fa5f852 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -252,6 +252,17 @@ def _filter_names(fl): print(" filters: None") +def _json_default(o): + """Make CTable cell values JSON-serializable (numpy scalars, bytes, arrays).""" + if isinstance(o, bytes): + return o.decode(errors="replace") + if isinstance(o, np.generic): + return o.item() + if isinstance(o, np.ndarray): + return o.tolist() + return str(o) + + def _parse_row_slice(slice_, nrows): if not slice_: return 0, nrows @@ -274,7 +285,7 @@ def cmd_show(client, args, url): start, stop = _parse_row_slice(slice_, table.nrows) rows = table.rows(start, stop) if getattr(args, "json", False): - print(json.dumps(rows)) + print(json.dumps(rows, default=_json_default)) else: for row in rows: print(row) diff --git a/caterva2/services/server.py b/caterva2/services/server.py index ec940641..93a41164 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -582,21 +582,23 @@ async def fetch_data( return FileResponse(abspath, filename=abspath.name, media_type="application/octet-stream") if isinstance(array, blosc2.CTable): - # ponytail: slice_ is a single slice/int/tuple; extract row start/stop - if isinstance(slice_, slice): - row_start, row_stop = slice_.start or 0, slice_.stop or array.nrows - elif isinstance(slice_, int): - row_start, row_stop = slice_, slice_ + 1 - elif isinstance(slice_, tuple) and len(slice_) > 0: - sl0 = slice_[0] - if isinstance(sl0, slice): - row_start, row_stop = sl0.start or 0, sl0.stop or array.nrows - elif isinstance(sl0, int): - row_start, row_stop = sl0, sl0 + 1 - else: - row_start, row_stop = 0, array.nrows + # slice_ is a single slice/int/tuple; extract row start/stop. + # Use `is None` (not truthiness) so that stop == 0 stays 0. + sl0 = slice_[0] if isinstance(slice_, tuple) and len(slice_) > 0 else slice_ + if isinstance(sl0, slice): + row_start = 0 if sl0.start is None else sl0.start + row_stop = array.nrows if sl0.stop is None else sl0.stop + elif isinstance(sl0, int): + row_start, row_stop = sl0, sl0 + 1 else: row_start, row_stop = 0, array.nrows + # Normalize negatives and clamp to [0, nrows]. + if row_start < 0: + row_start += array.nrows + if row_stop < 0: + row_stop += array.nrows + row_start = max(0, min(row_start, array.nrows)) + row_stop = max(row_start, min(row_stop, array.nrows)) view = array.slice(row_start, row_stop) data = view.to_cframe() elif isinstance(array, hdf5.HDF5Proxy): @@ -1881,6 +1883,8 @@ async def htmx_path_view( try: mtime = abspath.stat().st_mtime arr, idx = get_filtered_array(abspath, path, filter, sortby, mtime) + except AssertionError: + return htmx_error(request, "Filtering/sorting is not supported for this dataset type.") except TypeError as exc: return htmx_error(request, f"Error in filter: {exc}") except NameError as exc: From ebd52a82b92af94c9e409137b4f175d6b31ec4fb Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 13:37:28 +0200 Subject: [PATCH 05/23] Much cleaner error message for cat2-client on invalid commands --- caterva2/clients/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index 8fa5f852..7833b08d 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -496,7 +496,7 @@ def main(): # Make --json a global flag so it applies to all commands that support JSON output parser.add_argument("--json", action="store_true", help="Output JSON when supported by the command") - subparsers = parser.add_subparsers(required=True) + subparsers = parser.add_subparsers(required=True, metavar="command", dest="command") # roots help = "List all the available roots." From 86e850a03a04db433075459535124e770098a053 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 13:45:38 +0200 Subject: [PATCH 06/23] Much cleaner error message for cat2-client on invalid commands --- plans/ctable-support-orig-gpt5.5.md | 808 +++++++++++++++++ plans/ctable-support-tasks.md | 161 ++++ plans/ctable-support.md | 1304 +++++++++++++++++++++++++++ 3 files changed, 2273 insertions(+) create mode 100644 plans/ctable-support-orig-gpt5.5.md create mode 100644 plans/ctable-support-tasks.md create mode 100644 plans/ctable-support.md diff --git a/plans/ctable-support-orig-gpt5.5.md b/plans/ctable-support-orig-gpt5.5.md new file mode 100644 index 00000000..7bc904a7 --- /dev/null +++ b/plans/ctable-support-orig-gpt5.5.md @@ -0,0 +1,808 @@ +# Caterva2 CTable support plan + +## Goal + +Add first-class support for Blosc2 heterogeneous tables (`blosc2.CTable`), mainly compact single-file `.b2z` files, alongside the existing `NDArray`/`.b2nd` support. + +The first version should make tables discoverable, downloadable, inspectable, and previewable through the REST API, Python client, CLI, and web UI. + +## Non-goals for the first pass + +Keep the first implementation boring and small. Do **not** build a dataframe service yet. + +Skip initially: + +- `.b2d` directory-backed tables +- table append/edit/delete over Caterva2 +- joins/groupby/describe/cov server APIs +- Arrow/Parquet/CSV transport APIs +- lazy expressions involving `CTable` +- treating `CTable` as a `blosc2.Operand` +- full query language/UI +- server-side pagination cache beyond simple `start`/`stop` + +Add these only after a real use case needs them. + +## Current assumptions in Caterva2 + +Caterva2 currently mostly assumes Blosc2 datasets are homogeneous arrays: + +- `.b2nd`: `blosc2.NDArray` +- `.b2frame`: `blosc2.NDArray` or `SChunk`-style frame +- `.b2`: compressed regular file / `SChunk` + +Important places with array assumptions: + +- `caterva2/services/srv_utils.py` + - `read_metadata()` only accepts `.b2frame`, `.b2nd`, `.b2` + - metadata model assumes `shape`, `chunks`, `blocks`, `dtype`, `schunk` + +- `caterva2/services/server.py` + - suffix checks in `get_abspath()`, `/api/fetch`, upload, htmx upload/download paths + - `open_b2()` assumes objects have either `schunk` or `vlmeta`, then later sets `cparams/dparams` + - `/api/fetch` returns Blosc2 binary frames and decodes naturally as arrays/schunks + - web preview path assumes `arr.shape` + +- `caterva2/models.py` + - no table metadata model + +- `caterva2/client.py` + - `Root.__getitem__()` returns `Dataset` only for `.b2nd`/`.b2frame` + - `Dataset` assumes `shape`, `dtype`, `chunks`, `blocks` + - `Client._fetch_data()` assumes binary cframe response + +- `caterva2/clients/cli.py` + - `show` currently calls `client.fetch()` + - `info` formatting assumes array/schunk metadata + +## Design principles + +1. **CTable is a sibling data kind, not an NDArray variant.** + - Use `nrows`, `ncols`, `columns`, `schema`. + - Do not invent fake `shape`/`dtype` for API metadata. + +2. **Do not overload `/api/fetch` with JSON.** + - `/api/fetch` currently means Blosc2 binary frame data. + - Keep it for arrays/schunks. + - Add a separate table rows endpoint for JSON previews. + +3. **Start with compact `.b2z`.** + - It is a single file and fits Caterva2's current file model. + - `.b2d` can come later when directory-backed datasets are needed. + +4. **Centralize suffix logic.** + - Avoid adding `.b2z` manually in many places without a shared constant. + +5. **Small preview first.** + - JSON rows for `start`/`stop` and optional columns are enough for UI, CLI, and Python client. + - Large exports can use `/api/download` first. + +## Proposed suffix constants + +Add shared constants, preferably in `caterva2/services/srv_utils.py` or a small shared module if client/server both need them. + +Server-side minimum: + +```python +BLOSC2_ARRAY_SUFFIXES = {".b2nd", ".b2frame"} +BLOSC2_TABLE_SUFFIXES = {".b2z"} +BLOSC2_FRAME_SUFFIXES = {".b2"} +BLOSC2_NATIVE_SUFFIXES = ( + BLOSC2_ARRAY_SUFFIXES | BLOSC2_TABLE_SUFFIXES | BLOSC2_FRAME_SUFFIXES +) +HDF5_SUFFIXES = {".h5", ".hdf5"} +``` + +Use these in: + +- `read_metadata()` suffix assertions +- `get_abspath()` native-file detection +- upload/load-from-url native-file detection +- htmx upload compression decisions +- download handling +- path-list handling for compressed regular files + +Client-side can use a tiny local constant: + +```python +DATASET_SUFFIXES = (".b2nd", ".b2frame") +TABLE_SUFFIXES = (".b2z",) +``` + +No need to over-share constants across package boundaries unless already easy. + +## Metadata model + +Add a new Pydantic model in `caterva2/models.py`: + +```python +class CTableMetadata(pydantic.BaseModel): + kind: str = "ctable" + nrows: int + ncols: int + chunks: tuple + blocks: tuple + schema: dict + columns: list[str] + nbytes: int + cbytes: int + cratio: float + vlmeta: dict = {} + mtime: datetime.datetime | None +``` + +Notes: + +- `schema` should come from `table.schema_dict()`. +- `columns` should be derived from the schema for convenience. +- `vlmeta` should use `table.vlmeta`. +- `kind` lets clients distinguish tables without relying on missing `shape`. + +Possible future fields: + +- `indexes` +- `computed_columns` +- per-column compression stats +- per-column logical/physical dtype info + +Skip them initially unless they are already trivial and stable in Blosc2. + +## `read_metadata()` changes + +In `caterva2/services/srv_utils.py`: + +1. Allow `.b2z` in the suffix check. +2. After `obj = blosc2.open(path)`, detect `blosc2.CTable`. +3. Return `models.CTableMetadata`. + +Sketch: + +```python +if path.suffix not in BLOSC2_NATIVE_SUFFIXES: + ... + +obj = blosc2.open(path) +... +if isinstance(obj, blosc2.CTable): + schema = obj.schema_dict() + return models.CTableMetadata( + nrows=obj.nrows, + ncols=obj.ncols, + chunks=obj.chunks, + blocks=obj.blocks, + schema=schema, + columns=[col["name"] for col in schema.get("columns", [])], + nbytes=obj.nbytes, + cbytes=obj.cbytes, + cratio=obj.cratio, + vlmeta=obj.vlmeta, + mtime=mtime, + ) +``` + +Keep the existing `NDArray`, `SChunk`, `LazyArray` branches unchanged. + +## `open_b2()` changes + +In `caterva2/services/server.py`, `open_b2()` currently opens any Blosc2 object and then eventually tunes `container.cparams.nthreads` and `container.dparams.nthreads`. + +`CTable` does not expose table-level `cparams/dparams` like `NDArray`. + +Add an early branch after `blosc2.open()` and special-file detection setup: + +```python +container = blosc2.open(abspath) + +if isinstance(container, blosc2.CTable): + return container +``` + +This should happen before trying to set `container.cparams` or `container.dparams`. + +If `vlmeta` access is needed before this branch, keep it safe: + +```python +vlmeta = ( + container.schunk.vlmeta + if hasattr(container, "schunk") + else getattr(container, "vlmeta", {}) +) +``` + +## Path resolution and upload/download + +### `get_abspath()` + +Currently regular files are auto-compressed to `.b2` unless the suffix is native. + +Change native suffix check from: + +``` +if filepath.suffix not in {".b2frame", ".b2nd", ".h5"}: +``` + +to include `.b2z` and `.hdf5` via constants: + +``` +if filepath.suffix not in BLOSC2_ARRAY_SUFFIXES | BLOSC2_TABLE_SUFFIXES | HDF5_SUFFIXES: +``` + +This prevents uploaded/existing `.b2z` files from being wrapped into `.b2`. + +### Upload APIs + +Include `.b2z` wherever native Blosc2 files should be stored as-is: + +- `/api/upload/{path:path}` +- `/api/load_from_url/{path:path}` +- `/htmx/upload/{name}` + +Existing logic: + +```python +if abspath.suffix not in {".b2", ".b2frame", ".b2nd"}: + schunk = blosc2.SChunk(data=data) +``` + +Should become: + +```python +if abspath.suffix not in BLOSC2_NATIVE_SUFFIXES: + schunk = blosc2.SChunk(data=data) +``` + +And final write-as-cframe checks should include `.b2z`, `.h5`, `.hdf5` as native. + +### Download API + +`/api/download/{path:path}` should work for `.b2z` without special conversion. + +`get_file_content()` should return stored bytes for `.b2z`. Do not try `to_cframe()`. + +If current code opens `.b2nd`/`.b2frame` specially for HDF5Proxy, leave `.b2z` on the normal file path. + +## Table rows REST API + +Add a new endpoint: + +```text +GET /api/table/{path:path}?start=0&stop=50&columns=a,b +``` + +Response: + +```json +{ + "kind": "ctable", + "start": 0, + "stop": 50, + "nrows": 1000000, + "columns": ["a", "b"], + "rows": [ + {"a": 1, "b": "foo"}, + {"a": 2, "b": "bar"} + ] +} +``` + +Parameters: + +- `start: int = 0` +- `stop: int = 50` +- `columns: str | None = None` comma-separated + +Validation: + +- reject non-CTable with 400 +- clamp `start >= 0` +- clamp `stop <= table.nrows` +- enforce `stop >= start` +- optionally cap page size, e.g. `max_rows = 1000` + +Sketch: + +```python +@app.get("/api/table/{path:path}") +async def get_table_rows( + path: pathlib.Path, + start: int = 0, + stop: int = 50, + columns: str | None = None, + user: db.User = Depends(optional_user), +): + table = open_b2(get_abspath(path, user), path) + if not isinstance(table, blosc2.CTable): + srv_utils.raise_bad_request("Not a CTable") + + max_rows = 1000 + start = max(start, 0) + stop = min(max(stop, start), table.nrows, start + max_rows) + + schema = table.schema_dict() + all_columns = [col["name"] for col in schema.get("columns", [])] + selected = [c for c in columns.split(",") if c] if columns else all_columns + unknown = sorted(set(selected) - set(all_columns)) + if unknown: + srv_utils.raise_bad_request(f"Unknown columns: {unknown}") + + view = table[selected] if selected != all_columns else table + rows = [_jsonable_row(row) for row in view.slice(start, stop)] + + return { + "kind": "ctable", + "start": start, + "stop": stop, + "nrows": table.nrows, + "columns": selected, + "rows": rows, + } +``` + +Add small helpers: + +```python +def _jsonable(value): + if isinstance(value, np.generic): + return value.item() + if isinstance(value, np.ndarray): + return value.tolist() + if isinstance(value, bytes): + return value.decode(errors="replace") + return value + + +def _jsonable_row(row): + if hasattr(row, "_asdict"): + row = row._asdict() + return {k: _jsonable(v) for k, v in dict(row).items()} +``` + +If `dict(row)` is not reliable for CTable row objects, use `_asdict()` first. + +## `/api/fetch` behavior + +Do not support `.b2z` in `/api/fetch` initially. + +Change the error message to be accurate: + +```python +if abspath.suffix not in BLOSC2_ARRAY_SUFFIXES: + srv_utils.raise_bad_request( + "The fetch API only supports array datasets (.b2nd and .b2frame); " + "use /api/table for CTable row previews or /api/download for files" + ) +``` + +Reason: + +- `/api/fetch` returns binary Blosc2 frames. +- CTable does not currently fit the `ndarray_from_cframe`/`schunk_from_cframe` client path. +- JSON rows belong in a separate endpoint. + +## Python client API + +Add `Table` class in `caterva2/client.py`: + +```python +class Table(File): + def __repr__(self): + return f"" + + @property + def nrows(self): + return self.meta["nrows"] + + @property + def ncols(self): + return self.meta["ncols"] + + @property + def columns(self): + return self.meta["columns"] + + @property + def schema(self): + return self.meta["schema"] + + def rows(self, start=0, stop=50, columns=None): + return self.client.get_table_rows( + self.path, start=start, stop=stop, columns=columns + ) + + def head(self, n=10, columns=None): + return self.rows(0, n, columns=columns) +``` + +Change `Root.__getitem__()`: + +```python +if path.endswith((".b2nd", ".b2frame")): + return Dataset(self, path) +if path.endswith(".b2z"): + return Table(self, path) +return File(self, path) +``` + +Change `Client.get()` type check: + +```python +if isinstance(path, (File, Dataset, Table, Root)): + return path +``` + +Add `Client.get_table_rows()`: + +```python +def get_table_rows(self, path, start=0, stop=50, columns=None): + if isinstance(path, Table): + path = path.path + _, path = _format_paths(self.urlbase, path) + params = {"start": start, "stop": stop} + if columns: + params["columns"] = ( + ",".join(columns) if not isinstance(columns, str) else columns + ) + return self._get( + f"{self.urlbase}/api/table/{path}", + params=params, + auth_cookie=self.cookie, + timeout=self.timeout, + ) +``` + +Return the full response dict first. If callers want rows only: + +```python +table.head()["rows"] +``` + +A later ergonomic wrapper can return just rows, but the full response is more useful and avoids adding options. + +## CLI changes + +### `info` + +If metadata contains `kind == "ctable"`, print table-specific fields: + +```text +kind : ctable +nrows : 1000000 +ncols : 12 +chunks : (1048576,) +blocks : (32768,) +nbytes : 123 MB +cbytes : 12 MB +ratio : 10.2x +columns: a, b, c, ... +mtime : ... +``` + +For `--json`, no special work: dump metadata as returned. + +### `show` + +For `.b2z`, do not call `client.fetch()`. + +Minimal behavior: + +```sh +cat2-client show path/table.b2z +``` + +prints first 50 rows. + +Optional first-pass slice support: + +```sh +cat2-client show path/table.b2z[0:20] +``` + +maps to `start=0`, `stop=20`. + +Optional columns could come later via `--columns a,b`. Skip unless asked. + +Implementation: + +- detect `.b2z` path +- parse single row slice only +- call `client.get_table_rows()` +- print rows as a simple table or JSON + +Lazy first text output can use standard formatting: + +```python +rows = data["rows"] +print(json.dumps(rows, indent=2)) +``` + +Pretty tables can come later. + +## Web UI changes + +### `htmx_path_info()` + +Currently array display tab is added when `hasattr(meta, "shape")`. + +Add table display tab when `kind == "ctable"` or `hasattr(meta, "nrows")`: + +```python +if getattr(meta, "kind", None) == "ctable": + context["data_url"] = make_url(request, "htmx_path_view", path=path) + context["shape"] = (meta.nrows,) + tabs.append( + { + "name": "data", + "label": "Display", + "include": "includes/info_data.html", + } + ) +``` + +### `htmx_path_view()` + +Add an early CTable branch after `arr = open_b2(...)`: + +``` +if isinstance(arr, blosc2.CTable): + schema = arr.schema_dict() + cols = [col["name"] for col in schema.get("columns", [])] + fields = fields or cols[:5] + index = (0,) if index is None else tuple(index) + sizes = sizes or [10] + start = index[0] + stop = min(start + sizes[0], arr.nrows) + view = arr[fields] if fields else arr + rows = [fields] + [[row._asdict()[f] for f in fields] for row in view.slice(start, stop)] + ... render info_view.html ... +``` + +Reusing `info_view.html` keeps the diff small. + +Set context keys similarly to array preview: + +```python +context = { + "view_url": make_url(request, "htmx_path_view", path=path), + "inputs": [ + { + "start": start, + "start_max": max(arr.nrows - size, 0), + "size": size, + "size_max": arr.nrows, + "with_size": True, + } + ], + "rows": rows, + "cols": cols, + "fields": fields, + "filter": "", + "sortby": "", + "shape": (arr.nrows,), + "tags": list(range(start, stop)), +} +``` + +Filtering/sorting can come later. CTable has `where()`, `sort_by()`, `sorted_slice()`, but adding it now risks coupling Caterva2 to a new query surface. + +## Tests + +Add one small CTable fixture. + +Example creation helper: + +```python +from dataclasses import dataclass +import blosc2 + + +@dataclass +class Row: + id: int = blosc2.field(blosc2.int32()) + name: str = blosc2.field(blosc2.string(max_length=20)) + + +def make_ctable(path): + table = blosc2.CTable(Row, urlpath=str(path), mode="w", compact=True) + table.append((1, "alice")) + table.append((2, "bob")) + table.append((3, "carol")) + table.close() +``` + +Tests: + +1. `read_metadata()` on `.b2z` + - `kind == "ctable"` + - `nrows == 3` + - `columns == ["id", "name"]` + +2. `/api/info/...table.b2z` + - same assertions through HTTP + +3. `/api/download/...table.b2z` + - write response to temp file + - `blosc2.open(downloaded_path)` returns `blosc2.CTable` + - `len(table) == 3` + +4. `/api/table/...table.b2z?start=1&stop=3` + - response rows are bob/carol + - columns are present + +5. Python client + - `root["table.b2z"]` is `Table` + - `table.nrows == 3` + - `table.head(2)["rows"]` has two rows + +6. CLI smoke test + - `cat2-client info table.b2z --json` contains `kind: ctable` + - `cat2-client show table.b2z --json` returns first rows + +Keep tests small. No fixtures for huge data, no performance suite. + +## Edge cases + +### NumPy scalar JSON encoding + +CTable rows can contain NumPy scalar types. Convert with `.item()`. + +### NumPy array cells + +CTable can have nested/NDArray-like cells. For JSON preview, convert arrays to lists. + +### Bytes cells + +Decode as UTF-8 with replacement for preview. For exact bytes transport, use download/export later. + +### Large pages + +Cap JSON rows, e.g. 1000 rows per request. This prevents accidental huge responses from the web UI or CLI. + +### Unknown columns + +Reject unknown requested columns with 400. Do not silently ignore. + +### `.b2d` + +Directory-backed CTable should be rejected/not recognized initially. If uploaded as archive, it is just a regular file/archive. Native directory-backed support needs separate path-list and download semantics. + +## Future extensions + +Only add these once MVP is working and there is demand: + +### Table filtering + +Use CTable native APIs: + +```python +table.where("col > 10", columns=["a", "b"]) +``` + +Possible endpoint: + +```text +GET /api/table/{path}?where=col%20%3E%2010&columns=a,b&start=0&stop=50 +``` + +Need careful expression validation/error handling. + +### Sorting + +Use: + +```python +table.sorted_slice(...) +table.sort_by(...) +``` + +Possible endpoint params: + +```text +sort=col +ascending=true +``` + +### Arrow transport + +For larger table reads, add: + +```text +GET /api/table/{path}?format=arrow +``` + +Backed by `table.to_arrow()` or `iter_arrow_batches()`. + +Only if PyArrow is already an accepted dependency/extra. Do not add as core dependency just for first support. + +### CSV export + +Possible: + +```text +GET /api/table/{path}?format=csv +``` + +Backed by `table.to_csv()`. + +### Parquet export + +Possible: + +```text +GET /api/table/{path}?format=parquet +``` + +Backed by `table.to_parquet()`. + +Needs temp files and likely PyArrow. Not MVP. + +### `.b2d` support + +Directory-backed tables need decisions: + +- how to list them as one logical dataset +- how to upload/download them +- whether to tar/zip for download +- how to protect internal files from direct mutation + +Do later. + +## Suggested implementation order + +1. Add suffix constants and `.b2z` native handling. +2. Add `CTableMetadata` and `read_metadata()` support. +3. Make `open_b2()` return `CTable` safely. +4. Ensure `/api/download` preserves `.b2z` bytes. +5. Add `/api/table/{path:path}` JSON row window endpoint. +6. Add Python client `Table` and `Client.get_table_rows()`. +7. Add web preview branch reusing existing templates. +8. Add CLI `info`/`show` support. +9. Add the small CTable tests. + +## Open questions + +- Should Python `Table.rows()` return the full response dict or just `rows`? + - Proposed first pass: full dict. + +- Should web table preview support filters immediately? + - Proposed first pass: no. + +- Should `.b2z` be included in `/api/fetch` as a full table binary download? + - Proposed first pass: no; use `/api/download`. + +- Should Caterva2 expose Arrow if Blosc2 can export Arrow? + - Proposed first pass: no; avoid dependency/API expansion. + +- Should table metadata include per-column compression stats? + - Proposed first pass: no, unless Blosc2 exposes a stable cheap dict. + +## Definition of done for MVP + +A user can: + +```python +import caterva2 as cat2 + +client = cat2.Client("https://server") +root = client.get("@public") +table = root["example.b2z"] + +print(table.nrows) +print(table.columns) +print(table.head(5)) +table.download() +``` + +And in CLI: + +```sh +cat2-client info @public/example.b2z +cat2-client show @public/example.b2z +``` + +And in the web UI: + +- click a `.b2z` +- see metadata +- see a small row/column preview + +That is enough to say Caterva2 serves Blosc2 tables. diff --git a/plans/ctable-support-tasks.md b/plans/ctable-support-tasks.md new file mode 100644 index 00000000..bfe1c2a0 --- /dev/null +++ b/plans/ctable-support-tasks.md @@ -0,0 +1,161 @@ +# Caterva2 CTable (`.b2z`) — implementation tasks + +Task list distilled from `ctable-support.md`. **Rationale, options, and design +decisions live in that document** — this file is only *what to do*, ranked by +importance. Each task lists target files, concrete changes, and a "Done when" +acceptance check. + +**Status: all tasks below are done.** No open work remains from this plan. + +Priorities: **P0** = blocking bug, ship-stopper. **P1** = core MVP correctness / +API. **P2** = user-facing preview surface. **P3** = polish / deferred. + +--- + +## Baseline already committed (do not redo) + +- blosc2: `CTable.to_cframe()` + `blosc2.ctable_from_cframe()` (exported); + lazy-view `to_cframe()` works. +- `srv_utils.py`: suffix constants incl. `BLOSC2_TABLE_SUFFIXES = {".b2z"}`, + `BLOSC2_NATIVE_SUFFIXES`. +- `models.py`: `CTableMetadata` (field name is `schema_dict`). +- `srv_utils.read_metadata()`: maps `CTable` → `CTableMetadata`. +- `server.open_b2()`: returns `CTable` early (before `cparams`/`dparams`). +- `server.get_abspath()`: `.b2z` treated as native (line ~459). +- `server` `/api/fetch`: accepts `.b2z`, serializes **slices** via `to_cframe()`. +- `client.py`: `Table` class exists; `Root.__getitem__` returns `Table` for + `.b2z`; `_fetch_data` tries `ctable_from_cframe` first. + +--- + +## P0 — Blocking + +### T1. Fix whole-table `/api/fetch` (returns raw zip, client can't decode) — DONE + +- **File:** `caterva2/services/server.py`, `/api/fetch` handler (~line 575). +- **Bug:** the whole-dataset short-circuit returned `FileResponse(abspath)`; a + whole `.b2z` is a zip (`PK\x03\x04`), not a cframe, so the client's cframe + decoder raised `RuntimeError: Could not get the schunk from the cframe`. + `table[:]` (and any slice with `stop >= nrows`) failed. +- **Fix:** added `blosc2.CTable` to the whole-dataset short-circuit exclusion, + so whole tables also serialize via `to_cframe()` like slices already did. +- **Regression test:** `test_fetch_whole_and_slice` in `test_ctable.py` + (verified it fails without the fix, passes with it). + +--- + +## P1 — Core MVP + +### T2. `_fetch_data`: dispatch decode on known kind (not trial/except) — DONE + +- **File:** `caterva2/client.py`, `Client._fetch_data`/`get_slice`. +- `get_slice` now computes `kind` (`"ctable"` vs. not) from the `Table` + instance or path suffix *before* the request, and `_fetch_data` dispatches + `ctable_from_cframe` only when `kind == "ctable"`; otherwise the existing + `ndarray_from_cframe` → `schunk_from_cframe` fallback. No more + try/except-driven format sniffing. + +### T3. Client class hierarchy: `File → Dataset → {Array, Table}` — DONE + +- **File:** `caterva2/client.py`, `caterva2/__init__.py`. +- `Dataset(File)` is now a lean shared base (no `blosc2.Operand`). + `Array(Dataset, blosc2.Operand)` holds `dtype`/`shape`/`ndim`/`chunks`/ + `blocks`/`append` and the dtype-aware `__getitem__` (moved out of `File`, + which keeps a minimal generic `__getitem__`/`slice()` for plain files). + `Table(Dataset)`. `Root.__getitem__` returns `Array` for `.b2nd`/`.b2frame`, + `Table` for `.b2z`. Simplified redundant `isinstance` tuples in + `Client.get`/`get_info`/`get_slice`/`get_chunk`/`remove`. Updated repr + doctests. `Array`/`Table` exported from `caterva2/__init__.py`. + +### T4. `Table` client behavior — DONE + +- **File:** `caterva2/client.py`, `Table(Dataset)`. +- Added `nrows`, `ncols`, `columns`, `schema` (reads `meta["schema_dict"]`), + `rows(start, stop)`, `head(n)`; existing `slice()`/`__getitem__` return a + `blosc2.CTable`. + +### T5. `.b2z` upload/download suffix handling (+ use constants) — DONE + +- **File:** `caterva2/services/server.py`. +- Replaced hardcoded suffix sets with `srv_utils.BLOSC2_NATIVE_SUFFIXES` + (now includes `.b2z`) in `/api/upload`, `/api/load_from_url`, + `/htmx/upload` (both archive-extraction and single-file branches), and the + quick-search path-guessing code. `get_file_content`/`/api/download` needed + no change — `.b2z` already fell through to the raw-bytes path correctly. + Verified byte-identical upload→download round-trip. + +--- + +## P2 — Preview surface + +### T6. Web UI preview (reuse `info_view.html`) — DONE + +- **Files:** `caterva2/services/server.py` (`htmx_path_info`, + `htmx_path_view`); `info_view.html`; `includes/info_metadata.html`. +- `htmx_path_info`: Display tab added when `meta.kind == "ctable"`, + `shape = (nrows,)`. +- `htmx_path_view`: dedicated `CTable` branch (before `shape = arr.shape`) + builds `cols`/`rows`/`inputs`/`tags` from `schema_dict()`/`nrows`, using + keyed `row[name]` access (see T-blosc2 note below) and coercing + bytes/numpy scalars for HTML. +- Added a `filterable` context flag (`False` for `CTable`, `True` for + arrays); `info_view.html` guards the Filter box + Sort-by select with + `{% if filterable %}` (Fields selector stays available for both). +- Fixed a **pre-existing crash**: `includes/info_metadata.html`'s Meta tab + assumed any non-`schunk` metadata with `nbytes` also had `cparams` (true + for plain `.b2frame` `SChunk`, false for `CTableMetadata`), so viewing a + `.b2z` file's metadata tab threw `UndefinedError`. Added a dedicated + `ctable` branch. + +### T7. CLI `info` / `show` for `.b2z` — DONE + +- **File:** `caterva2/clients/cli.py`. +- `info`: prints table-shaped fields (`nrows`, `ncols`, `chunks`, `blocks`, + `columns`, `nbytes`/`cbytes`/`ratio`, `mtime`) when `kind == "ctable"` + instead of crashing on `cparams.get(...)` against `None`; `--json` + unchanged. +- `show`: for `.b2z`, parses the optional row-slice syntax + (`table.b2z[start:stop]`) client-side and prints `table.rows(start, stop)` + via the `Table` client class — no `client.fetch()`/`/api/table` call. + +--- + +## P3 — Tests & fixtures + +### T8. CTable fixture + tests — DONE + +- **File:** `caterva2/tests/test_ctable.py` (22 tests). +- All 7 items from the original list are covered: `read_metadata()` unit + tests, HTTP `/api/info`, `/api/download` round-trip, `/api/fetch` whole + + slice (T1 regression guard), Python client `Table` class behavior, CLI + `info`/`show` (incl. row-slice syntax), and web preview + (`htmx_path_info`/`htmx_path_view` render without error, Filter/Sort-by + absent, Fields present). +- Also added regression tests for a bug found during manual testing (see + note below): non-identifier/nested CTable column names in the web preview. + +--- + +## Post-plan fix: nested/non-identifier column names (found during manual QA) + +Manually uploading a real-world table (`chicago-taxi-indexed.b2z`, with +struct columns like `trip.sec`, `trip.begin.lon`) crashed `htmx_path_view` +twice in a row: + +1. `row._asdict()[f]` used the *renamed* namedtuple keys (blosc2's + `CTableRow` is built with `namedtuple(..., rename=True)`, so a column + named `trip.sec` becomes `_1` internally) instead of blosc2's own + name-safe `row[name]`/`row.as_dict()`. Fixed in caterva2 by switching to + `row[f]`. +2. `schema_dict()` flattens nested struct columns into dotted leaf paths + (`trip.sec` for a `trip` struct column with a `sec` leaf) that don't + exist as row fields at all — the row only exposes the top-level `trip` + dict. `row["trip.sec"]` raised `KeyError`. + +Root-caused (2) as a genuine blosc2 API gap — `col_names`/`schema_dict()` +advertised names that `CTableRow.__getitem__` couldn't resolve — and fixed +it **upstream in `python-blosc2`** (`CTableRow.__getitem__` now walks dotted +paths into nested structs via `split_field_path`, with tests in +`tests/ctable/test_nested_access_storage.py`). Once that landed, the +caterva2-side workaround was removed and `htmx_path_view` went back to a +plain `row[f]` for every field, relying on blosc2's native support. diff --git a/plans/ctable-support.md b/plans/ctable-support.md new file mode 100644 index 00000000..08bbfb5d --- /dev/null +++ b/plans/ctable-support.md @@ -0,0 +1,1304 @@ +# Caterva2 CTable support plan + +> **Status (2026-07-01): implemented, committed, and hardened.** All MVP work in +> this plan is done — see "Implementation status" near the end for the as-built +> summary (including deviations from the sketches below and post-review fixes). +> The design sections that follow are kept as the rationale/record; where a +> sketch and the shipped code differ, the code wins. + +## Revision 2026-07-01 + +Four decisions supersede the original text below; where a later section still +reads the old way, this block wins: + +1. **Whole-table `/api/fetch` must not short-circuit to `FileResponse`.** A whole + `.b2nd` file *is* a cframe, so the existing `FileResponse` short-circuit + decodes fine on the client. A whole `.b2z` file is a **zip** (`PK\x03\x04`), + which is *not* a cframe — the client's cframe decoder raises + `RuntimeError: Could not get the schunk from the cframe`. Reproduced against + the committed code: `table[:]` (and any `slice` with `stop >= nrows`) fails. + Fix: exclude `CTable` from the whole short-circuit so whole tables also go + through `to_cframe()`. See "`/api/fetch` behavior". + +2. **Client decode dispatches on known kind, not trial-and-except.** The + committed `_fetch_data` attempts `ctable_from_cframe` first and catches + `ValueError`, so every (common) array fetch pays a failed CTable parse, and + the chain is coupled to which exact exception each decoder raises (that + coupling is why the bug in #1 propagates uncaught). The client already knows + the kind (`Table` vs `Dataset`, `meta["kind"]`); dispatch on it. + +3. **Remove the JSON `/api/table` row-window endpoint entirely — not deferred, + dropped.** It is a lighter-weight *duplicate of the slice fetch* with no + consumer: the Python client already yields rows off the cframe + (`[tuple(row) for row in data[:]]`), so CLI `show` reuses it; the web UI + renders **server-side** in `htmx_path_view` from the live `CTable`; and + JupyterLite is Pyodide, so it has blosc2 and decodes cframes directly. The + only consumers it would serve — a non-Pyodide JS client or `curl` eyeballing + — are hypothetical, and if one appears the endpoint is ~20 trivial lines. A + specced-but-deferred endpoint is an attractive nuisance (invites a second + row-preview path), so it is removed with a short "rejected alternative" note + rather than parked. + +4. **Transport is orthogonal to operation; table ops ride two homes, neither is + `/api/table`.** `where`/`sort`/`group_by` all *produce a `CTable`*, which the + cframe path already serializes — so operations need a way to be *expressed in + the request*, not a JSON transport. Split by what the op does to the schema: + - **Schema-preserving** (`where`, `sort`) → `filter=`/`sortby=` params on + `/api/fetch`, cframe out (mirrors the array `get_filtered_array` path). + Near-term. Enabled now that the lazy-view `to_cframe()` copy() bug is fixed. + - **Schema-changing** (`group_by`, `aggregate`, `join`, `describe`) → a + future `POST /api/query/{path}` with a structured query body, cframe + response. Compute-not-fetch; a genuinely different resource. Post-MVP, + only when an analytical use case lands. + +5. **Client class hierarchy: `Table` is-a `Dataset`, not a sibling.** Reparent + the client leaf classes to `File → Dataset → {Array, Table}` (details in + "Client class hierarchy"). `Table(File)` as a sibling of the array class was + a workaround to keep `blosc2.Operand` off tables; the correct fix is to push + `Operand` **down onto `Array` only**, which gives symmetry (`Table` is-a + `Dataset`) *and* correctness (`Table` is-not-an `Operand`). Also relocates + `File`'s array-flavored `__getitem__` (it already references `self.dtype`) + down to `Array`, leaving `File` generic. + +## Goal + +Add first-class support for Blosc2 heterogeneous tables (`blosc2.CTable`), mainly compact single-file `.b2z` files, alongside the existing `NDArray`/`.b2nd` support. + +The first version should make tables discoverable, downloadable, inspectable, and previewable through the REST API, Python client, CLI, and web UI. + +## Non-goals for the first pass + +Keep the first implementation boring and small. Do **not** build a dataframe service yet. + +Skip initially: + +- `.b2d` directory-backed tables +- table append/edit/delete over Caterva2 +- joins/groupby/describe/cov server APIs +- Arrow/Parquet/CSV transport APIs +- lazy expressions involving `CTable` +- treating `CTable` as a `blosc2.Operand` +- full query language/UI +- server-side pagination cache beyond simple `start`/`stop` + +Add these only after a real use case needs them. + +## Current assumptions in Caterva2 + +Caterva2 currently mostly assumes Blosc2 datasets are homogeneous arrays: + +- `.b2nd`: `blosc2.NDArray` +- `.b2frame`: `blosc2.NDArray` or `SChunk`-style frame +- `.b2`: compressed regular file / `SChunk` + +Important places with array assumptions: + +- `caterva2/services/srv_utils.py` + - `read_metadata()` only accepts `.b2frame`, `.b2nd`, `.b2` + - metadata model assumes `shape`, `chunks`, `blocks`, `dtype`, `schunk` + +- `caterva2/services/server.py` + - suffix checks in `get_abspath()`, `/api/fetch`, upload, htmx upload/download paths + - `open_b2()` assumes objects have either `schunk` or `vlmeta`, then later sets `cparams/dparams` + - `/api/fetch` returns Blosc2 binary frames and decodes naturally as arrays/schunks + - web preview path assumes `arr.shape` + +- `caterva2/models.py` + - no table metadata model + +- `caterva2/client.py` + - `Root.__getitem__()` returns `Dataset` only for `.b2nd`/`.b2frame` + - `Dataset` assumes `shape`, `dtype`, `chunks`, `blocks` + - `Client._fetch_data()` assumes binary cframe response + +- `caterva2/clients/cli.py` + - `show` currently calls `client.fetch()` + - `info` formatting assumes array/schunk metadata + +## Design principles + +1. **CTable is a sibling data kind, not an NDArray variant.** + - Use `nrows`, `ncols`, `columns`, `schema`. + - Do not invent fake `shape`/`dtype` for API metadata. + - Consistent with the client hierarchy (Revision 2026-07-01 #5): `Array` and + `Table` are *siblings* under a shared `Dataset` base — a table is a kind of + dataset, but not a kind of array. + +2. **Do not put JSON in `/api/fetch`.** + - `/api/fetch` means Blosc2 binary frame data: cframes for arrays/schunks, + and (via `CTable.to_cframe()`) cframes for tables. Slices come + back as blosc2 objects on the client, mirroring the array workflow. + - A separate JSON row-window endpoint (`/api/table`) *was* planned for + previews, but is **removed** — see Revision 2026-07-01 #3. Preview is + served off the cframe path (CLI) and server-side rendering (web UI), so no + JSON-over-HTTP endpoint is needed. Table *operations* live on `/api/fetch` + params (schema-preserving) or a future `POST /api/query` (schema-changing), + never on a JSON row endpoint — see Revision 2026-07-01 #4. + +3. **Start with compact `.b2z`.** + - It is a single file and fits Caterva2's current file model. + - `.b2d` can come later when directory-backed datasets are needed. + +4. **Centralize suffix logic.** + - Avoid adding `.b2z` manually in many places without a shared constant. + +5. **Small preview first.** + - A row window (`start`/`stop`, optional columns) is enough for UI, CLI, and + Python client. It is produced from the cframe slice on the client and by + server-side row iteration in the web UI — no dedicated JSON endpoint (see + Revision 2026-07-01 #3). + - Large exports can use `/api/download` first. + +## Proposed suffix constants + +Add shared constants, preferably in `caterva2/services/srv_utils.py` or a small shared module if client/server both need them. + +Server-side minimum: + +```python +BLOSC2_ARRAY_SUFFIXES = {".b2nd", ".b2frame"} +BLOSC2_TABLE_SUFFIXES = {".b2z"} +BLOSC2_FRAME_SUFFIXES = {".b2"} +BLOSC2_NATIVE_SUFFIXES = ( + BLOSC2_ARRAY_SUFFIXES | BLOSC2_TABLE_SUFFIXES | BLOSC2_FRAME_SUFFIXES +) +HDF5_SUFFIXES = {".h5", ".hdf5"} +``` + +Use these in: + +- `read_metadata()` suffix assertions +- `get_abspath()` native-file detection +- upload/load-from-url native-file detection +- htmx upload compression decisions +- download handling +- path-list handling for compressed regular files + +Client-side can use a tiny local constant: + +```python +DATASET_SUFFIXES = (".b2nd", ".b2frame") +TABLE_SUFFIXES = (".b2z",) +``` + +No need to over-share constants across package boundaries unless already easy. + +## Metadata model + +Add a new Pydantic model in `caterva2/models.py`: + +```python +class CTableMetadata(pydantic.BaseModel): + kind: str = "ctable" + nrows: int + ncols: int + chunks: tuple + blocks: tuple + schema: dict + columns: list[str] + nbytes: int + cbytes: int + cratio: float + vlmeta: dict = {} + mtime: datetime.datetime | None +``` + +Notes: + +- `schema` should come from `table.schema_dict()`. +- `columns` should be derived from the schema for convenience. +- `vlmeta` should use `table.vlmeta`. +- `kind` lets clients distinguish tables without relying on missing `shape`. + +Possible future fields: + +- `indexes` +- `computed_columns` +- per-column compression stats +- per-column logical/physical dtype info + +Skip them initially unless they are already trivial and stable in Blosc2. + +## `read_metadata()` changes + +In `caterva2/services/srv_utils.py`: + +1. Allow `.b2z` in the suffix check. +2. After `obj = blosc2.open(path)`, detect `blosc2.CTable`. +3. Return `models.CTableMetadata`. + +Sketch: + +```python +if path.suffix not in BLOSC2_NATIVE_SUFFIXES: + ... + +obj = blosc2.open(path) +... +if isinstance(obj, blosc2.CTable): + schema = obj.schema_dict() + return models.CTableMetadata( + nrows=obj.nrows, + ncols=obj.ncols, + chunks=obj.chunks, + blocks=obj.blocks, + schema=schema, + columns=[col["name"] for col in schema.get("columns", [])], + nbytes=obj.nbytes, + cbytes=obj.cbytes, + cratio=obj.cratio, + vlmeta=obj.vlmeta, + mtime=mtime, + ) +``` + +Keep the existing `NDArray`, `SChunk`, `LazyArray` branches unchanged. + +## `open_b2()` changes + +In `caterva2/services/server.py`, `open_b2()` currently opens any Blosc2 object and then eventually tunes `container.cparams.nthreads` and `container.dparams.nthreads`. + +`CTable` does not expose table-level `cparams/dparams` like `NDArray`. + +Add an early branch after `blosc2.open()` and special-file detection setup: + +```python +container = blosc2.open(abspath) + +if isinstance(container, blosc2.CTable): + return container +``` + +This should happen before trying to set `container.cparams` or `container.dparams`. + +If `vlmeta` access is needed before this branch, keep it safe: + +```python +vlmeta = ( + container.schunk.vlmeta + if hasattr(container, "schunk") + else getattr(container, "vlmeta", {}) +) +``` + +## Path resolution and upload/download + +### `get_abspath()` + +Currently regular files are auto-compressed to `.b2` unless the suffix is native. + +Change native suffix check from: + +``` +if filepath.suffix not in {".b2frame", ".b2nd", ".h5"}: +``` + +to include `.b2z` and `.hdf5` via constants: + +``` +if filepath.suffix not in BLOSC2_ARRAY_SUFFIXES | BLOSC2_TABLE_SUFFIXES | HDF5_SUFFIXES: +``` + +This prevents uploaded/existing `.b2z` files from being wrapped into `.b2`. + +### Upload APIs + +Include `.b2z` wherever native Blosc2 files should be stored as-is: + +- `/api/upload/{path:path}` +- `/api/load_from_url/{path:path}` +- `/htmx/upload/{name}` + +Existing logic: + +```python +if abspath.suffix not in {".b2", ".b2frame", ".b2nd"}: + schunk = blosc2.SChunk(data=data) +``` + +Should become: + +```python +if abspath.suffix not in BLOSC2_NATIVE_SUFFIXES: + schunk = blosc2.SChunk(data=data) +``` + +And final write-as-cframe checks should include `.b2z`, `.h5`, `.hdf5` as native. + +### Download API + +`/api/download/{path:path}` should work for `.b2z` without special conversion. + +`get_file_content()` should return stored bytes for `.b2z`. Do not try `to_cframe()`. + +If current code opens `.b2nd`/`.b2frame` specially for HDF5Proxy, leave `.b2z` on the normal file path. + +## Rejected alternative: a JSON `/api/table` row-window endpoint + +An earlier draft proposed `GET /api/table/{path}?start&stop&columns` returning +`{"kind": "ctable", "rows": [...]}` JSON. **Rejected — removed, not deferred** +(Revision 2026-07-01 #3). + +Why it does not earn its place: + +- It is a lighter-weight *duplicate of the slice fetch*. Every would-be MVP + consumer is already covered without it: + - CLI `show` → rows off the cframe via the Python client. + - Web UI → rows rendered server-side in `htmx_path_view` from the live + `CTable`. + - JupyterLite → Pyodide has blosc2, so it decodes cframes directly. +- The only consumers it would uniquely serve are a **non-Pyodide JS client** or + **`curl` eyeballing** — both hypothetical today. If one appears, the endpoint + is ~20 trivial lines (JSON-ify a `slice()`), and the design is easy to + re-derive — unlike the cframe transport decision below, which is subtle and + worth preserving. +- A specced-but-deferred endpoint is an attractive nuisance: it invites a second + row-preview path alongside the cframe one. Better to have one path. + +If a real external JSON consumer ever lands, add it then. Note that table +*operations* (`where`/`sort`/`group_by`) do **not** motivate this endpoint — +transport (JSON vs cframe) is orthogonal to operation, and every operation +returns a `CTable` the cframe path already serializes. See "Future extensions → +table operations" for where operations actually live. + +## Transport format for table slices (decision) + +This section records the analysis of how to ship a slice of a `CTable` over the +wire as a Blosc2 object, mirroring how `/api/fetch` ships `NDArray` slices as +`to_cframe()` bytes. It is the most consequential design choice in this plan, +so the options and reasoning are kept here in full. + +### The core problem + +A cframe serializes **one** `SChunk`. An `NDArray`/`SChunk` is one schunk, so +`to_cframe()`/`ndarray_from_cframe()` is trivial. A `CTable` is **a collection** +of blosc2 objects: one `NDArray` per column, a `_valid_rows` mask, a `_meta` +manifest, `_vlmeta`, and a schema dict. There is no single schunk to serialize, +so `CTable.to_cframe()` does not exist today. The native container is `.b2z`, a +zip archive bundling the column leaves plus an `embed.b2e` `EmbedStore` — but +`.b2z` is a **file-based** format, not bytes. + +### Options considered + +**Option A: temp-file `.b2z` roundtrip (no new blosc2 API).** + +```python +# server (slice) +view = table.slice(start, stop) +view.to_b2z("/tmp/slice.b2z", overwrite=True) +# stream /tmp/slice.b2z bytes to the HTTP response + +# client +open("/tmp/slice.b2z", "wb").write(data) +rt = blosc2.open("/tmp/slice.b2z", mode="r") +``` + +Tested — it works. Findings: + +- `to_b2z()` rejects file-like objects (`os.fspath` + `endswith(".b2z")` + + `os.path.exists`); a `BytesIO` fails with `urlpath must have a .b2z extension`. + So the server must touch disk. +- `.b2z` read mode reads column leaves **lazily, by byte offset, from a file + path** at the C level (`blosc2_ext.open(path, offset=...)`). The client temp + file must stay alive: opening a `.b2z`, deleting the file, then touching a + row raises `RuntimeError: Error while getting the lazychunk`. +- For a slice/view, `to_b2z()` falls back to the logical `save()` path and + **recompresses** the live rows; the zero-recompression `ZIP_STORED` physical + pack only fires for a whole persistent `.b2d`/`.b2z` source. + +**Pyodide viability (revisited).** Earlier this plan flagged the path-based lazy +reader as a Pyodide blocker. That was wrong: Pyodide's **MEMFS** is a real +filesystem exposed to compiled C via Emscripten file syscalls, and blosc2's +default `open()` does **not** mmap (mmap is opt-in; the WASM path only forces +`nthreads=1`, not mmap). So `blosc2.open('/tmp/x.b2z')` on MEMFS works exactly +as on disk. The temp-file path is therefore viable on desktop **and** Pyodide. + +**Footprint.** MEMFS RAM is part of the Pyodide heap budget, but this is not a +penalty: the CTable's compressed bytes live on the heap either way. Steady +state is identical for both paths — compressed slice (C bytes) on the heap plus +decompressed columns touched (D bytes). The only transient difference is a +brief 2C during ingestion (response bytes copied into MEMFS, or into a Python +buffer), identical in both. MEMFS is neither cheaper nor dearer than a Python +buffer; footprint is a wash. + +**Cleanup — the real residual difference.** With MEMFS, the temp-file path is +viable, but cleanup is **extrinsic**: the MEMFS file holds C bytes until +explicitly deleted, tied to a `Table._local_b2z` attribute and a `__del__` +finalizer on `Table`. The cframe path is **intrinsic**: the bytes live in the +`CTable` object; freeing the object frees the bytes via ordinary refcounting. + + - With a `__del__`/`weakref.finalize` that does `os.remove(self._local_b2z)`, + the temp-file path reaches effective equality on CPython (and Pyodide runs + CPython, so refcounting is deterministic there). The user cannot leak. + - The maintainer can: the invariant is "every temp `.b2z` created on behalf + of this `Table` is recorded in `_local_b2z`" — a convention the codebase + must enforce on itself. A future fetch path that does its own `mkstemp` + without routing through one `_materialize()` method leaks silently. The + cframe path has no such invariant: resource and object are the same thing, + so new fetch methods cannot break cleanup. + - To make the temp-file path bulletproof, route every materialization + through one `_materialize()` method and forbid ad-hoc `mkstemp`. + +| aspect | temp-file + `__del__` | cframe | +|---|---|---| +| Footprint | equal | equal | +| Pyodide (MEMFS) | works | works | +| User can leak | no | no | +| Maintainer can leak (new fetch path) | yes, if it bypasses `_materialize` | no | +| Discipline required | "all materialization via one method, forever" | none | +| Symmetry with `NDArray.to_cframe()` | no | yes | +| New blosc2 API | none | `CTable.to_cframe`/`ctable_from_cframe` (small) | + +**Option B: `CTable.to_cframe()`/`ctable_from_cframe()` on `EmbedStore`.** + +`EmbedStore` is the existing blosc2 mechanism for bundling **multiple** arrays +into one serializable store: + +```python +es = blosc2.EmbedStore(urlpath=None, mode="w") +es["/a"] = blosc2.asarray(...) +es["/b"] = blosc2.asarray(...) +cf = es.to_cframe() # -> bytes, one blob for the whole collection +back = blosc2.from_cframe(cf) # -> EmbedStore, fully in-memory +``` + +Tested — pure bytes, no temp files, `from_cframe(bytes)` works today. A CTable += `schema_dict()` + N column `NDArray`s + a `valid_rows` `NDArray`, so it +packs cleanly into an `EmbedStore`. An in-memory materialized CTable's columns +are plain `blosc2.NDArray` with working `to_cframe()` (tested). + +A real `CTable.to_cframe()` is therefore a small, Python-level, well-scoped +feature on an existing primitive: serializer packs schema + each column's +`to_cframe()` + `valid_rows.to_cframe()` into one `EmbedStore` and returns +`es.to_cframe()` → `bytes`; `ctable_from_cframe(bytes)` rebuilds from an +`InMemoryTableStorage`. It mirrors `NDArray.to_cframe()`/`ndarray_from_cframe()` +exactly: pure bytes, no disk, lazy-safe (objects live in memory after +`from_cframe`). + +The one design reason blosc2 currently keeps columns as external leaves +(`FileTableStorage` forces `threshold=0`) is a *persistent-store* concern so +large columns stay out of one schunk and small `_meta` overwrites persist +reliably. It does not apply to an in-memory cframe transport, where bundling +everything is the point. + +**Option C: `DictStore.to_bytes()` (native `.b2z` bytes).** + +`DictStore.to_b2z()` already builds the `.b2z` zip via `zipfile.ZipFile(path, +"w", ZIP_STORED)`. A `to_bytes()` into `BytesIO` would mirror it, producing +native `.b2z` bytes with zero recompression for whole persistent tables. + +But on its own it's **half a feature**: + +- The zip **reader** is path/offset-based (`blosc2_ext.open(path, offset=...)`). + `to_bytes()` without a buffer-backed `from_bytes()` reader just shifts the + temp-file problem from server to client — the consumer must write the bytes + to a temp `.b2z` and `open()` it. That buffer-backed reader is a big C-level + change, and it's the part that does the actual work. +- For a **slice**, `to_bytes()` falls to the logical `save()` path and + recompresses — same cost as the cframe, but producing `.b2z` bytes with worse + client ergonomics and no array symmetry. +- For a **whole table** that is already a file, `/api/download` already serves + those exact bytes (`FileResponse` on the on-disk `.b2z`); `to_bytes()` + duplicates it. It only differs for an in-memory/sliced table, which is the + slice case above. + +### Decision + +**Build `CTable.to_cframe()`/`ctable_from_cframe()` on `EmbedStore` (Option B). +Do not build `DictStore.to_bytes()` now.** + +Reasons: + +1. **Symmetry with the existing array-slice workflow.** `/api/fetch` streams + `array.slice(...).to_cframe()` and the client does + `blosc2.ndarray_from_cframe(data)`. A table fetch streams + `table.slice(...).to_cframe()` and the client does `ctable_from_cframe(data)` + — one code shape, no JSON-for-small / download-for-whole special-casing, + no second client-side pattern in `Client._fetch_data`. +2. **It closes a real gap in blosc2.** `CTable` is the only major container + without a bytes serialization. Adding it is appealing on its own merits, + independent of Caterva2. +3. **Intrinsic, invariant-free resource cleanup.** No client temp file, no + `_local_b2z`/`__del__` lifecycle to maintain, no "all materialization via one + method" convention for future maintainers to violate. Resource and object + are the same thing; ordinary refcounting is the cleanup. +4. **Pyodide-safe and footprint-equal.** Pure bytes; works in the browser + without MEMFS round-trips and with the same heap budget as the temp-file + path. +5. **Small, Python-level blosc2 change** on an existing primitive + (`EmbedStore`), unlike `DictStore.to_bytes()` which is gated behind a big + buffer-backed C reader to be useful. + +On the "`EmbedStore` is less battle-proven than `DictStore`/`TreeStore`" concern: +this is an argument **for** the cframe route, not against. As a *transport codec* +it gets real exercise in a simpler, more contained stress than being a +persistence backend, which hardens it. + +### `DictStore.to_bytes()` — deferred, with a revisit condition + +Not built now. It only becomes the best all-around answer **if and when a +buffer-backed zip reader** (`DictStore.from_bytes()` / `blosc2_ext.open` from a +buffer) lands in blosc2 — at which point `.b2z` bytes could do whole-table *and* +slice transport uniformly in one native format, and the cframe could even be +retired in favor of it. That is a blosc2-wide decision driven by "read `.b2z` +from a buffer" demand (e.g. S3/HTTP bytes, Pyodide `fetch()` without a MEMFS +round-trip), not a Caterva2 need. Revisit when such a reader exists; until then +`to_bytes()` is redundant with `/api/download` (whole files) and the cframe +(slices). + +### Consequences for the endpoints + +- `/api/fetch` is extended to tables via the cframe path (see next section), + **not** left array-only. This supersedes the earlier "do not support `.b2z` + in `/api/fetch`" stance: with `CTable.to_cframe()` available, the clean path + is to support it. +- `/api/download` continues to serve whole `.b2z` files as raw bytes + (`FileResponse`), unchanged. +- A JSON `/api/table` rows endpoint is **removed entirely** (Revision + 2026-07-01 #3), not just deferred: CLI `show` derives rows from the cframe via + the Python client, and the web UI renders rows server-side in + `htmx_path_view`, so neither needs a JSON-over-HTTP window. Table *operations* + live on `/api/fetch` params or a future `POST /api/query`, not here. + +## `/api/fetch` behavior (with cframe) + +Support `.b2z` in `/api/fetch` using `CTable.to_cframe()`, mirroring the +`NDArray` path. This is the slice-as-blosc2-object transport. + +**Critical (Revision 2026-07-01 #1): do NOT let a whole `CTable` fetch hit the +generic `FileResponse` short-circuit.** That short-circuit streams the raw +on-disk file. For `.b2nd` that file *is* a cframe, so the client decodes it. For +`.b2z` the file is a **zip** (`PK\x03\x04`), not a cframe — the client's +cframe decoder fails with `RuntimeError: Could not get the schunk from the +cframe`. So a whole `CTable` must also be serialized via `to_cframe()`. + +Concretely, exclude `CTable` from the whole short-circuit condition: + +```python +if ( + whole + and not isinstance( + array, blosc2.LazyArray | hdf5.HDF5Proxy | blosc2.NDField | blosc2.CTable + ) # <-- add CTable + and not filter +): + return FileResponse( + abspath, filename=abspath.name, media_type="application/octet-stream" + ) +``` + +Then the `CTable` branch always produces a cframe (whole or sliced): + +```python +if isinstance(container, blosc2.CTable): + row_start = 0 if whole else start + row_stop = container.nrows if whole else stop + view = container.slice(row_start, row_stop) # materialized CTable + data = view.to_cframe() # bytes, via EmbedStore + downloader = srv_utils.iterchunk(data) + return responses.StreamingResponse( + downloader, media_type="application/octet-stream" + ) +``` + +A whole-table fetch costs one recompress this way, but it is correct and keeps +the client's single decode path honest. `/api/download` still serves the raw +`.b2z` bytes (`FileResponse`) for actual file downloads — that path is fine +because the client writes bytes to disk rather than decoding a cframe. + +(Where exactly `ctable_to_cframe`/`ctable_from_cframe` live — `blosc2.CTable` +methods or module functions mirroring `ndarray_from_cframe` — is a blosc2 API +choice; Caterva2 just calls them.) + +Keep the accurate error message for non-supported suffixes: + +```python +if abspath.suffix not in BLOSC2_ARRAY_SUFFIXES | BLOSC2_TABLE_SUFFIXES: + srv_utils.raise_bad_request( + "The fetch API only supports datasets (.b2nd, .b2frame, .b2z); " + "use /api/download for other files" + ) +``` + +## Client class hierarchy + +Reparent the client leaf classes so a table is a *kind of dataset*, not a +sibling of one (Revision 2026-07-01 #5). Target: + +``` +File( ) # any leaf: vlmeta / download / move / copy / remove +└── Dataset(File) # a fetchable blosc2 data leaf: generic slice() -> blosc2 object + ├── Array(Dataset, blosc2.Operand) # shape / dtype / chunks / blocks / ndim / append; lazyexpr operand + └── Table(Dataset) # nrows / ncols / columns / schema +``` + +Current state (for reference) is asymmetric: + +``` +File +├── Dataset(File, blosc2.Operand) # the array class +└── Table(File) # sibling, not a Dataset +``` + +### Why this shape (not just cosmetics) + +- **`blosc2.Operand` must move down onto `Array` only.** `Dataset` inheriting + `Operand` is what lets an array participate in lazy expressions; a `Table` + must **not** be an `Operand` (a non-goal of this plan). The sibling design was + a blunt way to keep `Operand` off `Table`. Pushing `Operand` down to `Array` + gives both symmetry (`Table` is-a `Dataset`) and correctness (`Table` + is-not-an `Operand`). This is the reason the base cannot simply be today's + `Dataset`. +- **`File` stops leaking array semantics.** `File.__getitem__` already + references `self.dtype` (only defined on the array class), so `File` is + already array-flavored. Move that dtype/field-aware `__getitem__` down to + `Array`; keep on `Dataset` only the generic `slice() -> blosc2 object` + plumbing that both `Array` and `Table` share; keep `File` truly generic. +- **`Dataset` is effectively abstract.** `Root.__getitem__` always resolves to + `Array`, `Table`, or `File` — a bare `Dataset` is never instantiated, so there + is no ambiguous "what is a plain Dataset." + +### Backward-compat ledger + +- **`isinstance(x, Dataset)` broadens** from "array" to "array *or* table" — the + originally intended meaning of `Dataset` as the generic data leaf. Behavior + change: a `.b2z` was `isinstance … Dataset == False`, becomes `True`. Audit + the 4 sites in `client.py` (`get()` ~998, `download`/`get_slice` ~1060/1126, + `get_chunk` ~1190); the first three *simplify* (`Dataset ⊂ File`, + `Table ⊂ Dataset`), and `get_chunk` (array-flavored) needs a check that it is + not wrongly handed a `Table`. `test_api.py`'s `isinstance(myds, cat2.Dataset)` + on a `.b2nd` keeps passing since `Array(Dataset)`. +- **Repr doctests**: 5 doctests print `` (client.py ~123, 234, + 524, 557, 586) → become ``. Mechanical update. +- **Exports**: keep `Dataset` in `__init__.py` / `__all__` (now the base, still + importable — no import breakage); add `Array` and `Table`. +- **Naming**: use `Array`, not `NDArray`, to parallel `Table` and avoid + colliding with the `NDArray` alias already imported for type hints + (client.py:416). +- Blast radius is confined to `client.py` plus those doctests/tests; server, web + UI, and CLI do not touch these client classes. + +### `Root.__getitem__` dispatch + +```python +if path.endswith((".b2nd", ".b2frame")): + return Array(self, path) # was Dataset(self, path) +if path.endswith(".b2z"): + return Table(self, path) +return File(self, path) +``` + +## Python client API + +With the hierarchy above, `Table` subclasses `Dataset` (not `File`). Note the +metadata field is `schema_dict`, not `schema` (a pydantic `BaseModel` already +defines `schema`, so naming the field `schema` shadows/deprecates it — the model +uses `schema_dict`). Keep `ncols` and a `schema` *property* for parity with the +metadata even though they are cheap conveniences: + +```python +class Table(Dataset): + def __repr__(self): + return f"" + + @property + def nrows(self): + return self.meta["nrows"] + + @property + def ncols(self): + return self.meta["ncols"] + + @property + def columns(self): + return self.meta["columns"] + + @property + def schema(self): + return self.meta["schema_dict"] + + def slice(self, key): + """Row slice as a blosc2.CTable (via the cframe /api/fetch path).""" + return self.client.get_slice(self.path, key, as_blosc2=True) + + def __getitem__(self, key): + return self.slice(key) + + def rows(self, start=0, stop=50): + """Preview rows as a list of tuples, off the cframe slice.""" + ct = self.slice(slice(start, stop)) + return [tuple(row) for row in ct[:]] + + def head(self, n=10): + return self.rows(0, n) +``` + +Rows come from the **cframe path** (`get_slice` → `/api/fetch` → a +`blosc2.CTable`), not a JSON endpoint. There is no `get_table_rows()` and no +`/api/table` — both removed (Revision 2026-07-01 #3). + +`Root.__getitem__()` dispatch is shown in "Client class hierarchy" (returns +`Array` for `.b2nd`/`.b2frame`, `Table` for `.b2z`, `File` otherwise). + +`Client.get()`'s type check simplifies with the new hierarchy — since +`Array`/`Table` are both `Dataset ⊂ File`, the tuple collapses: + +```python +if isinstance(path, (File, Root)): # was (File, Dataset, Table, Root) + return path +``` + +### `_fetch_data` decode: dispatch on known kind (Revision 2026-07-01 #2) + +Do **not** decode by trial-and-except (`ctable_from_cframe` first, catch +`ValueError`, then `ndarray_from_cframe`, catch `RuntimeError`, …). That makes +every array fetch pay a failed CTable parse and couples the code to each +decoder's exception type. The caller already knows the kind — an `Array` vs a +`Table` (both `Dataset`s), or `meta["kind"] == "ctable"` — so pass it down and +pick the decoder directly: + +```python +def _fetch_data(self, path, urlbase, params, kind=None, as_blosc2=False, timeout=5): + data = self._xget(...).content + if kind == "ctable": + obj = blosc2.ctable_from_cframe(data) + else: + try: + obj = blosc2.ndarray_from_cframe(data) + except RuntimeError: + obj = blosc2.schunk_from_cframe(data) + if as_blosc2: + return obj + if isinstance(obj, blosc2.CTable): + return [tuple(row) for row in obj[:]] + if hasattr(obj, "ndim"): + return obj[()] if obj.ndim == 0 else obj[:] + return obj[:] +``` + +The `Array`/`Table` wrappers already know their kind, so threading it through +`get_slice`/`_fetch_data` is local. (The remaining array `ndarray`/`schunk` +try/except is fine: both are cframes and the ambiguity is genuine.) + +## CLI changes + +### `info` + +If metadata contains `kind == "ctable"`, print table-specific fields: + +```text +kind : ctable +nrows : 1000000 +ncols : 12 +chunks : (1048576,) +blocks : (32768,) +nbytes : 123 MB +cbytes : 12 MB +ratio : 10.2x +columns: a, b, c, ... +mtime : ... +``` + +For `--json`, no special work: dump metadata as returned. + +### `show` + +For `.b2z`, do not call `client.fetch()`. + +Minimal behavior: + +```sh +cat2-client show path/table.b2z +``` + +prints first 50 rows. + +Optional first-pass slice support: + +```sh +cat2-client show path/table.b2z[0:20] +``` + +maps to `start=0`, `stop=20`. + +Optional columns could come later via `--columns a,b`. Skip unless asked. + +Implementation: + +- detect `.b2z` path +- parse single row slice only +- call `table.rows(start, stop)` (which uses the cframe path); there is no + `get_table_rows()`/`/api/table` (removed, Revision 2026-07-01 #3) +- print rows as a simple table or JSON + +Lazy first text output can use standard formatting: + +```python +rows = client.get(path).rows(start, stop) # list of tuples off the cframe +print(json.dumps(rows, indent=2, default=str)) +``` + +Pretty tables can come later. + +## Web UI changes + +### Reuse of the existing structured-array visualizer + +Caterva2 already has a table visualizer for **structured NDArrays** (dtypes with +fields). It splits into two layers, and CTable reuse differs per layer +(verified against the live `CTable`/`NDArray` APIs, 2026-07-01): + +- **Template `info_view.html` — reusable as-is.** It is driven entirely by + generic context keys (`rows`, `cols`, `fields`, `sortby`, `filter`, `shape`, + `inputs`, `tags`); nothing in it is array-specific. The Fields dropdown, + Sort-by select, Filter box, and the row/column grid all render from a plain + table model. A CTable is exactly the "1-D, has-fields" case this template + already handles, so no template change is needed to show rows. + +- **Server extraction (`htmx_path_view` + `get_filtered_array`) — NOT reusable + directly.** It is hard-wired to the structured-`NDArray` API, and a `CTable` + is a different type exposing none of those attributes: + + | Extraction needs | structured `NDArray` | `CTable` | CTable equivalent | + |---|---|---|---| + | `.fields` (detect + col names) | yes | **no** | `schema_dict()["columns"]` → names | + | `.shape` / `.ndim` (windowing) | yes | **no** | `nrows` / `ncols` (conceptually 1-D) | + | `.tolist()` (materialize rows) | yes | **no** | iterate `slice(a,b)` → `CTableRow._asdict()` | + | positional `row[i]` | yes | **no** (`CTableRow`, keyed) | `row._asdict()[field]` | + | `.argsort()` / `.sort()` (filter/sort) | yes | **no** | `where()` / `sort_by()` / `sorted_slice()` | + | column select | `arr.fields[name]` (NDField) | `t[name]`→`Column`, `t[[names]]`→`CTable` | usable, different type | + + **Pitfall:** `htmx_path_view` starts with `shape = arr.shape` — a CTable has + no `.shape`, so this line `AttributeError`s. The CTable branch must come + **before** it. + + **Rejected shortcut:** converting a CTable slice to a structured `NDArray` + (its columns already are NDArrays) to feed the existing `has_ndfields` path + verbatim — it breaks on **variable-length string columns**, which don't fit a + fixed numpy structured dtype and are a first-class CTable case. Use a native + branch. + +### `htmx_path_info()` + +Currently the array display tab is added when `hasattr(meta, "shape")`. + +Add the table display tab when `kind == "ctable"`: + +```python +if getattr(meta, "kind", None) == "ctable": + context["data_url"] = make_url(request, "htmx_path_view", path=path) + context["shape"] = (meta.nrows,) # synthesize a 1-D shape; meta has no .shape + tabs.append( + { + "name": "data", + "label": "Display", + "include": "includes/info_data.html", + } + ) +``` + +### `htmx_path_view()` + +Add an early CTable branch **before** `shape = arr.shape` (which would otherwise +fail). It fills the *same* context keys the template expects, from CTable APIs — +note **keyed** row access via `_asdict()`, not the structured array's positional +`row[i]`: + +```python +if isinstance(arr, blosc2.CTable): + cols = [c["name"] for c in arr.schema_dict()["columns"]] + fields = fields or cols[:5] + index = (0,) if index is None else tuple(index) + size = (sizes or [10])[0] + start = index[0] + stop = min(start + size, arr.nrows) + # keyed access (CTableRow._asdict()), NOT positional row[i] + rows = [fields] + [ + [_cell(row._asdict()[f]) for f in fields] for row in arr.slice(start, stop) + ] + context = { + "view_url": make_url(request, "htmx_path_view", path=path), + "inputs": [ + { + "start": start, + "start_max": max(arr.nrows - size, 0), + "size": size, + "size_max": arr.nrows, + "with_size": True, + } + ], + "rows": rows, + "cols": cols, + "fields": fields, + "filter": "", + "sortby": "", + "shape": (arr.nrows,), + "tags": list(range(start, stop)), + "filterable": False, # see caveat below + } + return templates.TemplateResponse(request, "info_view.html", context) +``` + +`_cell()` coerces numpy scalars / bytes for HTML (mirrors the JSON edge cases: +`np.generic` → `.item()`, `bytes` → decode). Column subset could also use +`arr[fields]` (returns a CTable), but per-row `_asdict()` keyed access is simpler +and avoids re-slicing. + +**Filter/sort caveat.** The template shows the Filter box and Sort-by select +whenever `cols` is truthy — but `get_filtered_array` is **not** reusable for +CTable (it opens with `assert has_ndfields` and uses NDArray lazy-expr +`argsort`/`sort`; a CTable needs a parallel path on `where()` + +`sort_by()`/`sorted_slice()`). Filtering/sorting is deferred (Revision +2026-07-01 #4), so for the MVP **render rows only and hide those two controls**: +add a `filterable` context flag (set `False` here, `True` on the array path) and +guard the Filter/Sort-by blocks in `info_view.html` with `{% if filterable %}`. +The Fields (column subset) dropdown can stay — it works with a plain slice. + +When filtering does land, it rides the schema-preserving `/api/fetch` +`filter=`/`sortby=` path (Revision 2026-07-01 #4), backed by a CTable +`where()`/`sort_by()` branch — not `get_filtered_array` and not `/api/table`. + +## Tests + +Add one small CTable fixture. + +Example creation helper: + +```python +from dataclasses import dataclass +import blosc2 + + +@dataclass +class Row: + id: int = blosc2.field(blosc2.int32()) + name: str = blosc2.field(blosc2.string(max_length=20)) + + +def make_ctable(path): + table = blosc2.CTable(Row, urlpath=str(path), mode="w", compact=True) + table.append((1, "alice")) + table.append((2, "bob")) + table.append((3, "carol")) + table.close() +``` + +Tests: + +1. `read_metadata()` on `.b2z` + - `kind == "ctable"` + - `nrows == 3` + - `columns == ["id", "name"]` + +2. `/api/info/...table.b2z` + - same assertions through HTTP + +3. `/api/download/...table.b2z` + - write response to temp file + - `blosc2.open(downloaded_path)` returns `blosc2.CTable` + - `len(table) == 3` + +4. `/api/fetch` cframe transport (the bug from Revision 2026-07-01 #1) + - **whole** fetch (`table[:]`) decodes to a `blosc2.CTable` with `nrows == 3` + — guards against the raw-zip regression + - **slice** fetch (`table[1:3]`) decodes to bob/carol + +5. Python client + - `root["table.b2z"]` is `Table` + - `table.nrows == 3` + - `table.head(2)` has two rows (list of tuples, off the cframe) + - `table[:]` / `table.slice(slice(1, 3))` return a `blosc2.CTable` + +6. CLI smoke test + - `cat2-client info table.b2z --json` contains `kind: ctable` + - `cat2-client show table.b2z --json` returns first rows + +7. Web preview smoke test (`htmx_path_view` CTable branch) + - a `.b2z` renders rows without error (guards the `arr.shape` pitfall) + - Filter/Sort-by controls are absent (`filterable=False`); Fields dropdown + present + +No `/api/table` test — that endpoint is removed (Revision 2026-07-01 #3). + +Keep tests small. No fixtures for huge data, no performance suite. + +## Edge cases + +### NumPy scalar JSON encoding + +CTable rows can contain NumPy scalar types. Convert with `.item()`. + +### NumPy array cells + +CTable can have nested/NDArray-like cells. For JSON preview, convert arrays to lists. + +### Bytes cells + +Decode as UTF-8 with replacement for preview. For exact bytes transport, use download/export later. + +### Large pages + +Cap preview rows (e.g. 1000/request) to prevent accidental huge responses from +the web UI or CLI. (Row-count caps and column validation as HTTP 400 were tied +to the removed `/api/table` endpoint; for the MVP the web UI/CLI simply bound +their own preview windows.) + +### Unknown columns + +Only relevant once column selection is exposed (as a `field=`/`columns=` param +on `/api/fetch`, not a JSON endpoint). When it lands, reject unknown requested +columns with 400 — do not silently ignore. + +### `.b2d` + +Directory-backed CTable should be rejected/not recognized initially. If uploaded as archive, it is just a regular file/archive. Native directory-backed support needs separate path-list and download semantics. + +## Future extensions + +Only add these once MVP is working and there is demand. + +### Table operations: two homes, split by schema effect + +Transport (cframe vs JSON) is orthogonal to operation. Every table op +(`where`/`sort`/`group_by`/…) *returns a `CTable`* (or an NDArray for scalar +reductions), which the cframe path already serializes. So operations never need +a JSON row endpoint — they need a way to be *expressed in the request*. Route +them by what the op does to the schema (Revision 2026-07-01 #4): + +**Schema-preserving → params on `/api/fetch`, cframe response.** +`where`/`sort` return the same columns — a subset/reordering of the same rows — +so they are still "a region of this dataset." This is exactly the array +`get_filtered_array` path, now viable for tables since lazy-view `to_cframe()` +works. + +```text +GET /api/fetch/{path}?filter=col%20%3E%2010&field=a,b&sortby=col&start=0&stop=50 +``` + +```python +table.where("col > 10", columns=["a", "b"]) # -> CTable -> cframe +table.sort_by("col") / table.sorted_slice(...) # -> CTable -> cframe +``` + +Needs careful expression validation/error handling. **Not** a JSON `/api/table` +variant. + +**Schema-changing → a future `POST /api/query/{path}`, cframe response.** +`group_by`/`aggregate`/`join`/`describe` produce a *new* dataset with a +different schema/cardinality. Cramming these into `/api/fetch` GET params would +turn query strings into a stringly-typed query language and erode fetch's +"give me bytes of this dataset" contract. Give them a structured JSON *request +body* and a cframe *response*: + +```text +POST /api/query/{path} +{ "where": "...", "groupby": ["a"], "aggregations": {"s": "sum(b)"}, + "orderby": "s", "limit": 100 } +``` + +This is compute-not-fetch — a genuinely different resource. Post-MVP, only when +an analytical use case lands. (The response is still a cframe `CTable`, so no +JSON-rows machinery is resurrected.) + +### Export formats (Arrow / CSV / Parquet) + +Distinct from both fetch and query: these re-serialize a table (or a query +result) into a foreign format for download. Hang them off `/api/download` (or a +future query result) with a `format=` param — **not** `/api/table`: + +```text +GET /api/download/{path}?format=arrow # table.to_arrow() / iter_arrow_batches() +GET /api/download/{path}?format=csv # table.to_csv() +GET /api/download/{path}?format=parquet # table.to_parquet() +``` + +Arrow/Parquet likely need PyArrow and temp files — add only if PyArrow is +already an accepted dependency/extra, never as a core dependency just for first +support. + +### `.b2d` support + +Directory-backed tables need decisions: + +- how to list them as one logical dataset +- how to upload/download them +- whether to tar/zip for download +- how to protect internal files from direct mutation + +Do later. + +## Suggested implementation order + +1. Add suffix constants and `.b2z` native handling. +2. Add `CTableMetadata` and `read_metadata()` support. +3. Make `open_b2()` return `CTable` safely. +4. Ensure `/api/download` preserves `.b2z` bytes. +5. **blosc2 prerequisite:** implement `CTable.to_cframe()` / + `ctable_from_cframe()` on `EmbedStore` (see the transport decision), then + wire `/api/fetch` table whole+slices + `Client.get_slice`/`Table.slice` to + it. This is the slice/whole-as-blosc2-object transport and the basis for + preview. +6. Refactor the client leaf hierarchy to `File → Dataset → {Array, Table}` + (see "Client class hierarchy"): introduce `Array`, push `blosc2.Operand` + down onto it, reparent `Table` under `Dataset`, update `Root.__getitem__`, + exports, the `isinstance(_, Dataset)` sites, and the repr doctests. +7. Add Python client `Table` behavior (`slice`/`rows`/`head` off the cframe + path) on the reparented class. +8. Add web preview: a CTable branch in `htmx_path_view` (before `arr.shape`) + reusing `info_view.html`, plus a `filterable` flag + `{% if filterable %}` + guard to hide filter/sort (see "Reuse of the existing structured-array + visualizer"). `get_filtered_array` is not reused. +9. Add CLI `info`/`show` support (rows off the cframe path). +10. Add the small CTable tests (including the whole-table cframe regression). + +There is **no** `/api/table` JSON endpoint step — removed (Revision +2026-07-01 #3). + +## Implementation status + +**Complete (2026-07-01).** Everything in the MVP scope is implemented and +committed. Commits: `3f4748c` (metadata + fetch pipeline + initial `Table`), +`32947e8` (whole-fetch fix + `Array`/`Table` hierarchy + `.b2z` storage), +`6a45518` (web preview + CLI), plus a post-review hardening commit. + +### blosc2 side (done) + +- `CTable.to_cframe()` + `blosc2.ctable_from_cframe(bytes)` on `EmbedStore` + (`EmbedStoreTableStorage` backend), exported; fails cleanly (`ValueError`) on + non-CTable cframes. Handles scalar/string/list/dict/vlstring columns + vlmeta. +- Lazy-view `to_cframe()` copy() limitation fixed (`where()`/`view()` serialize). +- **Extra (found during Caterva2 QA):** `CTableRow.__getitem__` now resolves + dotted paths into nested struct columns (`row["trip.sec"]`) via + `split_field_path`, so the names `schema_dict()` advertises are all + addressable. Fixed upstream with tests + (`tests/ctable/test_nested_access_storage.py`); the temporary caterva2-side + workaround was removed once it landed. + +### Caterva2 side (done) + +- **Constants/metadata:** `BLOSC2_NATIVE_SUFFIXES` (incl. `.b2z`) in + `srv_utils.py`; `CTableMetadata` in `models.py` (field is **`schema_dict`**, + not `schema` — the "Metadata model" sketch above predates the rename); + `read_metadata()` → `CTableMetadata`; `open_b2()` returns `CTable` early; + `get_abspath()` treats `.b2z` as native. +- **`/api/fetch` (Revision #1):** whole *and* sliced `.b2z` serialize via + `CTable.slice(...).to_cframe()`; `CTable` excluded from the `FileResponse` + whole short-circuit so `table[:]` no longer returns raw zip bytes. +- **`_fetch_data` (Revision #2):** `get_slice` computes `kind` before the request + and `_fetch_data` dispatches `ctable_from_cframe` only for `kind=="ctable"`; + no more trial-and-except format sniffing. +- **Client hierarchy (Revision #5):** `File → Dataset → {Array, Table}`; + `blosc2.Operand` on `Array` only; array-flavored `__getitem__` moved off + `File`; `Array`/`Table` exported; redundant `isinstance` tuples simplified; + repr doctests updated. +- **`Table` client:** `nrows`, `ncols`, `columns`, `schema` (reads + `schema_dict`), `slice`/`__getitem__` → `blosc2.CTable`, `rows`, `head`. + (As-built defaults differ from the sketch: `rows(start=0, stop=50)` and + `head(n=5)`.) +- **Upload/download:** switched hardcoded suffix sets to + `BLOSC2_NATIVE_SUFFIXES` (incl. `.b2z`) across `/api/upload`, + `/api/load_from_url`, `/htmx/upload`, and path-guessing; + `get_file_content`/`/api/download` already served `.b2z` raw bytes correctly. +- **Web preview:** `htmx_path_info` Display tab for `kind=="ctable"`; + `htmx_path_view` CTable branch before `arr.shape`, keyed `row[...]` access, + `cell()` coercion, `filterable=False` + `{% if filterable %}` guard in + `info_view.html`. **Extra:** fixed a pre-existing `info_metadata.html` crash + (the Meta tab assumed `cparams` on any `nbytes`-bearing metadata, false for + `CTableMetadata`). +- **CLI:** `info` prints table fields for `kind=="ctable"`; `show` parses + `table.b2z[start:stop]` and prints `Table.rows()` off the cframe (no `fetch`). +- **Tests:** `caterva2/tests/test_ctable.py` (25 tests) covering all 7 planned + items + nested/non-identifier column-name regressions. + +### Post-review hardening (done) + +- `Table.rows()` default bounded to `[0:50)` (was an unbounded whole-table fetch). +- `/api/fetch` CTable slice: `is None` instead of truthiness (so `[0:0]` is + empty, not the whole table), plus negative-index normalization and clamping. +- CLI `show --json`: numpy/bytes cell coercion via a `json.dumps` default. +- Filtering/sorting an unsupported dataset (e.g. a `.b2z`) now returns a clean + htmx error instead of an uncaught `AssertionError` (500). + +### Not built (intentionally deferred — see "Future extensions") + +- JSON `/api/table` row endpoint — **removed** (Revision #3). +- CTable filtering/sorting (`where`/`sort_by`) over `/api/fetch` params. +- `POST /api/query` (group_by/join/describe); Arrow/CSV/Parquet export; `.b2d`. + +## Open questions + +- Should web table preview support filters immediately? + - Proposed first pass: no. (Transport is settled — cframe `/api/fetch` — so + adding it later is a param, not a new endpoint.) + +- Should `.b2z` be included in `/api/fetch`? + - **Decided: yes**, via the `CTable.to_cframe()` cframe path, for **whole and + sliced** fetches (see Revision 2026-07-01 #1). `/api/download` remains for + whole-file raw bytes. + +- Should there be a JSON `/api/table` endpoint? + - **Decided: removed, not just deferred** (Revision 2026-07-01 #3) — a + consumer-less duplicate of the cframe slice. Revisit only if a real external + non-blosc2 JSON consumer appears. Table *operations* never motivate it. + +- Where do table operations (`where`/`sort`/`group_by`) live? + - **Decided** (Revision 2026-07-01 #4): schema-preserving (`where`/`sort`) → + params on `/api/fetch` (cframe); schema-changing (`group_by`/`join`/ + `describe`) → a future `POST /api/query` (cframe). Neither is `/api/table`. + +- Should `Table` be a sibling of the array class or a `Dataset`? + - **Decided: a `Dataset`** (Revision 2026-07-01 #5). Reparent to + `File → Dataset → {Array, Table}`, pushing `blosc2.Operand` down onto + `Array` so `Table` is a dataset without being an operand. See "Client class + hierarchy". + +- Should Caterva2 expose Arrow if Blosc2 can export Arrow? + - Proposed first pass: no; avoid dependency/API expansion. If added later, it + is a `format=` param on `/api/download`, not a JSON endpoint. + +- Should table metadata include per-column compression stats? + - Proposed first pass: no, unless Blosc2 exposes a stable cheap dict. + +## Definition of done for MVP + +The cframe transport (`CTable.to_cframe()`, blosc2) is a **prerequisite for the +whole MVP**, not a final add-on: preview itself now rides it. With it in place, +a user can: + +```python +import caterva2 as cat2 + +client = cat2.Client("https://server") +root = client.get("@public") +table = root["example.b2z"] + +print(table.nrows) +print(table.columns) +print(table.head(5)) +table.download() +``` + +And in CLI: + +```sh +cat2-client info @public/example.b2z +cat2-client show @public/example.b2z +``` + +And in the web UI: + +- click a `.b2z` +- see metadata +- see a small row/column preview + +**Slice/whole fetch as a blosc2 object** (via `CTable.to_cframe()`): + +```python +ds = root["example.b2z"] +view = ds.slice(slice(10, 20)) # blosc2.CTable, mirroring ds.slice() for arrays +whole = ds[:] # blosc2.CTable too — must NOT return raw zip bytes +``` + +This mirrors the existing array workflow (`ds.slice(...)` -> `blosc2.NDArray`) +and closes the symmetry gap. Both slice **and** whole must decode as a +`blosc2.CTable` on the client (Revision 2026-07-01 #1). From 2adefd8acb8361746b781e4f05ad1f68dd2701e5 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 14:02:02 +0200 Subject: [PATCH 07/23] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- caterva2/services/server.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/caterva2/services/server.py b/caterva2/services/server.py index 93a41164..aa4d1ed1 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -588,15 +588,19 @@ async def fetch_data( if isinstance(sl0, slice): row_start = 0 if sl0.start is None else sl0.start row_stop = array.nrows if sl0.stop is None else sl0.stop + if row_start < 0: + row_start += array.nrows + if row_stop < 0: + row_stop += array.nrows elif isinstance(sl0, int): - row_start, row_stop = sl0, sl0 + 1 + row_start = sl0 + if row_start < 0: + row_start += array.nrows + row_stop = row_start + 1 else: row_start, row_stop = 0, array.nrows - # Normalize negatives and clamp to [0, nrows]. - if row_start < 0: - row_start += array.nrows - if row_stop < 0: - row_stop += array.nrows + + # Clamp to [0, nrows]. row_start = max(0, min(row_start, array.nrows)) row_stop = max(row_start, min(row_stop, array.nrows)) view = array.slice(row_start, row_stop) From 6ce34caca5e3a6a12ff65136021bc0bc7fbf3504 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 1 Jul 2026 12:02:53 +0000 Subject: [PATCH 08/23] Fix hardcoded suffix set in get_abspath() to use srv_utils.BLOSC2_NATIVE_SUFFIXES --- caterva2/services/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caterva2/services/server.py b/caterva2/services/server.py index aa4d1ed1..12206cf9 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -456,7 +456,7 @@ def get_abspath( return cachedir / filepath # HDF5 files cannot be compressed, as they are supported natively - if filepath.suffix not in {".b2frame", ".b2nd", ".b2z", ".h5"} and not may_not_exist: + if filepath.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES | {".h5", ".hdf5"} and not may_not_exist: if filepath.is_file(): srv_utils.compress_file(filepath) filepath = f"{filepath}.b2" From e3fc9b31c686a53c19390e44a57f3860d0636d4c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 14:03:26 +0200 Subject: [PATCH 09/23] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- caterva2/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/caterva2/client.py b/caterva2/client.py index 6d12718c..cb58e2bc 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -1258,9 +1258,10 @@ def get_chunk(self, path, nchunk): >>> info_schunk['chunksize'] / len(chunk) 6.453000645300064 """ - if isinstance(path, File): + if isinstance(path, Array): path = path.path - _, path = _format_paths(self.urlbase, path) + elif isinstance(path, File): + raise TypeError("get_chunk() only supports Array datasets (.b2nd/.b2frame), not regular files or tables") data = self._xget( f"{self.urlbase}/api/chunk/{path}", {"nchunk": nchunk}, From e410ea397532b1b153ec85b83ce3e942fdfb2a94 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 14:11:50 +0200 Subject: [PATCH 10/23] Add test for the table[-1] fix --- caterva2/tests/test_ctable.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/caterva2/tests/test_ctable.py b/caterva2/tests/test_ctable.py index c84e3a07..d864f4e4 100644 --- a/caterva2/tests/test_ctable.py +++ b/caterva2/tests/test_ctable.py @@ -287,6 +287,33 @@ def test_fetch_whole_and_slice(fill_ctable_public, client): assert tuple(part[1]) == (2, "v2") +def test_fetch_negative_index(fill_ctable_public, client): + """Regression (PR #288 review): negative indexing must not clamp to empty. + + `table[-1]` previously resolved to an empty slice because ``row_stop`` was + derived (``sl0 + 1`` -> 0) before negative normalization was applied. + """ + fname, root = fill_ctable_public + path = f"{root.name}/{fname}" # 3 rows: (0, "v0"), (1, "v1"), (2, "v2") + + # The exact failing case: -1 must return the last row, not an empty table. + last = client.get_slice(path, -1, as_blosc2=True) + assert isinstance(last, blosc2.CTable) + assert len(last) == 1 + assert tuple(last[0]) == (2, "v2") + + # A non-(-1) negative int also resolves to a single row. + second_last = client.get_slice(path, -2, as_blosc2=True) + assert len(second_last) == 1 + assert tuple(second_last[0]) == (1, "v1") + + # Negative slice start still works. + tail = client.get_slice(path, slice(-2, None), as_blosc2=True) + assert len(tail) == 2 + assert tuple(tail[0]) == (1, "v1") + assert tuple(tail[1]) == (2, "v2") + + def test_client_table_class(fill_ctable_public): fname, root = fill_ctable_public table = root[fname] From 8ee2af25867bfdbc1e7ba3cbe7154f405ee93c70 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 18:42:55 +0200 Subject: [PATCH 11/23] Support browsing TreeStore .b2z containers via virtual paths A .b2z may hold a TreeStore (a hierarchy of leaves), not just a CTable. Address leaves by path (tree.b2z/level1/ctable) without unfolding to disk: list descends, info/fetch open the leaf, the web tree expands into leaf rows, and the client dispatches by server-reported kind. --- caterva2/client.py | 18 +++++++-- caterva2/services/server.py | 68 +++++++++++++++++++++++++++++----- caterva2/services/srv_utils.py | 41 ++++++++++++++++++-- 3 files changed, 110 insertions(+), 17 deletions(-) diff --git a/caterva2/client.py b/caterva2/client.py index cb58e2bc..4149c16b 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -125,10 +125,18 @@ def __getitem__(self, path): path = path.as_posix() if hasattr(path, "as_posix") else str(path) if path.endswith((".b2nd", ".b2frame")): return Array(self, path) - if path.endswith(".b2z"): - return Table(self, path) - else: + if path.endswith(".b2z") or ".b2z/" in path: + # A .b2z may be a CTable or a TreeStore (a folder whose leaves are + # addressed by inner path). Inner leaves carry no file suffix, so the + # server's metadata is what tells us the kind. + meta = self.client.get_info(f"{self.name}/{path}") + if meta.get("kind") == "ctable": + return Table(self, path) + if "shape" in meta: + return Array(self, path) + # A TreeStore container itself: browse via get_list / inner paths. return File(self, path) + return File(self, path) def __contains__(self, path): """ @@ -1261,7 +1269,9 @@ def get_chunk(self, path, nchunk): if isinstance(path, Array): path = path.path elif isinstance(path, File): - raise TypeError("get_chunk() only supports Array datasets (.b2nd/.b2frame), not regular files or tables") + raise TypeError( + "get_chunk() only supports Array datasets (.b2nd/.b2frame), not regular files or tables" + ) data = self._xget( f"{self.urlbase}/api/chunk/{path}", {"nchunk": nchunk}, diff --git a/caterva2/services/server.py b/caterva2/services/server.py index 12206cf9..b8bb5248 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -367,9 +367,20 @@ async def get_list( list The list of datasets, as name strings relative to path. """ - # List the datasets in root or directory - directory = get_writable_path(path, user) + # A path may descend into a container (e.g. a TreeStore .b2z); list its members. + container_path, inner_key = srv_utils.split_container_path(path) + directory = get_writable_path(container_path, user) if directory.is_file(): + if directory.suffix == ".b2z": + tree = blosc2.open(directory) + if isinstance(tree, blosc2.TreeStore): + # Deep listing of leaves relative to the requested path, matching + # walk_files() semantics for directories. + prefix = inner_key or "/" + strip = prefix.rstrip("/") + "/" + return sorted(d[len(strip) :] for d in srv_utils.treestore_leaves(tree, prefix)) + if inner_key is not None: + srv_utils.raise_not_found() name = pathlib.Path(directory.name) return [str(name.with_suffix("") if name.suffix == ".b2" else name)] # Sort the list of datasets and return @@ -398,7 +409,14 @@ async def get_info( dict The metadata of the dataset. """ - abspath = get_abspath(path, user) + container_path, inner_key = srv_utils.split_container_path(path) + abspath = get_abspath(container_path, user) + if inner_key is not None: + # A member inside a container (e.g. a TreeStore .b2z leaf). + leaf = blosc2.open(abspath)[inner_key] + if isinstance(leaf, blosc2.TreeStore): + srv_utils.raise_not_found() # a group node is directory-like; use /api/list + return srv_utils.read_metadata(leaf, mtime=abspath.stat().st_mtime) if abspath.is_dir(): srv_utils.raise_not_found() return srv_utils.read_metadata(abspath) @@ -522,7 +540,8 @@ async def fetch_data( """ slice_ = parse_slice(slice_) - abspath = get_abspath(path, user) + container_path, inner_key = srv_utils.split_container_path(path) + abspath = get_abspath(container_path, user) if abspath.suffix not in {".b2frame", ".b2nd", ".b2z"}: srv_utils.raise_bad_request( @@ -530,7 +549,12 @@ async def fetch_data( "use the download API if you only want to download the file" ) - if filter: + if inner_key is not None: + # A member inside a container (e.g. a TreeStore .b2z leaf). + container = blosc2.open(abspath)[inner_key] + if isinstance(container, blosc2.TreeStore): + srv_utils.raise_not_found() + elif filter: if field: srv_utils.raise_bad_request("Cannot handle both field and filter parameters at the same time") filter = filter.strip() @@ -576,6 +600,7 @@ async def fetch_data( whole and (not isinstance(array, blosc2.LazyArray | hdf5.HDF5Proxy | blosc2.NDField | blosc2.CTable)) and (not filter) + and inner_key is None # abspath is the container, not this leaf: never stream it whole ): # Send the data in the file straight to the client, # avoiding slicing and re-compression. @@ -1676,6 +1701,16 @@ def add_dataset(path, abspath): if relpath.suffix == ".b2": relpath = relpath.with_suffix("") path = f"{root}/{relpath}" + # A .b2z holding a TreeStore is browsed as a folder: expand it into + # one row per leaf (e.g. tree.b2z/level1/ctable). + if relpath.suffix == ".b2z": + tree = blosc2.open(abspath) + if isinstance(tree, blosc2.TreeStore): + for key in srv_utils.treestore_leaves(tree): + leaf_path = f"{path}{key}" + if search in leaf_path: + add_dataset(leaf_path, abspath) + continue if search in path: add_dataset(path, abspath) @@ -1757,9 +1792,13 @@ async def htmx_path_info( else: push_url = None - # Read metadata - abspath = get_abspath(path, user) - meta = srv_utils.read_metadata(abspath) + # Read metadata (a path may descend into a container, e.g. a TreeStore .b2z leaf) + container_path, inner_key = srv_utils.split_container_path(path) + abspath = get_abspath(container_path, user) + if inner_key is not None: + meta = srv_utils.read_metadata(blosc2.open(abspath)[inner_key], mtime=abspath.stat().st_mtime) + else: + meta = srv_utils.read_metadata(abspath) # Context current_url = push_url or hx_current_url @@ -1881,9 +1920,18 @@ async def htmx_path_view( # Depends user: db.User = Depends(optional_user), ): - abspath = get_abspath(path, user) + container_path, inner_key = srv_utils.split_container_path(path) + abspath = get_abspath(container_path, user) filter = filter.strip() - if filter or sortby: + # ponytail: filter/sort on a TreeStore leaf would run against the container; the + # default (unfiltered) view is what renders on click, so leave that path as-is. + if inner_key is not None and not (filter or sortby): + try: + arr = blosc2.open(abspath)[inner_key] + except ValueError: + return htmx_error(request, "Cannot open container member.") + idx = None + elif filter or sortby: try: mtime = abspath.stat().st_mtime arr, idx = get_filtered_array(abspath, path, filter, sortby, mtime) diff --git a/caterva2/services/srv_utils.py b/caterva2/services/srv_utils.py index 2741f086..98ff3e2a 100644 --- a/caterva2/services/srv_utils.py +++ b/caterva2/services/srv_utils.py @@ -36,6 +36,34 @@ BLOSC2_FRAME_SUFFIXES = {".b2"} BLOSC2_NATIVE_SUFFIXES = BLOSC2_ARRAY_SUFFIXES | BLOSC2_TABLE_SUFFIXES | BLOSC2_FRAME_SUFFIXES +# Container suffixes whose paths may descend into internal (virtual) members. +BLOSC2_CONTAINER_SUFFIXES = {".b2z"} + + +def split_container_path(path): + """Split a request path at a container-file boundary. + + A ``.b2z`` may hold a TreeStore, so a request path can descend *into* it, + e.g. ``@public/dir/tree.b2z/level1/ctable``. Return + ``(container_path, inner_key)`` where ``container_path`` is the ``.b2z`` + file and ``inner_key`` is a ``/...`` TreeStore key, or ``(path, None)`` when + the path does not descend into a container. + """ + parts = pathlib.Path(path).parts + for i, part in enumerate(parts): + if pathlib.PurePath(part).suffix in BLOSC2_CONTAINER_SUFFIXES and i < len(parts) - 1: + return pathlib.Path(*parts[: i + 1]), "/" + "/".join(parts[i + 1 :]) + return pathlib.Path(path), None + + +def treestore_leaves(tree, prefix="/"): + """Full leaf keys (e.g. ``/g/a``) under ``prefix`` of an open TreeStore. + + Leaves are nodes with no children (groups are skipped), matching the + file-only semantics of :func:`walk_files` for directories. + """ + return [d for d in tree.get_descendants(prefix) if not tree.get_children(d)] + def compress_file(path): with open(path, "rb") as src: @@ -88,7 +116,9 @@ def getter(o, k): return model_class(**data) -def read_metadata(obj): +def read_metadata(obj, mtime=None): + # `mtime` is used when `obj` is an already-opened object (e.g. a container + # leaf) with no file of its own; callers pass the container's mtime. # Open dataset if isinstance(obj, pathlib.Path): path = obj @@ -120,8 +150,13 @@ def read_metadata(obj): ) except RuntimeError: return get_model_from_obj(obj, models.Corrupt, mtime=mtime, error="Unrecognized format") - else: - mtime = None + + # A .b2z may hold a TreeStore (a hierarchical container); it is browsed + # as a directory (its leaves are addressed as inner paths), so present + # the container itself like a plain file (as we do for HDF5). + if isinstance(obj, blosc2.TreeStore): + return get_model_from_obj(path, models.File, mtime=mtime, size=size) + # else: obj is an already-opened object; keep the caller-supplied mtime # Read metadata if isinstance(obj, blosc2.ndarray.NDArray): From 4a7e9545ec7168311b7fbb741b10e9c18602701f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 19:07:09 +0200 Subject: [PATCH 12/23] Browse hierarchical .b2z (TreeStore) containers and directories A .b2z may hold a TreeStore (a hierarchy of NDArray/CTable leaves), not just a single CTable. Address leaves by virtual path (tree.b2z/level1/ctable) without unfolding to disk: - split_container_path()/treestore_leaves() split a request path at the .b2z boundary and enumerate leaves. - API: list descends, info/fetch open the leaf; leaves inherit the container mtime. - Web: the tree expands a container into leaf rows; info/view tabs work on leaves. Unify group-like things behind models.Directory (kind="dir", mtime, size, nfiles): info returns it for a real directory, a TreeStore container (root group), and a virtual group inside one. Group size is summed cheaply from the .b2z zip index (no per-leaf open). Client: new Group class (browsable/indexable); Root.__getitem__ dispatches on server-reported kind (dir->Group, ctable->Table, shape->Array, else File), reusing already-fetched metadata to avoid a double info round-trip. --- caterva2/__init__.py | 3 +- caterva2/client.py | 103 +++++++++++++++++++++++++++------ caterva2/clients/cli.py | 6 ++ caterva2/models.py | 11 ++++ caterva2/services/server.py | 21 +++++-- caterva2/services/srv_utils.py | 17 +++++- 6 files changed, 134 insertions(+), 27 deletions(-) diff --git a/caterva2/__init__.py b/caterva2/__init__.py index e1741c86..48ee42b6 100644 --- a/caterva2/__init__.py +++ b/caterva2/__init__.py @@ -9,7 +9,7 @@ """Caterva2 - On demand access to remote Blosc2 data repositories""" -from .client import Array, BasicAuth, Client, Dataset, File, Root, Table +from .client import Array, BasicAuth, Client, Dataset, File, Group, Root, Table __version__ = "2025.12.4.dev0" """The version in use of the Caterva2 package.""" @@ -20,6 +20,7 @@ "Client", "Dataset", "File", + "Group", "Root", "Table", ] diff --git a/caterva2/client.py b/caterva2/client.py index 4149c16b..fbd1949d 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -125,18 +125,18 @@ def __getitem__(self, path): path = path.as_posix() if hasattr(path, "as_posix") else str(path) if path.endswith((".b2nd", ".b2frame")): return Array(self, path) - if path.endswith(".b2z") or ".b2z/" in path: - # A .b2z may be a CTable or a TreeStore (a folder whose leaves are - # addressed by inner path). Inner leaves carry no file suffix, so the - # server's metadata is what tells us the kind. - meta = self.client.get_info(f"{self.name}/{path}") - if meta.get("kind") == "ctable": - return Table(self, path) - if "shape" in meta: - return Array(self, path) - # A TreeStore container itself: browse via get_list / inner paths. - return File(self, path) - return File(self, path) + # Anything else (a directory, a .b2z CTable/TreeStore, a virtual group or + # leaf inside one, or a plain file) has no reliable suffix, so the + # server's metadata is what tells us the kind. + meta = self.client.get_info(f"{self.name}/{path}") + kind = meta.get("kind") + if kind == "dir": + return Group(self, path, meta) + if kind == "ctable": + return Table(self, path, meta) + if "shape" in meta: + return Array(self, path, meta) + return File(self, path, meta) def __contains__(self, path): """ @@ -281,7 +281,7 @@ def load_from_url(self, urlpath, remotepath=None): class File: - def __init__(self, root, path): + def __init__(self, root, path, meta=None): """ Represents a file, which can be a Blosc2 dataset or a regular file on a root repository. @@ -317,8 +317,16 @@ def __init__(self, root, path): _, root = _format_paths(root.urlbase, root.name) self.name = name self.path = pathlib.PurePosixPath(f"{self.root}/{self.name}") - self.meta = self.root.client._get( - f"{self.urlbase}/api/info/{self.path}", auth_cookie=self.cookie, timeout=self.root.client.timeout + # `meta` may be supplied by the caller (e.g. Root.__getitem__ already + # fetched it to pick the class) to avoid a redundant /api/info round-trip. + self.meta = ( + meta + if meta is not None + else self.root.client._get( + f"{self.urlbase}/api/info/{self.path}", + auth_cookie=self.cookie, + timeout=self.root.client.timeout, + ) ) # TODO: 'cparams' is not always present (e.g. for .b2nd files) # print(f"self.meta: {self.meta['cparams']}") @@ -590,7 +598,7 @@ def __repr__(self): class Array(Dataset, blosc2.Operand): - def __init__(self, root, path): + def __init__(self, root, path, meta=None): """ Represents an array dataset (.b2nd, .b2frame) within a Blosc2 container. @@ -619,7 +627,7 @@ def __init__(self, root, path): >>> ds.blocks (10,) """ - super().__init__(root, path) + super().__init__(root, path, meta) def __str__(self): return self.path.as_posix() @@ -793,6 +801,67 @@ def head(self, n=5): return self.rows(0, n) +class Group: + """A browsable group (HDF5 jargon): a directory, a TreeStore ``.b2z``, or a + virtual group inside one. Children are addressed by (inner) path. + + Not instantiated directly; obtained from a :class:`Root`, e.g. + ``root['dir1']`` or ``root['tree.b2z/level1']``. + """ + + def __init__(self, root, path, meta=None): + self.root = root + _, name = _format_paths(root.urlbase, path) + _, rootname = _format_paths(root.urlbase, root.name) + self.name = name + self.path = pathlib.PurePosixPath(f"{rootname}/{self.name}") + self._meta = meta + + @property + def client(self): + return self.root.client + + @property + def urlbase(self): + return self.client.urlbase + + @property + def cookie(self): + return self.client.cookie + + @property + def meta(self): + if self._meta is None: + self._meta = self.client.get_info(str(self.path)) + return self._meta + + @property + def file_list(self): + """Deep list of leaf paths under this group (relative to it).""" + return self.client.get_list(str(self.path)) + + def __iter__(self): + return iter(self.file_list) + + def __len__(self): + return len(self.file_list) + + def __contains__(self, item): + item = item.as_posix() if hasattr(item, "as_posix") else str(item) + return item in self.file_list + + def __getitem__(self, item): + """Retrieve a child (dataset, file, or subgroup) by relative path.""" + item = item.as_posix() if hasattr(item, "as_posix") else str(item) + return self.root[f"{self.name}/{item}"] + + def __repr__(self): + return f"" + + def __str__(self): + return self.path.as_posix() + + class BasicAuth: """ Basic authentication for HTTP requests. diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index 7833b08d..b460bc15 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -200,6 +200,12 @@ def _filter_names(fl): names.append(f"f{f}") return names + if data.get("kind") == "dir": + print(f"nfiles : {data.get('nfiles')}") + print(f"size : {_human_bytes(data.get('size'))}") + print(f"mtime : {data.get('mtime')}") + return + if data.get("kind") == "ctable": nbytes = data.get("nbytes") cbytes = data.get("cbytes") diff --git a/caterva2/models.py b/caterva2/models.py index fbd157c1..90f14f38 100644 --- a/caterva2/models.py +++ b/caterva2/models.py @@ -115,6 +115,17 @@ class File(pydantic.BaseModel): size: int +class Directory(pydantic.BaseModel): + """A group-like container: a real directory, a TreeStore .b2z, or a + virtual group inside one. ``size`` is None when it is not cheap to compute + (virtual groups).""" + + kind: str = "dir" + mtime: datetime.datetime | None + size: int | None = None + nfiles: int + + class Root(pydantic.BaseModel): name: str diff --git a/caterva2/services/server.py b/caterva2/services/server.py index b8bb5248..29397843 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -412,13 +412,22 @@ async def get_info( container_path, inner_key = srv_utils.split_container_path(path) abspath = get_abspath(container_path, user) if inner_key is not None: - # A member inside a container (e.g. a TreeStore .b2z leaf). - leaf = blosc2.open(abspath)[inner_key] - if isinstance(leaf, blosc2.TreeStore): - srv_utils.raise_not_found() # a group node is directory-like; use /api/list - return srv_utils.read_metadata(leaf, mtime=abspath.stat().st_mtime) + # A member inside a container (e.g. a TreeStore .b2z leaf or group). + tree = blosc2.open(abspath) + node = tree[inner_key] + if isinstance(node, blosc2.TreeStore): + # A virtual group: report leaf count and on-disk size (summed from + # the .b2z zip index, no per-leaf open). + return models.Directory( + mtime=abspath.stat().st_mtime, + size=srv_utils.treestore_size(tree, inner_key), + nfiles=len(srv_utils.treestore_leaves(tree, inner_key)), + ) + return srv_utils.read_metadata(node, mtime=abspath.stat().st_mtime) if abspath.is_dir(): - srv_utils.raise_not_found() + files = list(srv_utils.walk_files(abspath)) + size = sum(f.stat().st_size for f, _ in files) + return models.Directory(mtime=abspath.stat().st_mtime, size=size, nfiles=len(files)) return srv_utils.read_metadata(abspath) diff --git a/caterva2/services/srv_utils.py b/caterva2/services/srv_utils.py index 98ff3e2a..e29a7394 100644 --- a/caterva2/services/srv_utils.py +++ b/caterva2/services/srv_utils.py @@ -65,6 +65,17 @@ def treestore_leaves(tree, prefix="/"): return [d for d in tree.get_descendants(prefix) if not tree.get_children(d)] +def treestore_size(tree, prefix="/"): + """On-disk size (bytes) of leaves under ``prefix``, summed cheaply from the + ``.b2z`` zip index without opening any leaf. Returns None if unavailable.""" + get_offsets = getattr(tree, "_get_zip_offsets", None) + if get_offsets is None: + return None + rel = prefix.strip("/") + rel = f"{rel}/" if rel else "" + return sum(info.get("length", 0) for m, info in get_offsets().items() if m.startswith(rel)) + + def compress_file(path): with open(path, "rb") as src: data = src.read() @@ -152,10 +163,10 @@ def read_metadata(obj, mtime=None): return get_model_from_obj(obj, models.Corrupt, mtime=mtime, error="Unrecognized format") # A .b2z may hold a TreeStore (a hierarchical container); it is browsed - # as a directory (its leaves are addressed as inner paths), so present - # the container itself like a plain file (as we do for HDF5). + # as a group (its leaves are addressed as inner paths), so present the + # container itself as a directory (the root group). if isinstance(obj, blosc2.TreeStore): - return get_model_from_obj(path, models.File, mtime=mtime, size=size) + return models.Directory(mtime=mtime, size=size, nfiles=len(treestore_leaves(obj, "/"))) # else: obj is an already-opened object; keep the caller-supplied mtime # Read metadata From af2cce620dff2e889492b0d3e611c386caf01180 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 1 Jul 2026 19:18:19 +0200 Subject: [PATCH 13/23] Add plan for virtual descent for h5 too --- plans/b2z-h5-tree.md | 107 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 plans/b2z-h5-tree.md diff --git a/plans/b2z-h5-tree.md b/plans/b2z-h5-tree.md new file mode 100644 index 00000000..1f7a9787 --- /dev/null +++ b/plans/b2z-h5-tree.md @@ -0,0 +1,107 @@ +# Hierarchical containers in Caterva2 (`.b2z` TreeStore, and `.h5`) + +Design notes + plan for browsing/querying hierarchical containers via **virtual +descent** (address inner members by path, no unfolding to disk). + +## Background + +A `.b2z` may hold a **TreeStore** (a hierarchy of NDArray/CTable leaves), not +just a single `CTable`. Rather than the old HDF5 `unfold` approach (walk the +container and write proxy files replicating the hierarchy), we address leaves by +virtual path, e.g. `@personal/tree-store.b2z/level1/ctable`, resolving on the fly. + +Key asymmetry: TreeStore leaves are already native blosc2 objects +(`tree[key]` → NDArray/CTable), so the seam is cheap. HDF5 leaves are not, so +they must be wrapped in an `HDF5Proxy` — which today writes a `.b2nd` proxy file +(the unfold path). Virtual descent for `.h5` needs a *file-less* proxy. + +## Done (branch `new-table`) — `.b2z` TreeStore + +Implemented and tested (`caterva2/tests/test_treestore.py`), full suite green. + +Core seam (`srv_utils.py`): +- `split_container_path(path)` → `(container_path, inner_key)`, split at the + `.b2z` boundary. `BLOSC2_CONTAINER_SUFFIXES = {".b2z"}` is the knob. +- `treestore_leaves(tree, prefix)` → deep leaf keys (groups skipped). +- `treestore_size(tree, prefix)` → on-disk size summed from the `.b2z` zip index + (`_get_zip_offsets`, guarded), no per-leaf open. +- `read_metadata(obj, mtime=None)` — TreeStore container → `models.Directory`; + accepts an mtime for opened leaf objects (leaves inherit the container mtime). + +API (`server.py`): `get_list` descends; `get_info`/`fetch_data` open +`tree[inner_key]` (leaf → metadata/slice; group → `Directory`); `fetch` guards +the whole-file `FileResponse` fast path so it never streams the whole container. + +Web (`server.py`): `htmx_path_list` expands a container into one row per leaf; +`htmx_path_info`/`htmx_path_view` split + open the leaf. + +Groups unified via `models.Directory{kind:"dir", mtime, size, nfiles}`: +`get_info` returns it for a real directory, a TreeStore container (root group), +and a virtual group inside one. Size is cheap (zip index for `.b2z` groups; +aggregate file stat for real dirs). + +Client (`client.py`): new `Group` class (browsable/indexable; exported from +`caterva2`). `Root.__getitem__` dispatches on server `kind` +(dir→`Group`, ctable→`Table`, shape→`Array`, else `File`) for any +non-`.b2nd`/`.b2frame` path; `File`/`Array` accept `meta=` to avoid a double +`/api/info` round-trip. + +## Open questions + +1. **Route `.h5` through the same seam** (chosen direction; plan below). Would + let us eventually retire the `unfold` proxy-file path. +2. **Web cosmetics** (minor, marked with `ponytail:` comments): + - Each expanded leaf row in the web tree shows the *container's* size + (`add_dataset` stats the container file). Would need per-leaf size. + - Filter/sort on a TreeStore leaf falls back to unfiltered (default view is + fine). +3. **Client perf tradeoff** (accepted, not a bug): `Root.__getitem__` now does + one `/api/info` call for any non-`.b2nd`/`.b2frame` path (dirs, `.b2z`, plain + files). Deduped via `meta=`, but still one round-trip where plain files used + to be lazy. Revisit only if a hot path does many `root['plainfile']` lookups. +4. **Not committed** — all changes are in the working tree on `new-table`. + +## Plan: route `.h5` through the seam (~70–90 lines, mostly `hdf5.py`) + +More than `.b2z` (which was ~1 helper) because HDF5 isn't native, but bounded — +the heavy lifting (HDF5Proxy, chunk readers, `to_cframe`) already exists from the +unfold work. Feasibility of the two hard parts verified with a scratch h5 file: +`visititems` enumerates leaves cheaply, `dset.id.get_storage_size()` gives +per-dataset size, and `blosc2.empty(shape, dtype, **b2args)` with no urlpath +yields an in-memory b2arr carrying cparams. + +1. **File-less proxy** (~15 lines, `hdf5.py`) — factory `HDF5Proxy.open(h5file, + dsetname)` that sets `self.dset` to the live h5 dataset and builds an + in-memory `self.b2arr = blosc2.empty(shape, dtype, **b2args_from_h5dset(...))` + (no urlpath → no disk write). Reuses existing `slice`/`to_cframe`/properties + (`slice` uses `self.dset` + `self.b2arr.cparams`, both satisfied). This is the + one genuinely new mechanism. +2. **Enumerate + size helpers** (~12 lines, `hdf5.py`) — `hdf5_leaves(path, + prefix)` via `visititems`; `hdf5_size(path, prefix)` via + `dset.id.get_storage_size()`. Analogs of `treestore_leaves`/`treestore_size`. +3. **`read_metadata`** (~8 lines) — plain `.h5` (no inner key) → `Directory` + (currently `File`) with nfiles/size; add an `HDF5Proxy` branch for a leaf + object. +4. **Server descent** (~25 lines) — `get_list`/`get_info`/`fetch_data` currently + hardcode `.b2z` + `blosc2.open` + TreeStore. Add `.h5` siblings; cleanest is a + tiny "container adapter" (`open → is-group? Directory : leaf-proxy`) so both + formats share the flow. +5. **`split_container_path`** — add `.h5`, `.hdf5` to + `BLOSC2_CONTAINER_SUFFIXES` (1 line; `get_abspath` already permits `.h5`). +6. **Web** (~8 lines) — generalize the path-list leaf-row expansion to `.h5`. +7. **Client** — zero change; already dispatches on server `kind`. + +### Decisions / risks +- **Keep `unfold` intact** in this change; virtual descent and unfold both target + the same `.h5` differently. Retiring unfold (drop the command + + `create_hdf5_proxies` + the proxy-file reconstruct mode of `HDF5Proxy`) is a + clean follow-up once this proves out. +- **Compat**: `read_metadata` on a plain `.h5` flips `File`→`Directory`; a couple + of `test_hdf5_proxy` assertions may expect `File` — check/adjust. +- **Incompatible leaves** (compound/vlen/scalar h5 datasets): + `h5dset_is_compatible` exists; skip or surface them in enumeration rather than + crash. + +### Suggested increments +- Spike: file-less proxy (#1) + enumeration (#2) alone, verified with a test. +- Then wire server descent (#3–#5), then web (#6). From 04ba21362c3bbe06783dba76a4d72839743e1125 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 2 Jul 2026 11:05:34 +0200 Subject: [PATCH 14/23] Add mountable virtual roots for .b2z TreeStore containers in web UI A .b2z TreeStore now shows as a single mountable row in the datasets list instead of auto-expanding into one row per leaf (which flooded the list). Clicking the plug icon "mounts" it as a virtual root alongside @personal/@shared/@public, with its own checkbox and an unmount control; checking it lists that container's leaves. Mount state lives client-side in localStorage (key caterva2:mounted), bridged to the server via an htmx:configRequest listener that adds `mounted=` params to the root-list request. No new endpoints, DB, or per-user server state. - server.py: htmx_root_list accepts `mounted` and filters it through get_rootdir_or_none; htmx_path_list renders TreeStores as single mountable rows and expands mounted containers into leaf rows. - templates: root_list.html renders mounted roots, path_list.html adds the plug button, home.html holds the mount/unmount JS. - Rename Directory.kind "dir" -> "group" (models/client/cli). Review fixes: - Avoid stored XSS: read paths from data-* attributes at click time instead of interpolating into inline handler JS source. - Don't 500 the listing on a corrupt/non-TreeStore/stale .b2z (untrusted localStorage input); skip it in both the walk and virtual-root loops. - Dedup roots in mountRoot so a repeat click can't double-list leaves. - stat() the container once per mount instead of once per leaf. - Update test_treestore.py for single-row behavior; add coverage for virtual-root leaf expansion and bogus-container safety. --- caterva2/client.py | 2 +- caterva2/clients/cli.py | 2 +- caterva2/models.py | 2 +- caterva2/services/server.py | 86 +++++++++++++++++----- caterva2/services/templates/home.html | 66 +++++++++++++++++ caterva2/services/templates/path_list.html | 6 +- caterva2/services/templates/root_list.html | 15 ++++ 7 files changed, 157 insertions(+), 22 deletions(-) diff --git a/caterva2/client.py b/caterva2/client.py index fbd1949d..30899db0 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -130,7 +130,7 @@ def __getitem__(self, path): # server's metadata is what tells us the kind. meta = self.client.get_info(f"{self.name}/{path}") kind = meta.get("kind") - if kind == "dir": + if kind == "group": return Group(self, path, meta) if kind == "ctable": return Table(self, path, meta) diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index b460bc15..2277b59f 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -200,7 +200,7 @@ def _filter_names(fl): names.append(f"f{f}") return names - if data.get("kind") == "dir": + if data.get("kind") == "group": print(f"nfiles : {data.get('nfiles')}") print(f"size : {_human_bytes(data.get('size'))}") print(f"mtime : {data.get('mtime')}") diff --git a/caterva2/models.py b/caterva2/models.py index 90f14f38..1041fdac 100644 --- a/caterva2/models.py +++ b/caterva2/models.py @@ -120,7 +120,7 @@ class Directory(pydantic.BaseModel): virtual group inside one. ``size`` is None when it is not cheap to compute (virtual groups).""" - kind: str = "dir" + kind: str = "group" mtime: datetime.datetime | None size: int | None = None nfiles: int diff --git a/caterva2/services/server.py b/caterva2/services/server.py index 29397843..59d27c79 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -1625,11 +1625,21 @@ async def htmx_root_list( request: Request, # Query roots: list[str] = fastapi.Query([]), + mounted: list[str] = fastapi.Query([]), # Depends user: db.User = Depends(optional_user), ): + seen = set() + mounted_ok = [] + for path in mounted: + prefix = path.split("/", 1)[0] + if get_rootdir_or_none(prefix, user) is not None and path not in seen: + seen.add(path) + mounted_ok.append(path) + context = { "checked": roots, + "mounted": mounted_ok, "user": user, } return templates.TemplateResponse(request, "root_list.html", context) @@ -1694,14 +1704,15 @@ def get_names(): datasets = [] query = {"roots": roots, "search": search} - def add_dataset(path, abspath): + def add_dataset(path, abspath, mountable=False, size=None): datasets.append( { "name": "_", "path": path, - "size": abspath.stat().st_size, + "size": abspath.stat().st_size if size is None else size, "url": make_url(request, "html_home", path=path, query=query), "label": truncate_path(path), + "mountable": mountable, } ) @@ -1710,19 +1721,48 @@ def add_dataset(path, abspath): if relpath.suffix == ".b2": relpath = relpath.with_suffix("") path = f"{root}/{relpath}" - # A .b2z holding a TreeStore is browsed as a folder: expand it into - # one row per leaf (e.g. tree.b2z/level1/ctable). + # A .b2z holding a TreeStore shows as a single mountable row; its + # leaves are browsed once it's mounted as a virtual root. A corrupt + # or non-container .b2z (uploads aren't validated) falls through to + # a plain row instead of crashing the whole listing. if relpath.suffix == ".b2z": - tree = blosc2.open(abspath) + try: + tree = blosc2.open(abspath) + except Exception: + tree = None if isinstance(tree, blosc2.TreeStore): - for key in srv_utils.treestore_leaves(tree): - leaf_path = f"{path}{key}" - if search in leaf_path: - add_dataset(leaf_path, abspath) + if search in path: + add_dataset(path, abspath, mountable=True) continue if search in path: add_dataset(path, abspath) + # Virtual roots: mounted .b2z TreeStore containers, expanded into leaves. + for root in roots: + proot = pathlib.PurePosixPath(root) + if proot.suffix not in srv_utils.BLOSC2_CONTAINER_SUFFIXES: + continue # classic roots handled by filter_roots above + rootdir = get_rootdir_or_none(proot.parts[0], user) + if rootdir is None: + continue + abspath = (rootdir / pathlib.Path(*proot.parts[1:])).resolve() + if rootdir.resolve() not in abspath.parents: + continue + # `root` is untrusted (client localStorage): a stale, non-container, or + # corrupt path must skip silently, not 500 the whole listing. blosc2.open + # raises OSError (missing/dir), zipfile.BadZipFile, or RuntimeError here. + try: + tree = blosc2.open(abspath) + except Exception: + continue + if not isinstance(tree, blosc2.TreeStore): + continue + size = abspath.stat().st_size # same container file for every leaf + for key in srv_utils.treestore_leaves(tree): + leaf_path = f"{root}{key}" + if search in leaf_path: + add_dataset(leaf_path, abspath, size=size) + # Add current path if not already in the list current_path = hx_current_url.path segments = current_path.segments @@ -1735,12 +1775,18 @@ def add_dataset(path, abspath): root = segments[1] rootdir = get_rootdir_or_none(root, user) if rootdir is not None: - relpath = pathlib.Path(*segments[2:]) - abspath = rootdir / relpath - if abspath.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES: - abspath = pathlib.Path(f"{abspath}.b2") + # Path may descend into a container (e.g. an unmounted TreeStore + # leaf); size/stat then come from the container file itself. + container_path, inner_key = srv_utils.split_container_path(path) + if inner_key is not None: + abspath = rootdir / pathlib.Path(*container_path.parts[1:]) + else: + relpath = pathlib.Path(*segments[2:]) + abspath = rootdir / relpath + if abspath.suffix not in srv_utils.BLOSC2_NATIVE_SUFFIXES: + abspath = pathlib.Path(f"{abspath}.b2") - with contextlib.suppress(FileNotFoundError): + with contextlib.suppress(FileNotFoundError, NotADirectoryError): add_dataset(path, abspath) # Assign names to datasets @@ -1804,10 +1850,14 @@ async def htmx_path_info( # Read metadata (a path may descend into a container, e.g. a TreeStore .b2z leaf) container_path, inner_key = srv_utils.split_container_path(path) abspath = get_abspath(container_path, user) - if inner_key is not None: - meta = srv_utils.read_metadata(blosc2.open(abspath)[inner_key], mtime=abspath.stat().st_mtime) - else: - meta = srv_utils.read_metadata(abspath) + try: + if inner_key is not None: + meta = srv_utils.read_metadata(blosc2.open(abspath)[inner_key], mtime=abspath.stat().st_mtime) + else: + meta = srv_utils.read_metadata(abspath) + except FileNotFoundError: + # e.g. a bare root (@personal) or another directory with no dataset of its own + raise fastapi.HTTPException(status_code=404) from None # NotFound # Context current_url = push_url or hx_current_url diff --git a/caterva2/services/templates/home.html b/caterva2/services/templates/home.html index 263c6afb..daa46589 100644 --- a/caterva2/services/templates/home.html +++ b/caterva2/services/templates/home.html @@ -91,6 +91,72 @@ {% endif %}